Hi all, I just run the code today out of curiosity and it just worked without any Exception. I used a single node cluster with DBR 15.4 ML (Spark 3.5.0).
Here is my code to confirm:
from pyspark.sql import Row
from pyspark.sql.functions import lit
from pyspark.ml.feature import VectorAssembler
# Create sample data
data = [
Row(feature1=1.0, feature2=2.0),
Row(feature1=3.0, feature2=4.0)
]
# Create DataFrame
df = spark.createDataFrame(data)
df = df.withColumn("pcainput_valgbp_avg_y1", lit(0))
# Initialize VectorAssembler
assembler = VectorAssembler(
inputCols=["feature1", "feature2", "pcainput_valgbp_avg_y1"],
outputCol="features"
)
# Transform the DataFrame
output_df = assembler.transform(df)
display(output_df)
.png){kind=link}