artsheiko
Databricks Employee
Databricks Employee

Hi, it seems you do not need a pandas udf here. Try the following :

import numpy as np
from pyspark.sql.types import FloatType
from pyspark.sql import functions as f
 
data = [{"Identifier": 123, "RawData": "1,2,4,2,34,6,7,8"},
        {"Identifier": 456, "RawData": "4,5,7,8,9,3,4,7,8"}]
 
df = spark.createDataFrame(data)
 
series_mean = f.udf(lambda x: float(np.mean(x)), FloatType()) # replace by Metric1 logic
series_max = f.udf(lambda x: float(np.max(x)), FloatType()) # replace by Metric2 logic
 
df = (df
      .withColumn("series_int", f.split(f.col('RawData'), ',').cast('array<int>'))
      .withColumn("mean", series_mean("series_int"))
      .withColumn("max", series_max("series_int"))
     )
 
display(df)