Hi,
i am getting data from event hub and stored in delta table as a row table, i data i received in json , the problem i data i have different schema in each row but i code i use it take first row a json schema i am stuck how to do please any one guide me
from pyspark.sql.functions import explode, from_json, col,schema_of_json
from pyspark.sql.types import ArrayType, StringType
from pyspark.sql import functions as F
from pyspark.sql.types import *
sample_json = df.select("Value").first()[0]
json_schema = schema_of_json(sample_json)
print(json_schema)
value_df = (
df.select(
from_json(col("Value").cast("string"), json_schema).alias("analytics")
)
.select(explode(col("analytics")).alias("Value"))
).select("Value.customer_id")
value_df.display()
