Hi @Joost1024 ,

So here's the issue. It seems that JSON DataFrameReader expects to have a JSON object. But in your case we're dealing with JSON array at root level - not a JSON object.

So for instance,if we would just rewrite your file in following way then spark would be able to infer schema without any issues:

{
    "data": [
            [
                {
                "entity_id": "sensor.solaredge_lifetime_energy",
                "state": "19848848.0",
                "attributes": {
                    "state_class": "total",
                    "unit_of_measurement": "Wh",
                    "device_class": "energy",
                    "friendly_name": "solaredge Lifetime energy"
                },
                "last_changed": "2025-12-14T23:00:00+00:00",
                "last_updated": "2025-12-14T23:00:00+00:00"
                },
                {
                "entity_id": "sensor.solaredge_lifetime_energy",
                "state": "19849120.0",
                "attributes": {
                    "state_class": "total",
                    "unit_of_measurement": "Wh",
                    "device_class": "energy",
                    "friendly_name": "solaredge Lifetime energy"
                },
                "last_changed": "2025-12-14T23:15:00+00:00",
                "last_updated": "2025-12-14T23:15:00+00:00"
                },
                {
                "entity_id": "sensor.solaredge_lifetime_energy",
                "state": "19849580.0",
                "attributes": {
                    "state_class": "total",
                    "unit_of_measurement": "Wh",
                    "device_class": "energy",
                    "friendly_name": "solaredge Lifetime energy"
                },
                "last_changed": "2025-12-14T23:30:00+00:00",
                "last_updated": "2025-12-14T23:30:00+00:00"
                }
            ],
            [
                {
                "entity_id": "sensor.home_temperature",
                "state": "21.5",
                "attributes": {
                    "state_class": "measurement",
                    "unit_of_measurement": "°C",
                    "device_class": "temperature",
                    "friendly_name": "Home Temperature"
                },
                "last_changed": "2025-12-14T23:00:00+00:00",
                "last_updated": "2025-12-14T23:00:00+00:00"
                },
                {
                "entity_id": "sensor.home_temperature",
                "state": "21.3",
                "attributes": {
                    "state_class": "measurement",
                    "unit_of_measurement": "°C",
                    "device_class": "temperature",
                    "friendly_name": "Home Temperature"
                },
                "last_changed": "2025-12-14T23:15:00+00:00",
                "last_updated": "2025-12-14T23:15:00+00:00"
                }
            ],
            [
                {
                "entity_id": "sensor.power_consumption",
                "state": "1250.0",
                "attributes": {
                    "state_class": "measurement",
                    "unit_of_measurement": "W",
                    "device_class": "power",
                    "friendly_name": "Power Consumption"
                },
                "last_changed": "2025-12-14T23:00:00+00:00",
                "last_updated": "2025-12-14T23:00:00+00:00"
                },
                {
                "entity_id": "sensor.power_consumption",
                "state": "1180.0",
                "attributes": {
                    "state_class": "measurement",
                    "unit_of_measurement": "W",
                    "device_class": "power",
                    "friendly_name": "Power Consumption"
                },
                "last_changed": "2025-12-14T23:15:00+00:00",
                "last_updated": "2025-12-14T23:15:00+00:00"
                },
                {
                "entity_id": "sensor.power_consumption",
                "state": "1320.0",
                "attributes": {
                    "state_class": "measurement",
                    "unit_of_measurement": "W",
                    "device_class": "power",
                    "friendly_name": "Power Consumption"
                },
                "last_changed": "2025-12-14T23:30:00+00:00",
                "last_updated": "2025-12-14T23:30:00+00:00"
                },
                {
                "entity_id": "sensor.power_consumption",
                "state": "1295.0",
                "attributes": {
                    "state_class": "measurement",
                    "unit_of_measurement": "W",
                    "device_class": "power",
                    "friendly_name": "Power Consumption"
                },
                "last_changed": "2025-12-14T23:45:00+00:00",
                "last_updated": "2025-12-14T23:45:00+00:00"
                }
            ]
            ]

}

Ok, so for json arrays as a top objects we can try different approach. We can read json file as a text (important thing here - we want to use option wholeText=True to not split by new lines) and then use from_json function to parse it correctly:

from pyspark.sql.functions import lit, from_json, col, explode
from pyspark.sql.types import StructType, StructField, ArrayType, StringType


attributes_schema = StructType([
    StructField("state_class", StringType(), nullable=True),
    StructField("unit_of_measurement", StringType(), nullable=True),
    StructField("device_class", StringType(), nullable=True),
    StructField("friendly_name", StringType(), nullable=True)
])

sensor_reading_schema = StructType([
    StructField("entity_id", StringType(), nullable=True),
    StructField("state", StringType(), nullable=True),
    StructField("attributes", attributes_schema, nullable=True),
    StructField("last_changed", StringType(), nullable=True),
    StructField("last_updated", StringType(), nullable=True)
])


df_text = spark.read.text('/Volumes/logging_demo/default/logs/sample_data.json', wholetext=True)

array_schema = ArrayType(ArrayType(sensor_reading_schema))
df_parsed = df_text.select(from_json(col("value"), array_schema).alias("data"))


# df_flat = df_parsed.select(explode(col("data")).alias("inner_array")) \
#                    .select(explode(col("inner_array")).alias("sensor")) \
#                    .select("sensor.*")

display(df_parsed)

And as you can see on below screenshot - now we parsed our file correctly:

Of course you can flattened it further - just uncomment df_flat dataframe.