Comment
01-16-2025
01:33 PM
01-16-2025
01:33 PM
For the change feed section, I had to modify the code a bit as spark.cosmos.changeFeed.checkpointLocation was not valid.
checkpoint_path = "dbfs:/mnt/checkpoints/cosmoschangefeed"
# Configure change feed options
changeFeedConfig = {
"spark.cosmos.changeFeed.startFrom": "Beginning",
"spark.cosmos.changeFeed.mode": "Incremental",
"spark.cosmos.changeFeed.batchCheckpointLocation": checkpoint_path,
"spark.cosmos.accountEndpoint": cosmosEndpoint,
"spark.cosmos.accountKey": cosmosMasterKey,
"spark.cosmos.database": cosmosDatabaseName,
"spark.cosmos.container": cosmosContainerName,
"spark.cosmos.read.partitioning.strategy": "Default",
"spark.cosmos.read.inferSchema.enabled" : "true",
}