ambar2595
New Contributor III

I'm using DBR 16.3 and all partitions are still being deleted. This is the code I'm using. No success. 

 

spark = (
        SparkSession.builder.config("spark.datasource.bigquery.intermediateFormat", "orc")
        .config("spark.sql.sources.partitionOverwriteMode", "dynamic")
        .getOrCreate()
    )
visiting_client_day = (
        spark.read.format("delta")
        .load("s3://bucket-2/gold/visiting_client_day")
        .where(col("date_utc") == lit("2025-05-04"))
    )
(
        visiting_client_day.write.format("bigquery")
        .option("parentProject", "parentProject")
        .option("project", "project")
        .option("temporaryGcsBucket", "bucket")
        .mode("overwrite")
        .option("table", FINAL_TABLE)
        .save()
    )