Let us try to simulate this error

from pyspark.sql import SparkSession
import os

# Create a SparkSession
spark = SparkSession.builder.appName("SomeTestsForIncompleteWriteSimulation").getOrCreate()

# Sample DataFrame
data = [("A", 1), ("B", 2), ("A", 3), ("C", 5)]
df = spark.createDataFrame(data, ["col1", "col2"])

# Simulate an error during write
try:
  df.write.partitionBy("col1").mode("overwrite").csv("/tmp/output", header=True)
except Exception as e:
  print("Simulating write error:", e)

# Check for existence of "_SUCCESS" file in local /tmp
success_file = "/tmp/output/_SUCCESS"
if os.path.exists(success_file):
  print("_SUCCESS file found (might not reflect reality if error occurred earlier)")
else:
  print("_SUCCESS file missing (indicates incomplete write)")

and the output

_SUCCESS file missing (indicates incomplete write)