This is my approach:

from databricks.sdk.runtime import dbutils
from pyspark.sql.types import DataFrame

output_base_url = "abfss://..."


def write_single_parquet_file(df: DataFrame, filename: str):
    print(f"Writing '{filename}.parquet' to ABFS")
    filepath = f"{output_base_url}/{filename}.parquet"
    temp_filepath = f"{output_base_url}/temp_{filename}.parquet"

    # Write a temporary folder containing one Parquet file
    df.repartition(1).write.format("parquet").save(temp_filepath, mode="overwrite")

    # Find the Parquet file in the temporary folder
    files = dbutils.fs.ls(temp_filepath)
    output_file = next(x for x in files if x.name.startswith("part-"))

    # Delete filepath, if it exists
    try:
        dbutils.fs.ls(filepath)
        # Filepath exists
        print(f"Deleting old {filepath}")
        dbutils.fs.rm(filepath, recurse=True)
    except Exception as e:
        if "java.io.FileNotFoundException" not in str(e):
            raise e

    # Move the Parquet file to the final location
    dbutils.fs.mv(output_file.path, filepath)

    # Delete temporary folder
    dbutils.fs.rm(temp_filepath, True)
    print(f"Wrote '{filepath}'")