Have you considered using session scopes instead of cluster scopes? I have a function stored at databricks. functions. azure. py that does this:

from pyspark.sql import SparkSession
 
def set_session_scope(scope: str, client_id: str, client_secret: str, tenant_id: str, storage_account_name: str, container_name: str) -> str:
    
    """Connects to azure key vault, authenticates, and sets spark session to use specified service principal for read/write to adls
    
    Args:
        scope: The azure key vault scope name
        client_id: The key name of the secret for the client id
        client_secret: The key name of the secret for the client secret
        tenant_id: The key name of the secret for the tenant id
        storage_account_name: The name of the storage account resource to read/write from
        container_name: The name of the container resource in the storage account to read/write from
 
    Returns:
        Spark configs get set appropriately
        abfs_path (string): The abfss:// path to the storage account and container
    """
 
    spark = SparkSession.builder.getOrCreate()
 
    try:
        from pyspark.dbutils import DBUtils
        dbutils = DBUtils(spark)
    except ImportError:
        import IPython
        dbutils = IPython.get_ipython().user_ns["dbutils"]
 
    client_id = dbutils.secrets.get(scope = scope, key = client_id)
    client_secret = dbutils.secrets.get(scope = scope, key = client_secret)
    tenant_id = dbutils.secrets.get(scope = scope, key = tenant_id)
 
    spark.conf.set(f"fs.azure.account.auth.type.{storage_account_name}.dfs.core.windows.net", "OAuth")
    spark.conf.set(f"fs.azure.account.oauth.provider.type.{storage_account_name}.dfs.core.windows.net", "org.apache.hadoop.fs.azurebfs.oauth2.ClientCredsTokenProvider")
    spark.conf.set(f"fs.azure.account.oauth2.client.id.{storage_account_name}.dfs.core.windows.net", client_id)
    spark.conf.set(f"fs.azure.account.oauth2.client.secret.{storage_account_name}.dfs.core.windows.net", client_secret)
    spark.conf.set(f"fs.azure.account.oauth2.client.endpoint.{storage_account_name}.dfs.core.windows.net", f"https://login.microsoftonline.com/{tenant_id}/oauth2/token")
    
    abfs_path = "abfss://" + container_name + "@" + storage_account_name + ".dfs.core.windows.net/"
    
    return abfs_path

And its usage is like this:

from databricks.functions.azure import set_session_scope
# Set session scope and connect to abfss to read source data
 
client_id = "databricks-serviceprincipal-id"
client_secret = "databricks-serviceprincipal-secret"
tenant_id = "tenant-id"
storage_account_name = "your-storage-account-name"
container_name = "your-container-name"
folder_path = "" #path/to/folder/
 
abfs_path = set_session_scope(
    scope = scope,
    client_id = client_id, 
    client_secret = client_secret, 
    tenant_id = tenant_id, 
    storage_account_name = storage_account_name, 
    container_name = container_name 
)
 
file_list = dbutils.fs.ls(abfs_path + folder_path)