It turns out my spark config was wrong

    #Set Spark configuration

    configs = {"fs.azure.account.auth.type": "OAuth",

          "fs.azure.account.oauth.provider.type": "org.apache.hadoop.fs.azurebfs.oauth2.ClientCredsTokenProvider",

          "fs.azure.account.oauth2.client.id": client_ID,

          "fs.azure.account.oauth2.client.secret": client_secret,

          "fs.azure.account.oauth2.client.endpoint": f"https://login.microsoftonline.com/{tenant_ID}/oauth2/token"}