Intermittent failures with auto loader on Azure

Malthe
Valued Contributor II

We've been using auto loader to ingest data from a storage account on Azure (format "cloudFiles").

Today, we're starting to see failures during the setup of event notification:

25/09/11 19:06:28 ERROR MicroBatchExecution: Non-interrupted exception thrown for queryId=[REDACTED],runId=[REDACTED]: org.json4s.MappingException: Do not know how to convert JArray(List(JString([REDACTED]))) into class java.lang.String
org.json4s.MappingException: Do not know how to convert JArray(List(JString([REDACTED]))) into class java.lang.String
	at org.json4s.reflect.package$.fail(package.scala:53)
	at org.json4s.Extraction$.convert(Extraction.scala:888)
	at org.json4s.Extraction$.$anonfun$extract$10(Extraction.scala:456)
	at org.json4s.Extraction$.$anonfun$customOrElse$1(Extraction.scala:780)
	at scala.PartialFunction.applyOrElse(PartialFunction.scala:127)
	at scala.PartialFunction.applyOrElse$(PartialFunction.scala:126)
	at scala.PartialFunction$$anon$1.applyOrElse(PartialFunction.scala:257)
	at org.json4s.Extraction$.customOrElse(Extraction.scala:780)
	at org.json4s.Extraction$.extract(Extraction.scala:454)
	at org.json4s.Extraction$.org$json4s$Extraction$$extractDetectingNonTerminal(Extraction.scala:482)
	at org.json4s.Extraction$.$anonfun$extract$8(Extraction.scala:426)
	at scala.collection.immutable.List.map(List.scala:297)
	at org.json4s.Extraction$.$anonfun$extract$7(Extraction.scala:424)
	at org.json4s.Extraction$.$anonfun$customOrElse$1(Extraction.scala:780)
	at scala.PartialFunction.applyOrElse(PartialFunction.scala:127)
	at scala.PartialFunction.applyOrElse$(PartialFunction.scala:126)
	at scala.PartialFunction$$anon$1.applyOrElse(PartialFunction.scala:257)
	at org.json4s.Extraction$.customOrElse(Extraction.scala:780)
	at org.json4s.Extraction$.extract(Extraction.scala:420)
	at org.json4s.Extraction$.extract(Extraction.scala:56)
	at org.json4s.ExtractableJsonAstNode.extract(ExtractableJsonAstNode.scala:22)
	at org.json4s.jackson.JacksonSerialization.read(Serialization.scala:62)
	at org.json4s.Serialization.read(Serialization.scala:31)
	at org.json4s.Serialization.read$(Serialization.scala:31)
	at org.json4s.jackson.JacksonSerialization.read(Serialization.scala:23)
	at com.databricks.sql.aqs.EventGridClient.generateAccessTokenUsingClientSecret(EventGridClient.scala:180)
	at com.databricks.sql.aqs.EventGridClient.generateAccessToken(EventGridClient.scala:238)
	at com.databricks.sql.aqs.autoIngest.AzureEventNotificationSetup$.getToken(AzureEventNotificationSetup.scala:345)
	at com.databricks.sql.aqs.autoIngest.AzureEventNotificationSetup$.$anonfun$buildStorageClient$2(AzureEventNotificationSetup.scala:387)
	at scala.Option.getOrElse(Option.scala:189)
	at com.databricks.sql.aqs.autoIngest.AzureEventNotificationSetup$.buildStorageClient(AzureEventNotificationSetup.scala:384)
	at com.databricks.sql.aqs.autoIngest.AzureEventNotificationSetup.<init>(AzureEventNotificationSetup.scala:70)
	at java.base/jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method)
	at java.base/jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77)
	at java.base/jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45)
	at java.base/java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:500)
	at java.base/java.lang.reflect.Constructor.newInstance(Constructor.java:481)
	at com.databricks.sql.fileNotification.autoIngest.EventNotificationSetup$.$anonfun$create$1(EventNotificationSetup.scala:68)
	at com.databricks.sql.fileNotification.autoIngest.ResourceManagementUtils$.unwrapInvocationTargetException(ResourceManagementUtils.scala:42)
	at com.databricks.sql.fileNotification.autoIngest.EventNotificationSetup$.create(EventNotificationSetup.scala:50)
	at com.databricks.sql.fileNotification.autoIngest.CloudFilesSourceProvider.$anonfun$createSource$2(CloudFilesSourceProvider.scala:143)
	at scala.Option.getOrElse(Option.scala:189)
	at com.databricks.sql.fileNotification.autoIngest.CloudFilesSourceProvider.createSource(CloudFilesSourceProvider.scala:128)
	at org.apache.spark.sql.execution.datasources.DataSource.createSource(DataSource.scala:346)
	at org.apache.spark.sql.execution.streaming.MicroBatchExecution$$anonfun$2.$anonfun$applyOrElse$2(MicroBatchExecution.scala:223)
...