Hi,

I'm having a problem while trying to write to a delta table from a org.apache.spark.sql.util.QueryExecutionListener

Code:
val df = SparkSession.active().createDataFrame(batch, MyClass::class.java)
df
.write()
.mode("append")
.format("delta")
.saveAsTable("table")

Exception:
com.databricks.unity.error.MissingCredentialScopeException: [UNITY_CREDENTIAL_SCOPE_MISSING_SCOPE] Missing Credential Scope. Unity Credential Scope id not found in thread locals.. SQLSTATE: XXKUC
at com.databricks.unity.error.MissingCredentialScopeException$.withDebugLog(UCSExceptions.scala:62)
at com.databricks.unity.UCSDriver$Manager.$anonfun$currentScopeId$1(UCSDriver.scala:184)
at scala.Option.getOrElse(Option.scala:189)
at com.databricks.unity.UCSDriver$Manager.currentScopeId(UCSDriver.scala:184)
at com.databricks.unity.UCSDriver$Manager.currentScope(UCSDriver.scala:187)
at com.databricks.unity.UnityCredentialScope$.currentScope(UnityCredentialScope.scala:116)
at com.databricks.unity.UnityCredentialScope$.getSAMRegistry(UnityCredentialScope.scala:144)
at com.databricks.unity.SAMRegistry$.snapshot(SAMRegistry.scala:500)
at org.apache.spark.sql.execution.QueryExecution.$anonfun$lazyAnalyzed$3(QueryExecution.scala:303)
at com.databricks.spark.util.FrameProfiler$.record(FrameProfiler.scala:94)
at org.apache.spark.sql.catalyst.QueryPlanningTracker.measurePhase(QueryPlanningTracker.scala:532)
at org.apache.spark.sql.execution.QueryExecution.$anonfun$executePhase$5(QueryExecution.scala:659)
at org.apache.spark.sql.execution.SQLExecution$.withExecutionPhase(SQLExecution.scala:150)
at org.apache.spark.sql.execution.QueryExecution.$anonfun$executePhase$4(QueryExecution.scala:659)
at org.apache.spark.sql.execution.QueryExecution$.withInternalError(QueryExecution.scala:1299)
at org.apache.spark.sql.execution.QueryExecution.$anonfun$executePhase$2(QueryExecution.scala:652)
at com.databricks.util.LexicalThreadLocal$Handle.runWith(LexicalThreadLocal.scala:63)
at org.apache.spark.sql.execution.QueryExecution.$anonfun$executePhase$1(QueryExecution.scala:649)
at org.apache.spark.sql.SparkSession.withActive(SparkSession.scala:1450)
at org.apache.spark.sql.execution.QueryExecution.executePhase(QueryExecution.scala:649)
at org.apache.spark.sql.execution.QueryExecution.$anonfun$lazyAnalyzed$2(QueryExecution.scala:287)
at com.databricks.sql.util.MemoryTrackerHelper.withMemoryTracking(MemoryTrackerHelper.scala:80)
at org.apache.spark.sql.execution.QueryExecution.$anonfun$lazyAnalyzed$1(QueryExecution.scala:286)
at scala.util.Try$.apply(Try.scala:213)
at org.apache.spark.util.Utils$.doTryWithCallerStacktrace(Utils.scala:1683)
at org.apache.spark.util.Utils$.getTryWithCallerStacktrace(Utils.scala:1744)
at org.apache.spark.util.LazyTry.get(LazyTry.scala:58)
at org.apache.spark.sql.execution.QueryExecution.analyzed(QueryExecution.scala:320)
at org.apache.spark.sql.execution.QueryExecution.assertAnalyzed(QueryExecution.scala:266)
at org.apache.spark.sql.Dataset$.$anonfun$ofRows$1(Dataset.scala:108)
at org.apache.spark.sql.SparkSession.withActive(SparkSession.scala:1450)
at org.apache.spark.sql.SparkSession.$anonfun$withActiveAndFrameProfiler$1(SparkSession.scala:1457)
at com.databricks.spark.util.FrameProfiler$.record(FrameProfiler.scala:94)
at org.apache.spark.sql.SparkSession.withActiveAndFrameProfiler(SparkSession.scala:1457)
at org.apache.spark.sql.Dataset$.ofRows(Dataset.scala:106)
at org.apache.spark.sql.SparkSession.$anonfun$createDataFrame$7(SparkSession.scala:725)
at org.apache.spark.sql.SparkSession.withActive(SparkSession.scala:1450)
at org.apache.spark.sql.SparkSession.createDataFrame(SparkSession.scala:722)
at listener.DataQualityRuleValidationListener.flush$lambda$1(MyQueryExecutionListener.kt:75) <-- Listener
at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539)
at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
at java.base/java.lang.Thread.run(Thread.java:840)
Suppressed: org.apache.spark.util.Utils$OriginalTryStackTraceException: Full stacktrace of original doTryWithCallerStacktrace caller
at com.databricks.unity.error.MissingCredentialScopeException$.withDebugLog(UCSExceptions.scala:62)
at com.databricks.unity.UCSDriver$Manager.$anonfun$currentScopeId$1(UCSDriver.scala:184)
at scala.Option.getOrElse(Option.scala:189)
at com.databricks.unity.UCSDriver$Manager.currentScopeId(UCSDriver.scala:184)
at com.databricks.unity.UCSDriver$Manager.currentScope(UCSDriver.scala:187)
at com.databricks.unity.UnityCredentialScope$.currentScope(UnityCredentialScope.scala:116)
at com.databricks.unity.UnityCredentialScope$.getSAMRegistry(UnityCredentialScope.scala:144)
at com.databricks.unity.SAMRegistry$.snapshot(SAMRegistry.scala:500)
at org.apache.spark.sql.execution.QueryExecution.$anonfun$lazyAnalyzed$3(QueryExecution.scala:303)
at com.databricks.spark.util.FrameProfiler$.record(FrameProfiler.scala:94)
at org.apache.spark.sql.catalyst.QueryPlanningTracker.measurePhase(QueryPlanningTracker.scala:532)
at org.apache.spark.sql.execution.QueryExecution.$anonfun$executePhase$5(QueryExecution.scala:659)
at org.apache.spark.sql.execution.SQLExecution$.withExecutionPhase(SQLExecution.scala:150)
at org.apache.spark.sql.execution.QueryExecution.$anonfun$executePhase$4(QueryExecution.scala:659)
at org.apache.spark.sql.execution.QueryExecution$.withInternalError(QueryExecution.scala:1299)
at org.apache.spark.sql.execution.QueryExecution.$anonfun$executePhase$2(QueryExecution.scala:652)
at com.databricks.util.LexicalThreadLocal$Handle.runWith(LexicalThreadLocal.scala:63)
at org.apache.spark.sql.execution.QueryExecution.$anonfun$executePhase$1(QueryExecution.scala:649)
at org.apache.spark.sql.SparkSession.withActive(SparkSession.scala:1450)
at org.apache.spark.sql.execution.QueryExecution.executePhase(QueryExecution.scala:649)
at org.apache.spark.sql.execution.QueryExecution.$anonfun$lazyAnalyzed$2(QueryExecution.scala:287)
at com.databricks.sql.util.MemoryTrackerHelper.withMemoryTracking(MemoryTrackerHelper.scala:80)
at org.apache.spark.sql.execution.QueryExecution.$anonfun$lazyAnalyzed$1(QueryExecution.scala:286)
at scala.util.Try$.apply(Try.scala:213)
at org.apache.spark.util.Utils$.doTryWithCallerStacktrace(Utils.scala:1683)
at org.apache.spark.util.LazyTry.tryT$lzycompute(LazyTry.scala:46)
at org.apache.spark.util.LazyTry.tryT(LazyTry.scala:46)

Appreciate any ideas on solving this problem.