In our Databricks workspace via the AWS marketplace, serverless works, but when I run my notebooks from dedicated computes they can't reach UnityCatalog and therefore my Spark tables.
Nothing was configured manually by us, everything is configured by Databricks itself.
There seems to be an unexpected grpc error in the driver: `java.lang.NullPointerException: Cannot invoke "com.databricks.backend.daemon.driver.DriverLocal.inspectRequest(String, long, long)" because "this.repl" is null` (status code 500)
Afterwards I also get a Timeout on the WsfsHttpClient.
Anybody experienced something similar and can help out? Below the Log4j stacktrace:
26/03/13 14:53:29 ERROR LoggingService: [sreqId=a1678ad5, chanId=0a94acf2, raddr=10.53.212.221:34106, laddr=10.53.212.221:6061][h1c://0310-130605-4150n9jh-10-53-212-221/#POST] Request: {startTime=2026-03-13T14:53:29.011Z(1773413609011000), length=174B, duration=4080µs(4080925ns), scheme=ws+h1c, name=POST, headers=[:method=POST, :path=/?type="com.databricks.api.proto.chauffeur.InspectInternalRequest", x-request-id=d4644e2b-7799-46c2-a60d-0f1a3cbe7e93, content-length=174, content-type=application/octet-stream, traceparent=00-03f196430cfc04bf1b39d20b8632bde6-af543517eb6cceb7-00]}
26/03/13 14:53:29 ERROR LoggingService: [sreqId=a1678ad5, chanId=0a94acf2, raddr=10.53.212.221:34106, laddr=10.53.212.221:6061][h1c://0310-130605-4150n9jh-10-53-212-221/#POST] Response: {startTime=2026-03-13T14:53:29.047Z(1773413609047000), length=9036B, duration=0ns, totalDuration=35754µs(35754443ns), cause=java.lang.NullPointerException: Cannot invoke "com.databricks.backend.daemon.driver.DriverLocal.inspectRequest(String, long, long)" because "this.repl" is null, headers=[:status=500, content-length=9036, content-type=application/octet-stream]}
java.lang.NullPointerException: Cannot invoke "com.databricks.backend.daemon.driver.DriverLocal.inspectRequest(String, long, long)" because "this.repl" is null
at com.databricks.backend.daemon.driver.DriverWrapper.inspectRequest(DriverWrapper.scala:437)
at com.databricks.backend.daemon.driver.DriverCorralRPCRequestHandlerHelper$.$anonfun$handleInspectRequest$1(DriverCorralRPCRequestHandlerHelper.scala:30)
at scala.Option.map(Option.scala:242)
at com.databricks.backend.daemon.driver.DriverCorralRPCRequestHandlerHelper$.handleInspectRequest(DriverCorralRPCRequestHandlerHelper.scala:30)
at com.databricks.backend.daemon.driver.DriverCorral.inspectInternal(DriverCorral.scala:2176)
at com.databricks.backend.daemon.driver.DriverCorralCompatServerBackend.$anonfun$handlers$35(DriverCorralCompatServerBackend.scala:193)
at com.databricks.rpc.armeria.JettyCompatibilityWrapperBlocking.$anonfun$unaryRpcHandler$1(CompatServerBackend.scala:473)
at com.databricks.rpc.armeria.UnaryRpcHandler$.$anonfun$blocking$2(UnaryRpcHandler.scala:442)
at com.databricks.rpc.armeria.UnaryRpcHandler.$anonfun$callFunc$2(UnaryRpcHandler.scala:314)
at com.databricks.rpc.armeria.Util$.handleUnexpectedExceptions(Util.scala:84)
at com.databricks.rpc.armeria.UnaryRpcHandler.callFunc(UnaryRpcHandler.scala:314)
at com.databricks.rpc.armeria.UnaryRpcHandlerInternal.callFuncWithHooks(UnaryRpcHandler.scala:647)
at com.databricks.rpc.armeria.UnaryRpcHandlerInternal.$anonfun$callFunc$3(UnaryRpcHandler.scala:625)
at com.databricks.rpc.OperationSpan.$anonfun$wrapFuture$1(OperationSpan.scala:69)
at com.databricks.logging.AttributionContextTracing.$anonfun$withAttributionContext$1(AttributionContextTracing.scala:117)
at com.databricks.logging.AttributionContext$.$anonfun$withValue$1(AttributionContext.scala:348)
at scala.util.DynamicVariable.withValue(DynamicVariable.scala:59)
at com.databricks.logging.AttributionContext$.withValue(AttributionContext.scala:344)
at com.databricks.logging.AttributionContextTracing.withAttributionContext(AttributionContextTracing.scala:115)
at com.databricks.logging.AttributionContextTracing.withAttributionContext$(AttributionContextTracing.scala:112)
at com.databricks.rpc.OperationSpan.withAttributionContext(OperationSpan.scala:22)
at com.databricks.rpc.OperationSpan.wrapFuture(OperationSpan.scala:68)
at com.databricks.rpc.armeria.UnaryRpcHandlerInternal.callFunc(UnaryRpcHandler.scala:623)
at com.databricks.rpc.armeria.UnaryRpcHandler.$anonfun$handleRpc0$3(UnaryRpcHandler.scala:274)
at com.databricks.logging.AttributionContextTracing.$anonfun$withAttributionContext$1(AttributionContextTracing.scala:117)
at com.databricks.logging.AttributionContext$.$anonfun$withValue$1(AttributionContext.scala:348)
at scala.util.DynamicVariable.withValue(DynamicVariable.scala:59)
at com.databricks.logging.AttributionContext$.withValue(AttributionContext.scala:344)
at com.databricks.logging.AttributionContextTracing.withAttributionContext(AttributionContextTracing.scala:115)
at com.databricks.logging.AttributionContextTracing.withAttributionContext$(AttributionContextTracing.scala:112)
at com.databricks.rpc.armeria.UnaryRpcHandler.withAttributionContext(UnaryRpcHandler.scala:43)
at com.databricks.rpc.armeria.UnaryRpcHandler.$anonfun$handleRpc0$2(UnaryRpcHandler.scala:234)
at com.databricks.context.integrity.IntegrityCheckContext$ThreadLocalStorage$.withValue(IntegrityCheckContext.scala:73)
at com.databricks.rpc.armeria.UnaryRpcHandler.handleRpc0(UnaryRpcHandler.scala:234)
at com.databricks.rpc.armeria.UnaryRpcHandler.$anonfun$handleRpc$1(UnaryRpcHandler.scala:205)
at com.databricks.rpc.armeria.server.internal.RequestCompletionTracker.wrap(RequestCompletionTracker.scala:184)
at com.databricks.rpc.armeria.UnaryRpcHandler.handleRpc(UnaryRpcHandler.scala:205)
at com.databricks.rpc.armeria.UnaryRpcHandler.handleJettyRpc(UnaryRpcHandler.scala:88)
at com.databricks.rpc.armeria.UnaryRpcServiceInternal.handleJettyRpcWithAggregatedContent(UnaryRpcService.scala:522)
at com.databricks.rpc.armeria.UnaryRpcServiceInternal.$anonfun$handleContextAwareJettyRpcWithAggregatedContent$2(UnaryRpcService.scala:433)
at com.databricks.logging.AttributionContextTracing.$anonfun$withAttributionContext$1(AttributionContextTracing.scala:117)
at com.databricks.logging.AttributionContext$.$anonfun$withValue$1(AttributionContext.scala:348)
at scala.util.DynamicVariable.withValue(DynamicVariable.scala:59)
at com.databricks.logging.AttributionContext$.withValue(AttributionContext.scala:344)
at com.databricks.logging.AttributionContextTracing.withAttributionContext(AttributionContextTracing.scala:115)
at com.databricks.logging.AttributionContextTracing.withAttributionContext$(AttributionContextTracing.scala:112)
at com.databricks.rpc.armeria.UnaryRpcServiceInternal.withAttributionContext(UnaryRpcService.scala:187)
at com.databricks.rpc.armeria.UnaryRpcServiceInternal.$anonfun$handleContextAwareJettyRpcWithAggregatedContent$1(UnaryRpcService.scala:428)
at com.databricks.context.integrity.IntegrityCheckContext$ThreadLocalStorage$.withValue(IntegrityCheckContext.scala:73)
at com.databricks.rpc.armeria.UnaryRpcServiceInternal.handleContextAwareJettyRpcWithAggregatedContent(UnaryRpcService.scala:427)
at com.databricks.rpc.armeria.UnaryRpcServiceInternal.$anonfun$handleJettyJsonRpc$2(UnaryRpcService.scala:407)
at com.databricks.rpc.armeria.Util$.handleUnexpectedExceptions(Util.scala:84)
at com.databricks.rpc.armeria.Util$.handleUnexpectedExceptions(Util.scala:108)
at com.databricks.rpc.armeria.UnaryRpcServiceInternal.$anonfun$handleJettyJsonRpc$1(UnaryRpcService.scala:402)
at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:467)
at com.databricks.threading.DatabricksExecutionContext$InstrumentedRunnable.run(DatabricksExecutionContext.scala:36)
at grpc_shaded.com.linecorp.armeria.common.DefaultContextAwareRunnable.run(DefaultContextAwareRunnable.java:45)
at com.databricks.threading.ContextBoundRunnable.$anonfun$run$2(ContextBoundRunnable.scala:16)
at scala.runtime.java8.JFunction0$mcV$sp.apply(JFunction0$mcV$sp.scala:18)
at com.databricks.logging.AttributionContextTracing.$anonfun$withAttributionContext$1(AttributionContextTracing.scala:117)
at com.databricks.logging.AttributionContext$.$anonfun$withValue$1(AttributionContext.scala:348)
at scala.util.DynamicVariable.withValue(DynamicVariable.scala:59)
at com.databricks.logging.AttributionContext$.withValue(AttributionContext.scala:344)
at com.databricks.logging.AttributionContextTracing.withAttributionContext(AttributionContextTracing.scala:115)
at com.databricks.logging.AttributionContextTracing.withAttributionContext$(AttributionContextTracing.scala:112)
at com.databricks.threading.ContextBoundRunnable.withAttributionContext(ContextBoundRunnable.scala:7)
at com.databricks.threading.ContextBoundRunnable.$anonfun$run$1(ContextBoundRunnable.scala:16)
at scala.runtime.java8.JFunction0$mcV$sp.apply(JFunction0$mcV$sp.scala:18)
at com.databricks.context.integrity.IntegrityCheckContext$ThreadLocalStorage$.withValue(IntegrityCheckContext.scala:73)
at com.databricks.threading.ContextBoundRunnable.run(ContextBoundRunnable.scala:15)
at com.databricks.threading.InstrumentedExecutorService.$anonfun$makeContextAware$2(InstrumentedExecutorService.scala:257)
at scala.runtime.java8.JFunction0$mcV$sp.apply(JFunction0$mcV$sp.scala:18)
at com.databricks.threading.InstrumentedExecutorService.$anonfun$instrumentationWrapper$1(InstrumentedExecutorService.scala:299)
at com.databricks.instrumentation.QueuedThreadPoolInstrumenter.trackActiveThreads(QueuedThreadPoolInstrumenter.scala:132)
at com.databricks.instrumentation.QueuedThreadPoolInstrumenter.trackActiveThreads$(QueuedThreadPoolInstrumenter.scala:129)
at com.databricks.threading.InstrumentedExecutorService.trackActiveThreads(InstrumentedExecutorService.scala:72)
at com.databricks.threading.InstrumentedExecutorService.instrumentationWrapper(InstrumentedExecutorService.scala:287)
at com.databricks.threading.InstrumentedExecutorService.$anonfun$makeContextAware$1(InstrumentedExecutorService.scala:259)
at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:572)
at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:317)
at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:304)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
at java.base/java.lang.Thread.run(Thread.java:1583)
26/03/13 14:54:01 ERROR WSFSManager: Failed to update command details for Files in Repos. " +
Set Spark configuration value spark.databricks.enableWsfs=false"
if you do not need the feature on this cluster.
java.util.concurrent.TimeoutException: timeout after 1 tries
at com.databricks.common.wsfs.WsfsHttpClient.sendJsonJetty(WsfsHttpClient.scala:328)
at com.databricks.common.wsfs.WsfsHttpClient.sendJson(WsfsHttpClient.scala:297)
at com.databricks.common.wsfs.WsfsHttpClient.putJson(WsfsHttpClient.scala:224)
at com.databricks.common.wsfs.WsfsHttpClient.putTokenForPidWithTags(WsfsHttpClient.scala:171)
at com.databricks.backend.daemon.driver.WSFSManager.sendCredential(WSFSDriverManager.scala:393)
at com.databricks.backend.daemon.driver.WSFSManager.$anonfun$sendCredentialToAllWithStatus$1(WSFSDriverManager.scala:321)
at com.databricks.backend.daemon.driver.WSFSManager.$anonfun$sendCredentialToAllWithStatus$1$adapted(WSFSDriverManager.scala:302)
at scala.collection.immutable.List.filter(List.scala:516)
at scala.collection.immutable.List.filter(List.scala:79)
at com.databricks.backend.daemon.driver.WSFSManager.sendCredentialToAllWithStatus(WSFSDriverManager.scala:302)
at com.databricks.backend.daemon.driver.JupyterDriverLocal.$anonfun$initFuseIfEnabled$1(JupyterDriverLocal.scala:1248)
at scala.Option.map(Option.scala:242)
at com.databricks.backend.daemon.driver.JupyterDriverLocal.initFuseIfEnabled(JupyterDriverLocal.scala:1244)
at com.databricks.backend.daemon.driver.JupyterDriverLocal.$anonfun$setUpBeforeCommandRun$3(JupyterDriverLocal.scala:1297)
at scala.runtime.java8.JFunction0$mcV$sp.apply(JFunction0$mcV$sp.scala:18)
at com.databricks.backend.daemon.driver.logging.DriverActivityLogging$.runWithExecuteCommandEventLogging(DriverActivityLogging.scala:251)
at com.databricks.backend.daemon.driver.JupyterDriverLocal.setUpBeforeCommandRun(JupyterDriverLocal.scala:1297)
at com.databricks.backend.daemon.driver.JupyterDriverLocal.$anonfun$executePython$3(JupyterDriverLocal.scala:1477)
at com.databricks.backend.daemon.driver.JupyterKernelListener.preExecuteCommand(JupyterKernelListener.scala:1336)
at com.databricks.backend.daemon.driver.JupyterKernelListener.executeCommand(JupyterKernelListener.scala:1562)
at com.databricks.backend.daemon.driver.JupyterDriverLocal.executePython(JupyterDriverLocal.scala:1477)
at com.databricks.backend.daemon.driver.JupyterDriverLocal.repl(JupyterDriverLocal.scala:1382)
at com.databricks.backend.daemon.driver.DriverLocal.$anonfun$execute$36(DriverLocal.scala:1055)
at com.databricks.unity.UCSEphemeralState$Handle.runWith(UCSEphemeralState.scala:51)
at com.databricks.unity.HandleImpl.runWith(UCSHandle.scala:104)
at com.databricks.backend.daemon.driver.DriverLocal.$anonfun$execute$32(DriverLocal.scala:1046)
at com.databricks.logging.AttributionContextTracing.$anonfun$withAttributionContext$1(AttributionContextTracing.scala:117)
at com.databricks.logging.AttributionContext$.$anonfun$withValue$1(AttributionContext.scala:348)
at scala.util.DynamicVariable.withValue(DynamicVariable.scala:59)
at com.databricks.logging.AttributionContext$.withValue(AttributionContext.scala:344)
at com.databricks.logging.AttributionContextTracing.withAttributionContext(AttributionContextTracing.scala:115)
at com.databricks.logging.AttributionContextTracing.withAttributionContext$(AttributionContextTracing.scala:112)
at com.databricks.backend.daemon.driver.DriverLocal.withAttributionContext(DriverLocal.scala:100)
at com.databricks.logging.AttributionContextTracing.withAttributionTags(AttributionContextTracing.scala:172)
at com.databricks.logging.AttributionContextTracing.withAttributionTags$(AttributionContextTracing.scala:153)
at com.databricks.backend.daemon.driver.DriverLocal.withAttributionTags(DriverLocal.scala:100)
at com.databricks.backend.daemon.driver.DriverLocal.$anonfun$execute$1(DriverLocal.scala:991)
at com.databricks.backend.daemon.driver.DriverLocal$.$anonfun$maybeSynchronizeExecution$4(DriverLocal.scala:1474)
at com.databricks.backend.daemon.driver.DriverLocal.execute(DriverLocal.scala:589)
at com.databricks.backend.daemon.driver.DriverWrapper.$anonfun$tryExecutingCommand$2(DriverWrapper.scala:989)
at scala.util.Try$.apply(Try.scala:217)
at com.databricks.backend.daemon.driver.DriverWrapper.$anonfun$tryExecutingCommand$1(DriverWrapper.scala:978)
at com.databricks.backend.daemon.driver.DriverWrapper.$anonfun$tryExecutingCommand$3(DriverWrapper.scala:1024)
at com.databricks.logging.UsageLogging.executeThunkAndCaptureResultTags$1(UsageLogging.scala:632)
at com.databricks.logging.UsageLogging.$anonfun$recordOperationWithResultTags$5(UsageLogging.scala:659)
at com.databricks.logging.AttributionContextTracing.$anonfun$withAttributionContext$1(AttributionContextTracing.scala:117)
at com.databricks.logging.AttributionContext$.$anonfun$withValue$1(AttributionContext.scala:348)
at scala.util.DynamicVariable.withValue(DynamicVariable.scala:59)
at com.databricks.logging.AttributionContext$.withValue(AttributionContext.scala:344)
at com.databricks.logging.AttributionContextTracing.withAttributionContext(AttributionContextTracing.scala:115)
at com.databricks.logging.AttributionContextTracing.withAttributionContext$(AttributionContextTracing.scala:112)
at com.databricks.backend.daemon.driver.DriverWrapper.withAttributionContext(DriverWrapper.scala:72)
at com.databricks.logging.AttributionContextTracing.withAttributionTags(AttributionContextTracing.scala:172)
at com.databricks.logging.AttributionContextTracing.withAttributionTags$(AttributionContextTracing.scala:153)
at com.databricks.backend.daemon.driver.DriverWrapper.withAttributionTags(DriverWrapper.scala:72)
at com.databricks.logging.UsageLogging.recordOperationWithResultTags(UsageLogging.scala:627)
at com.databricks.logging.UsageLogging.recordOperationWithResultTags$(UsageLogging.scala:521)
at com.databricks.backend.daemon.driver.DriverWrapper.recordOperationWithResultTags(DriverWrapper.scala:72)
at com.databricks.backend.daemon.driver.DriverWrapper.tryExecutingCommand(DriverWrapper.scala:1024)
at com.databricks.backend.daemon.driver.DriverWrapper.executeCommandAndGetError(DriverWrapper.scala:793)
at com.databricks.backend.daemon.driver.DriverWrapper.executeCommand(DriverWrapper.scala:886)
at com.databricks.backend.daemon.driver.DriverWrapper.$anonfun$runInnerLoop$4(DriverWrapper.scala:648)
at scala.runtime.java8.JFunction0$mcV$sp.apply(JFunction0$mcV$sp.scala:18)
at com.databricks.logging.AttributionContextTracing.$anonfun$withAttributionContext$1(AttributionContextTracing.scala:117)
at com.databricks.logging.AttributionContext$.$anonfun$withValue$1(AttributionContext.scala:348)
at scala.util.DynamicVariable.withValue(DynamicVariable.scala:59)
at com.databricks.logging.AttributionContext$.withValue(AttributionContext.scala:344)
at com.databricks.logging.AttributionContextTracing.withAttributionContext(AttributionContextTracing.scala:115)
at com.databricks.logging.AttributionContextTracing.withAttributionContext$(AttributionContextTracing.scala:112)
at com.databricks.backend.daemon.driver.DriverWrapper.withAttributionContext(DriverWrapper.scala:72)
at com.databricks.backend.daemon.driver.DriverWrapper.runInnerLoop(DriverWrapper.scala:643)
at com.databricks.backend.daemon.driver.DriverWrapper.runInner(DriverWrapper.scala:546)
at com.databricks.backend.daemon.driver.DriverWrapper.run(DriverWrapper.scala:375)
at java.base/java.lang.Thread.run(Thread.java:1583)
