We are intermittently experiencing the below issue when running mundane code in our databricks notebook environment using 13.3 LTS runtime, with a compute pool with r6id.large on-demand instances, using local storage.
We first noticed this late last week, around the same time as the 14.3 LTS release.
Observed error messages:
Spoiler
py4j.Py4JException: An exception was raised by the Python Proxy. Return Message: Traceback (most recent call last):
File "/databricks/spark/python/lib/py4j-0.10.9.7-src.zip/py4j/clientserver.py", line 617, in _call_proxy
return_value = getattr(self.pool[obj_id], method)(*params)
File "/databricks/python_shell/dbruntime/pythonPathHook.py", line 118, in initStartingDirectory
self._handle_sys_path_maybe_updated()
File "/databricks/python_shell/dbruntime/pythonPathHook.py", line 90, in _handle_sys_path_maybe_updated
self._restart_language_server_if_needed()
File "/databricks/python_shell/dbruntime/pythonPathHook.py", line 85, in _restart_language_server_if_needed
ls_manager.restart()
File "/databricks/python_shell/dbruntime/lsp_backend/lsp_manager.py", line 348, in restart
self.start()
File "/databricks/python_shell/dbruntime/lsp_backend/lsp_manager.py", line 290, in start
self.server_process = subprocess.Popen(
File "/usr/lib/python3.10/subprocess.py", line 971, in __init__
self._execute_child(args, executable, preexec_fn, close_fds,
File "/usr/lib/python3.10/subprocess.py", line 1863, in _execute_child
raise child_exception_type(errno_num, err_msg, err_filename)
FileNotFoundError: [Errno 2] No such file or directory: 'pylsp'
at py4j.Protocol.getReturnValue(Protocol.java:476)
at py4j.reflection.PythonProxyHandler.invoke(PythonProxyHandler.java:108)
at com.sun.proxy.$Proxy70.initStartingDirectory(Unknown Source)
at com.databricks.backend.daemon.driver.PythonDriverLocalBase.$anonfun$setWsfsWorkingDir$1(PythonDriverLocalBase.scala:316)
at com.databricks.backend.daemon.driver.PythonDriverLocalBase.$anonfun$setWsfsWorkingDir$1$adapted(PythonDriverLocalBase.scala:307)
at scala.Option.foreach(Option.scala:407)
at com.databricks.backend.daemon.driver.PythonDriverLocalBase.setWsfsWorkingDir(PythonDriverLocalBase.scala:307)
at com.databricks.backend.daemon.driver.PythonDriverLocalBase.addLibrariesToPythonPath(PythonDriverLocalBase.scala:244)
at com.databricks.backend.daemon.driver.JupyterDriverLocal.setUpBeforeCommandRun(JupyterDriverLocal.scala:628)
at com.databricks.backend.daemon.driver.JupyterDriverLocal.$anonfun$executePython$1(JupyterDriverLocal.scala:814)
at com.databricks.backend.daemon.driver.JupyterKernelListener.preExecuteCommand(JupyterKernelListener.scala:951)
at com.databricks.backend.daemon.driver.JupyterKernelListener.executeCommand(JupyterKernelListener.scala:1080)
at com.databricks.backend.daemon.driver.JupyterDriverLocal.executePython(JupyterDriverLocal.scala:814)
at com.databricks.backend.daemon.driver.JupyterDriverLocal.repl(JupyterDriverLocal.scala:697)
at com.databricks.backend.daemon.driver.DriverLocal.$anonfun$execute$33(DriverLocal.scala:997)
at com.databricks.unity.UCSEphemeralState$Handle.runWith(UCSEphemeralState.scala:41)
at com.databricks.unity.HandleImpl.runWith(UCSHandle.scala:99)
at com.databricks.backend.daemon.driver.DriverLocal.$anonfun$execute$22(DriverLocal.scala:980)
at com.databricks.logging.UsageLogging.$anonfun$withAttributionContext$1(UsageLogging.scala:426)
at scala.util.DynamicVariable.withValue(DynamicVariable.scala:62)
at com.databricks.logging.AttributionContext$.withValue(AttributionContext.scala:196)
at com.databricks.logging.UsageLogging.withAttributionContext(UsageLogging.scala:424)
at com.databricks.logging.UsageLogging.withAttributionContext$(UsageLogging.scala:418)
at com.databricks.backend.daemon.driver.DriverLocal.withAttributionContext(DriverLocal.scala:69)
at com.databricks.logging.UsageLogging.withAttributionTags(UsageLogging.scala:470)
at com.databricks.logging.UsageLogging.withAttributionTags$(UsageLogging.scala:455)
at com.databricks.backend.daemon.driver.DriverLocal.withAttributionTags(DriverLocal.scala:69)
at com.databricks.backend.daemon.driver.DriverLocal.execute(DriverLocal.scala:935)
at com.databricks.backend.daemon.driver.DriverWrapper.$anonfun$tryExecutingCommand$1(DriverWrapper.scala:798)
at scala.util.Try$.apply(Try.scala:213)
at com.databricks.backend.daemon.driver.DriverWrapper.tryExecutingCommand(DriverWrapper.scala:790)
at com.databricks.backend.daemon.driver.DriverWrapper.executeCommandAndGetError(DriverWrapper.scala:643)
at com.databricks.backend.daemon.driver.DriverWrapper.executeCommand(DriverWrapper.scala:744)
at com.databricks.backend.daemon.driver.DriverWrapper.runInnerLoop(DriverWrapper.scala:520)
at com.databricks.backend.daemon.driver.DriverWrapper.runInner(DriverWrapper.scala:436)
at com.databricks.backend.daemon.driver.DriverWrapper.run(DriverWrapper.scala:279)
at java.lang.Thread.run(Thread.java:750)
Additional related error messages:
Spoiler
py4j.Py4JException: Error while sending a command.
at py4j.CallbackClient.sendCommand(CallbackClient.java:397)
at py4j.CallbackClient.sendCommand(CallbackClient.java:356)
at py4j.reflection.PythonProxyHandler.invoke(PythonProxyHandler.java:106)
at com.sun.proxy.$Proxy58.updateWsfsWorkingDir(Unknown Source)
at com.databricks.backend.daemon.driver.PythonDriverLocalBase.$anonfun$setWsfsWorkingDir$1(PythonDriverLocalBase.scala:318)
at com.databricks.backend.daemon.driver.PythonDriverLocalBase.$anonfun$setWsfsWorkingDir$1$adapted(PythonDriverLocalBase.scala:307)
at scala.Option.foreach(Option.scala:407)
at com.databricks.backend.daemon.driver.PythonDriverLocalBase.setWsfsWorkingDir(PythonDriverLocalBase.scala:307)
at com.databricks.backend.daemon.driver.PythonDriverLocalBase.addLibrariesToPythonPath(PythonDriverLocalBase.scala:244)
at com.databricks.backend.daemon.driver.JupyterDriverLocal.setUpBeforeCommandRun(JupyterDriverLocal.scala:628)
at com.databricks.backend.daemon.driver.JupyterDriverLocal.$anonfun$executePython$1(JupyterDriverLocal.scala:814)
at com.databricks.backend.daemon.driver.JupyterKernelListener.preExecuteCommand(JupyterKernelListener.scala:951)
at com.databricks.backend.daemon.driver.JupyterKernelListener.executeCommand(JupyterKernelListener.scala:1080)
at com.databricks.backend.daemon.driver.JupyterDriverLocal.executePython(JupyterDriverLocal.scala:814)
at com.databricks.backend.daemon.driver.JupyterDriverLocal.repl(JupyterDriverLocal.scala:697)
at com.databricks.backend.daemon.driver.DriverLocal.$anonfun$execute$33(DriverLocal.scala:997)
at com.databricks.unity.UCSEphemeralState$Handle.runWith(UCSEphemeralState.scala:41)
at com.databricks.unity.HandleImpl.runWith(UCSHandle.scala:99)
at com.databricks.backend.daemon.driver.DriverLocal.$anonfun$execute$22(DriverLocal.scala:980)
at com.databricks.logging.UsageLogging.$anonfun$withAttributionContext$1(UsageLogging.scala:426)
at scala.util.DynamicVariable.withValue(DynamicVariable.scala:62)
at com.databricks.logging.AttributionContext$.withValue(AttributionContext.scala:196)
at com.databricks.logging.UsageLogging.withAttributionContext(UsageLogging.scala:424)
at com.databricks.logging.UsageLogging.withAttributionContext$(UsageLogging.scala:418)
at com.databricks.backend.daemon.driver.DriverLocal.withAttributionContext(DriverLocal.scala:69)
at com.databricks.logging.UsageLogging.withAttributionTags(UsageLogging.scala:470)
at com.databricks.logging.UsageLogging.withAttributionTags$(UsageLogging.scala:455)
at com.databricks.backend.daemon.driver.DriverLocal.withAttributionTags(DriverLocal.scala:69)
at com.databricks.backend.daemon.driver.DriverLocal.execute(DriverLocal.scala:935)
at com.databricks.backend.daemon.driver.DriverWrapper.$anonfun$tryExecutingCommand$1(DriverWrapper.scala:798)
at scala.util.Try$.apply(Try.scala:213)
at com.databricks.backend.daemon.driver.DriverWrapper.tryExecutingCommand(DriverWrapper.scala:790)
at com.databricks.backend.daemon.driver.DriverWrapper.executeCommandAndGetError(DriverWrapper.scala:643)
at com.databricks.backend.daemon.driver.DriverWrapper.executeCommand(DriverWrapper.scala:744)
at com.databricks.backend.daemon.driver.DriverWrapper.runInnerLoop(DriverWrapper.scala:520)
at com.databricks.backend.daemon.driver.DriverWrapper.runInner(DriverWrapper.scala:436)
at com.databricks.backend.daemon.driver.DriverWrapper.run(DriverWrapper.scala:279)
at java.lang.Thread.run(Thread.java:750)
Caused by: py4j.Py4JNetworkException: Error while sending a command: c
p0
updateWsfsWorkingDir
s/Workspace/Users/[user omitted]
e
at py4j.ClientServerConnection.sendCommand(ClientServerConnection.java:266)
at py4j.CallbackClient.sendCommand(CallbackClient.java:384)
... 37 more
Caused by: py4j.Py4JException: Received empty command
at py4j.ClientServerConnection.sendCommand(ClientServerConnection.java:249)
... 38 more
Spoiler
IllegalStateExeception: Internal error: notebook command received after detach
