Hi all
I struggle to find out why this error message suddenly pops up after running a cell in a notebook.
The notebook is trying to run a simple "INSERT INTO" command in SQL. When I only do a SELECT clause, the cell runs without error. Also, I only get this error message when running Databricks runtime 12 and 12.1 but 11.3LTS runs both INSERT INTO and SELECT without errors. The weird thing is that this error suddenly popped up after a week when I switched to runtime 12 from 11.3. Now I have reverted back to 11.3 so that my workflows can run.
Can someone help decyphering what this error message means?
Thanks!
Py4JJavaError Traceback (most recent call last)
File <command-3863647901736418>:7
5 display(df)
6 return df
----> 7 _sqldf = ____databricks_percent_sql()
8 finally:
9 del ____databricks_percent_sql
File <command-3863647901736418>:4, in ____databricks_percent_sql()
2 def ____databricks_percent_sql():
3 import base64
----> 4 df = spark.sql(base64.standard_b64decode("aW5zZXJ0IGludG8gZW5yaWNoZWQuZGFpbHlfcGVyaW9kaWNfc25hcHNob3RfZmFjdCAoCiAgLypnZXQgYWxsIHRoZSBzYWxlaWRzIHRoYXQgZXhpc3RzIGluIGVucmljaGVkLnRyYW5zYWN0aW9uc19mYWN0Ki8gIAogIHdpdGggc2FsZWlkcyBhcyAoCiAgCnNlbGVjdCBkaXN0aW5jdCBTYWxlSURTb3VyY2UgZnJvbSBlbnJpY2hlZC50cmFuc2FjdGlvbnNfZmFjdAogIAopLApjb250cmFjdF9kaW0gYXMgKAogIAogCiAgc2VsZWN0CiAgICAqLAogICAgCiAgICBST1dfTlVNQkVSKCkgT1ZFUiAoCiAgICAgIFBBUlRJVElPTiBCWSBTYWxlSURTb3VyY2UKICAgICAgT1JERVIgQlkKICAgICAgICBSZWdpc3RlcmVkIGRlc2MKICAgICkgYXMgcm93bnVtYmVyCiAgZnJvbQogICAgZW5yaWNoZWQuY29udHJhY3RfZGltCiAgCiAgd2hlcmUKICAgIFNhbGVJRFNvdXJjZSBpbiAoCiAgICAgIHNlbGVjdAogICAgICAgIGRpc3RpbmN0IFNhbGVJRFNvdXJjZQogICAgICBmcm9tCiAgICAgICAgc2FsZWlkcwogICAgKQogIAogICAgYW5kIERBVEUoUmVnaXN0ZXJlZCkgPD0gREFURSgnMjAyMy0wMS0yNicpCiksCnN0dWRlbnRfZGltIGFzICgKICAKICBzZWxlY3QKICAgICosCiAgICAKICAgIFJPV19OVU1CRVIoKSBPVkVSICgKICAgICAgUEFSVElUSU9OIEJZIENvbnRhY3RJRFNvdXJjZQogICAgICBPUkRFUiBCWQogICAgICAgIFJlZ2lzdGVyZWQgZGVzYwogICAgKSBhcyByb3dudW1iZXIKICBmcm9tCiAgICBlbnJpY2hlZC5zdHVkZW50X2RpbQogIAogIHdoZXJlCiAgICBDb250YWN0SURTb3VyY2UgaW4gKAogICAgICBzZWxlY3QKICAgICAgICBkaXN0aW5jdCBDb250YWN0SURTb3VyY2UKICAgICAgZnJvbQogICAgICAgIGNvbnRyYWN0X2... ").decode())
5 display(df)
6 return df
File /databricks/spark/python/pyspark/instrumentation_utils.py:48, in _wrap_function.<locals>.wrapper(*args, **kwargs)
46 start = time.perf_counter()
47 try:
---> 48 res = func(*args, **kwargs)
49 logger.log_success(
50 module_name, class_name, function_name, time.perf_counter() - start, signature
51 )
52 return res
File /databricks/spark/python/pyspark/sql/session.py:1205, in SparkSession.sql(self, sqlQuery, **kwargs)
1203 sqlQuery = formatter.format(sqlQuery, **kwargs)
1204 try:
-> 1205 return DataFrame(self._jsparkSession.sql(sqlQuery), self)
1206 finally:
1207 if len(kwargs) > 0:
File /databricks/spark/python/lib/py4j-0.10.9.5-src.zip/py4j/java_gateway.py:1321, in JavaMember.__call__(self, *args)
1315 command = proto.CALL_COMMAND_NAME +\
1316 self.command_header +\
1317 args_command +\
1318 proto.END_COMMAND_PART
1320 answer = self.gateway_client.send_command(command)
-> 1321 return_value = get_return_value(
1322 answer, self.gateway_client, self.target_id, self.name)
1324 for temp_arg in temp_args:
1325 temp_arg._detach()
File /databricks/spark/python/pyspark/sql/utils.py:209, in capture_sql_exception.<locals>.deco(*a, **kw)
207 def deco(*a: Any, **kw: Any) -> Any:
208 try:
--> 209 return f(*a, **kw)
210 except Py4JJavaError as e:
211 converted = convert_exception(e.java_exception)
File /databricks/spark/python/lib/py4j-0.10.9.5-src.zip/py4j/protocol.py:326, in get_return_value(answer, gateway_client, target_id, name)
324 value = OUTPUT_CONVERTER[type](answer[2:], gateway_client)
325 if answer[1] == REFERENCE_TYPE:
--> 326 raise Py4JJavaError(
327 "An error occurred while calling {0}{1}{2}.\n".
328 format(target_id, ".", name), value)
329 else:
330 raise Py4JError(
331 "An error occurred while calling {0}{1}{2}. Trace:\n{3}\n".
332 format(target_id, ".", name, value))
Py4JJavaError: An error occurred while calling o337.sql.
: java.lang.AssertionError: assertion failed
