java.lang.UnsupportedOperationException: com.databricks.backend.daemon.data.client.DBFSV1.resolvePathOnPhysicalStorage(path: Path)

---------------------------------------------------------------------------

Py4JJavaError Traceback (most recent call last)

<command-1919808831305181> in <module>

----> 1 query = autoload_to_table(data_source = f"{DA.paths.working_dir}/tracker",

2 source_format = "json",

3 table_name = "target_table",

4 checkpoint_directory = f"{DA.paths.checkpoints}/target_table")

<command-1919808831305179> in autoload_to_table(data_source, source_format, table_name, checkpoint_directory)

1 def autoload_to_table(data_source, source_format, table_name, checkpoint_directory):

----> 2 query = (spark.readStream

3 .format("cloudFiles")

4 .option("cloudFiles.format", source_format)

5 .option("cloudFiles.schemaLocation", checkpoint_directory)

/databricks/spark/python/pyspark/sql/streaming.py in load(self, path, format, schema, **options)

450 raise ValueError("If the path is provided for stream, it needs to be a " +

451 "non-empty string. List of paths are not supported.")

--> 452 return self._df(self._jreader.load(path))

453 else:

454 return self._df(self._jreader.load())