from databricks.connect import DatabricksSession
from data.dbx_conn_info import DbxConnInfo
class SparkSessionManager:
_instance = None
_spark = None
def __new__(cls):
if cls._instance is None:
cls._instance = super(SparkSessionManager, cls).__new__(cls)
return cls._instance
def get_spark_session(self):
if not self._spark:
self._spark = DatabricksSession.builder.remote(
host=f"https://{DbxConnInfo.retrieve_workspace_instance_name()}",
token=DbxConnInfo.retrieve_token(),
cluster_id=DbxConnInfo.retrieve_cluster_id()
).getOrCreate()
return self._spark
When I write and run test code for this code, it keeps telling me that the databricks package doesn't exist. I don't know why.