from databricks.connect import DatabricksSession
spark = DatabricksSession.builder.getOrCreate()

schema = StructType([StructField('category', StringType(), True), StructField('weight', DoubleType(), True)])
data_source = "abfss://......_index_v01_??????_????????.csv"

df = (spark.read.format("csv")
 .options(**{'header': 'true'})
 .schema(schema)
 .load(data_source))

[SESSION_ALREADY_EXIST] Cannot start a remote Spark session because there is a regular Spark session already running.

File /databricks/spark/python/pyspark/instrumentation_utils.py:47, in _wrap_function.<locals>.wrapper(*args, **kwargs) 45 start = time.perf_counter() 46 try: ---> 47 res = func(*args, **kwargs) 48 logger.log_success( 49 module_name, class_name, function_name, time.perf_counter() - start, signature 50 ) 51 return res

File /databricks/spark/python/pyspark/sql/readwriter.py:150, in DataFrameReader.schema(self, schema) 117 """Specifies the input schema. 118 119 Some data sources (e.g. JSON) can infer the input schema automatically from data. (...) 146 |-- col1: double (nullable = true) 147 """ 148 from pyspark.sql import SparkSession --> 150 spark = SparkSession._getActiveSessionOrCreate() 151 if isinstance(schema, StructType): 152 jschema = spark._jsparkSession.parseDataType(schema.json())

File /databricks/spark/python/pyspark/sql/session.py:1265, in SparkSession._getActiveSessionOrCreate(**static_conf) 1263 for k, v in static_conf.items(): 1264 builder = builder.config(k, v) -> 1265 spark = builder.getOrCreate() 1266 return spark

File /databricks/spark/python/pyspark/sql/session.py:521, in SparkSession.Builder.getOrCreate(self) 519 return RemoteSparkSession.builder.config(map=opts).getOrCreate() 520 else: --> 521 raise PySparkRuntimeError( 522 error_class="SESSION_ALREADY_EXIST", 523 message_parameters={}, 524 ) 526 session = SparkSession._instantiatedSession 527 if session is None or session._sc._jsc is None: