Databricks Community

Comment

New Contributor II

- edited

@silvyAlej You need to write python code to read the model.json file.

You can view the databricks code here .

Alternatively you can copy the below and change for your needs:

def get_spark_type(data_type, max_length, traits=None):
if data_type == 'guid':
return StringType()
elif data_type == 'dateTime':
return TimestampType()
elif data_type == 'dateTimeOffset':
return TimestampType()
elif data_type == 'int64':
return LongType()
elif data_type == 'int32':
return IntegerType()
elif data_type == 'decimal':
precision = 38
scale = 18
if traits:
for trait in traits:
if trait['traitReference'] == 'is.dataFormat.numeric.shaped':
for arg in trait['arguments']:
if arg['name'] == 'precision':
precision = arg['value']
elif arg['name'] == 'scale':
scale = arg['value']
return DecimalType(precision, scale)
elif data_type == 'string':
return StringType()
elif data_type == 'boolean':
return BooleanType()
elif data_type == 'date':
return DateType()
else:
return StringType()
#raise ValueError(f"Unsupported data type: {data_type}")

def create_spark_schema(fields):

schema_fields = []

for field in fields:

name = field['name']

data_type = field['dataType']

max_length = field.get('maxLength', -1)

traits = field.get('cdm:traits', None)

spark_type = get_spark_type(data_type, max_length, traits)

schema_fields.append(StructField(name, spark_type, True))

return StructType(schema_fields)

#Load Metadata
model_json_path = f"abfss://{container}@{storage_account}.dfs.core.windows.net/model.json"
model_json = spark.read.text(model_json_path).collect()[0][0]
model = json.loads(model_json)

#List of all Tables
tables_list = [entity['description'] for entity in model['entities']]

#Iterate through all tables
for table_name in tables_list:
schema_list = [entity['attributes'] for entity in model['entities'] if entity['description'] == table_name][0]
schema = create_spark_schema(schema_list)

#Do whatever you need with the schema.