- job_cluster_key: ${bundle.name}

new_cluster:

cluster_name: ""

spark_version: 15.4.x-scala2.12

azure_attributes:

first_on_demand: 1

availability: ON_DEMAND_AZURE

spot_bid_max_price: -1

node_type_id: Standard_E32d_v4 # Worker Node: 32 cores & 256GB RAM

# driver_node_type_id: Standard_E

spark_env_vars:

PYSPARK_PYTHON: /databricks/python3/bin/python3

enable_elastic_disk: true

policy_id: ${var.ds_compute_policy_id}

data_security_mode: SINGLE_USER

runtime_engine: PHOTON

num_workers: 0

# autoscale:

# min_workers: 1

# max_workers: 2