asset bundle configuration

seefoods
Valued Contributor

‎07-08-2025 12:12 AM

Hi Guys

I want to use this approach to give more flexibility of my asset bundle. But when i try to implement this i have some error like unknown field. SomeOne, could show me an example ? 

Thanx, 




databricks-dab-repo/
├── databricks.yml # Bundle definition
├── resources/
│ ├── workflows/
│ │ ├── my_pipeline.yml # YAML pipeline def
│ │ └── my_pipeline_job.yml # YAML job def that runs pipeline
│ ├── clusters/
│ │ ├── dev_cluster.yml # development cluster def
│ │ └── prod_cluster.yml # production def
├── src/
│ ├── dlt_pipeline.ipynb # pipeline notebook
│ └── mypython.py # Additional Python
└── README.md

0 Kudos

ilir_nuredini
Honored Contributor

‎07-08-2025 01:51 AM

Hello @seefoods ,

May you share the error codes you get and what example you wanna get exactly (like maybe databricks.yaml file that handles that structure, or..). Thank you!

Best, Ilir

seefoods
Valued Contributor

‎07-11-2025 12:21 AM

Actually its works, i have declare all my clusters config in variables field 

 

permissions:

  - level: CAN_VIEW

    group_name: data-engineer

  - level: CAN_RUN

    group_name: data-engineer

  - level: CAN_MANAGE

    group_name: data-engineer

  - level: CAN_RUN

    user_name: toto@company.com

 

 

variables:

  environnement:

    default: dev

  job_policy_id:

    default:

  notification_emails:

    default:

      - toto@company.com

  sp-cicd-dev-id:

    default:

  sp-cicd-qa-id:

    default:

  sp-cicd-preprod-id:

    default:

  sp-cicd-prod-id:

    default:

 

  databricks_utils:

    default: "

  personal_compute:

    default:

  use_autoloader:

    default: true

  xs_cluster:

    description: "very small cluster"

    type: complex

    default:

      spark_version: "16.4.x-scala2.12"

      spark_conf:

        spark.sql.sources.partitionOverwriteMode: dynamic

        delta.autoOptimize.optimizeWrite: true

        delta.autoOptimize.autoCompact: true

        spark.sql.streaming.stateStore.providerClass: org.apache.spark.sql.execution.streaming.state.RocksDBStateStoreProvider

        spark.sql.streaming.stateStore.rocksdb.changelogCheckpointing.enabled: true

      aws_attributes:

        first_on_demand: 1

      # availability: SPOT_WITH_FALLBACK

      num_workers: 1

      node_type_id: m5d.large

      driver_node_type_id: m5d.large

      data_security_mode: SINGLE_USER

      runtime_engine: STANDARD

      single_user_name: ${workspace.current_user.userName}

      policy_id: ${var.job_policy_id}

  s_cluster:

    description: "small cluster"

    type: complex

    default:

      spark_version: "16.4.x-scala2.12"

      spark_conf:

        spark.sql.sources.partitionOverwriteMode: dynamic

        delta.autoOptimize.optimizeWrite: true

        delta.autoOptimize.autoCompact: true

        spark.sql.streaming.stateStore.providerClass: org.apache.spark.sql.execution.streaming.state.RocksDBStateStoreProvider

        spark.sql.streaming.stateStore.rocksdb.changelogCheckpointing.enabled: true

      aws_attributes:

        first_on_demand: 1

      # availability: SPOT_WITH_FALLBACK

      num_workers: 2

      node_type_id: m5d.large

      driver_node_type_id: m5d.large

      data_security_mode: SINGLE_USER

      runtime_engine: STANDARD

      single_user_name: ${workspace.current_user.userName}

      policy_id: ${var.job_policy_id}

  m_cluster:

    default:

      spark_version: "16.4.x-scala2.12"

      spark_conf:

        spark.sql.sources.partitionOverwriteMode: dynamic

        delta.autoOptimize.optimizeWrite: true

        delta.autoOptimize.autoCompact: true

        spark.sql.streaming.stateStore.providerClass: org.apache.spark.sql.execution.streaming.state.RocksDBStateStoreProvider

        spark.sql.streaming.stateStore.rocksdb.changelogCheckpointing.enabled: true

      aws_attributes:

        first_on_demand: 1

      # availability: SPOT_WITH_FALLBACK

      num_workers: 1

      node_type_id: m5d.xlarge

      driver_node_type_id: m5d.large

      data_security_mode: SINGLE_USER

      runtime_engine: STANDARD

      single_user_name: ${workspace.current_user.userName}

      policy_id: ${var.job_policy_id}

  l_cluster:

    default:

      spark_version: "16.4.x-scala2.12"

      spark_conf:

        spark.sql.sources.partitionOverwriteMode: dynamic

        delta.autoOptimize.optimizeWrite: true

        delta.autoOptimize.autoCompact: true

        spark.sql.streaming.stateStore.providerClass: org.apache.spark.sql.execution.streaming.state.RocksDBStateStoreProvider

        spark.sql.streaming.stateStore.rocksdb.changelogCheckpointing.enabled: true

      aws_attributes:

        first_on_demand: 1

      # availability: SPOT_WITH_FALLBACK

      num_workers: 2

      node_type_id: m5d.xlarge

      driver_node_type_id: m5d.large

      data_security_mode: SINGLE_USER

      runtime_engine: STANDARD

      single_user_name: ${workspace.current_user.userName}

      policy_id: ${var.job_policy_id}

  xl_cluster:

    default:

      spark_version: "16.4.x-scala2.12"

      spark_conf:

        spark.sql.sources.partitionOverwriteMode: dynamic

        delta.autoOptimize.optimizeWrite: true

        delta.autoOptimize.autoCompact: true

        spark.sql.streaming.stateStore.providerClass: org.apache.spark.sql.execution.streaming.state.RocksDBStateStoreProvider

        spark.sql.streaming.stateStore.rocksdb.changelogCheckpointing.enabled: true

      aws_attributes:

        first_on_demand: 1

      # availability: SPOT_WITH_FALLBACK

      num_workers: 4

      node_type_id: m5d.xlarge

      driver_node_type_id: m5d.large

      data_security_mode: SINGLE_USER

      runtime_engine: STANDARD

      single_user_name: ${workspace.current_user.userName}

      policy_id: ${var.job_policy_id}

  xxl_cluster:

    default:

      spark_version: "16.4.x-scala2.12"

      spark_conf:

        spark.sql.sources.partitionOverwriteMode: dynamic

        delta.autoOptimize.optimizeWrite: true

        delta.autoOptimize.autoCompact: true

        spark.sql.streaming.stateStore.providerClass: org.apache.spark.sql.execution.streaming.state.RocksDBStateStoreProvider

        spark.sql.streaming.stateStore.rocksdb.changelogCheckpointing.enabled: true

      aws_attributes:

        first_on_demand: 1

      # availability: SPOT_WITH_FALLBACK

      num_workers: 4

      node_type_id: m5d.2xlarge

      driver_node_type_id: m5d.large

      data_security_mode: SINGLE_USER

      runtime_engine: STANDARD

      single_user_name: ${workspace.current_user.userName}

      policy_id: ${var.job_policy_id}

 

targets:

 

  dev:

    # The default target uses 'mode: development' to create a development copy.

    # - Deployed resources get prefixed with '[dev my_user_name]'

    # - Any job schedules and triggers are paused by default.

    # See also https://docs.databricks.com/dev-tools/bundles/deployment-modes.html.

    mode: development

    default: true

    workspace:

      host:

      root_path: /Workspace/Users/${workspace.current_user.userName}/.bundle/${bundle.name}/${bundle.target}

    variables:

      environnement: dev

      job_policy_id:

    permissions:

      - level: CAN_MANAGE

        user_name: ${workspace.current_user.userName}

      - level: CAN_VIEW

        service_principal_name: ${var.sp-cicd-dev-id}

      - level: CAN_MANAGE

        service_principal_name: ${var.sp-cicd-dev-id}

      - level: CAN_RUN

        user_name: ${workspace.current_user.userName}

      - level: CAN_VIEW

        user_name: ${workspace.current_user.userName}

    run_as:

      user_name: ${workspace.current_user.userName}

    sync:

      include:

        - config/