cancel
Showing results for 
Search instead for 
Did you mean: 
Data Engineering
Join discussions on data engineering best practices, architectures, and optimization strategies within the Databricks Community. Exchange insights and solutions with fellow data engineers.
cancel
Showing results for 
Search instead for 
Did you mean: 

asset bundle configuration

seefoods
Contributor II

Hi Guys

I want to use this approach to give more flexibility of my asset bundle. But when i try to implement this i have some error like unknown field. SomeOne, could show me an example ? 

Thanx, 




databricks-dab-repo/
├── databricks.yml # Bundle definition
├── resources/
│ ├── workflows/
│ │ ├── my_pipeline.yml # YAML pipeline def
│ │ └── my_pipeline_job.yml # YAML job def that runs pipeline
│ ├── clusters/
│ │ ├── dev_cluster.yml # development cluster def
│ │ └── prod_cluster.yml # production def
├── src/
│ ├── dlt_pipeline.ipynb # pipeline notebook
│ └── mypython.py # Additional Python
└── README.md

2 REPLIES 2

ilir_nuredini
Honored Contributor

Hello @seefoods ,

May you share the error codes you get and what example you wanna get exactly (like maybe databricks.yaml file that handles that structure, or..). Thank you!

Best, Ilir

seefoods
Contributor II

Actually its works, i have declare all my clusters config in variables field 

 

permissions:

  - level: CAN_VIEW

    group_name: data-engineer

  - level: CAN_RUN

    group_name: data-engineer

  - level: CAN_MANAGE

    group_name: data-engineer

  - level: CAN_RUN

    user_name: toto@company.com

 

 

variables:

  environnement:

    default: dev

  job_policy_id:

    default:

  notification_emails:

    default:

      - toto@company.com

  sp-cicd-dev-id:

    default:

  sp-cicd-qa-id:

    default:

  sp-cicd-preprod-id:

    default:

  sp-cicd-prod-id:

    default:

 

  databricks_utils:

    default: "

  personal_compute:

    default:

  use_autoloader:

    default: true

  xs_cluster:

    description: "very small cluster"

    type: complex

    default:

      spark_version: "16.4.x-scala2.12"

      spark_conf:

        spark.sql.sources.partitionOverwriteMode: dynamic

        delta.autoOptimize.optimizeWrite: true

        delta.autoOptimize.autoCompact: true

        spark.sql.streaming.stateStore.providerClass: org.apache.spark.sql.execution.streaming.state.RocksDBStateStoreProvider

        spark.sql.streaming.stateStore.rocksdb.changelogCheckpointing.enabled: true

      aws_attributes:

        first_on_demand: 1

      # availability: SPOT_WITH_FALLBACK

      num_workers: 1

      node_type_id: m5d.large

      driver_node_type_id: m5d.large

      data_security_mode: SINGLE_USER

      runtime_engine: STANDARD

      single_user_name: ${workspace.current_user.userName}

      policy_id: ${var.job_policy_id}

  s_cluster:

    description: "small cluster"

    type: complex

    default:

      spark_version: "16.4.x-scala2.12"

      spark_conf:

        spark.sql.sources.partitionOverwriteMode: dynamic

        delta.autoOptimize.optimizeWrite: true

        delta.autoOptimize.autoCompact: true

        spark.sql.streaming.stateStore.providerClass: org.apache.spark.sql.execution.streaming.state.RocksDBStateStoreProvider

        spark.sql.streaming.stateStore.rocksdb.changelogCheckpointing.enabled: true

      aws_attributes:

        first_on_demand: 1

      # availability: SPOT_WITH_FALLBACK

      num_workers: 2

      node_type_id: m5d.large

      driver_node_type_id: m5d.large

      data_security_mode: SINGLE_USER

      runtime_engine: STANDARD

      single_user_name: ${workspace.current_user.userName}

      policy_id: ${var.job_policy_id}

  m_cluster:

    default:

      spark_version: "16.4.x-scala2.12"

      spark_conf:

        spark.sql.sources.partitionOverwriteMode: dynamic

        delta.autoOptimize.optimizeWrite: true

        delta.autoOptimize.autoCompact: true

        spark.sql.streaming.stateStore.providerClass: org.apache.spark.sql.execution.streaming.state.RocksDBStateStoreProvider

        spark.sql.streaming.stateStore.rocksdb.changelogCheckpointing.enabled: true

      aws_attributes:

        first_on_demand: 1

      # availability: SPOT_WITH_FALLBACK

      num_workers: 1

      node_type_id: m5d.xlarge

      driver_node_type_id: m5d.large

      data_security_mode: SINGLE_USER

      runtime_engine: STANDARD

      single_user_name: ${workspace.current_user.userName}

      policy_id: ${var.job_policy_id}

  l_cluster:

    default:

      spark_version: "16.4.x-scala2.12"

      spark_conf:

        spark.sql.sources.partitionOverwriteMode: dynamic

        delta.autoOptimize.optimizeWrite: true

        delta.autoOptimize.autoCompact: true

        spark.sql.streaming.stateStore.providerClass: org.apache.spark.sql.execution.streaming.state.RocksDBStateStoreProvider

        spark.sql.streaming.stateStore.rocksdb.changelogCheckpointing.enabled: true

      aws_attributes:

        first_on_demand: 1

      # availability: SPOT_WITH_FALLBACK

      num_workers: 2

      node_type_id: m5d.xlarge

      driver_node_type_id: m5d.large

      data_security_mode: SINGLE_USER

      runtime_engine: STANDARD

      single_user_name: ${workspace.current_user.userName}

      policy_id: ${var.job_policy_id}

  xl_cluster:

    default:

      spark_version: "16.4.x-scala2.12"

      spark_conf:

        spark.sql.sources.partitionOverwriteMode: dynamic

        delta.autoOptimize.optimizeWrite: true

        delta.autoOptimize.autoCompact: true

        spark.sql.streaming.stateStore.providerClass: org.apache.spark.sql.execution.streaming.state.RocksDBStateStoreProvider

        spark.sql.streaming.stateStore.rocksdb.changelogCheckpointing.enabled: true

      aws_attributes:

        first_on_demand: 1

      # availability: SPOT_WITH_FALLBACK

      num_workers: 4

      node_type_id: m5d.xlarge

      driver_node_type_id: m5d.large

      data_security_mode: SINGLE_USER

      runtime_engine: STANDARD

      single_user_name: ${workspace.current_user.userName}

      policy_id: ${var.job_policy_id}

  xxl_cluster:

    default:

      spark_version: "16.4.x-scala2.12"

      spark_conf:

        spark.sql.sources.partitionOverwriteMode: dynamic

        delta.autoOptimize.optimizeWrite: true

        delta.autoOptimize.autoCompact: true

        spark.sql.streaming.stateStore.providerClass: org.apache.spark.sql.execution.streaming.state.RocksDBStateStoreProvider

        spark.sql.streaming.stateStore.rocksdb.changelogCheckpointing.enabled: true

      aws_attributes:

        first_on_demand: 1

      # availability: SPOT_WITH_FALLBACK

      num_workers: 4

      node_type_id: m5d.2xlarge

      driver_node_type_id: m5d.large

      data_security_mode: SINGLE_USER

      runtime_engine: STANDARD

      single_user_name: ${workspace.current_user.userName}

      policy_id: ${var.job_policy_id}

 

targets:

 

  dev:

    # The default target uses 'mode: development' to create a development copy.

    # - Deployed resources get prefixed with '[dev my_user_name]'

    # - Any job schedules and triggers are paused by default.

    # See also https://docs.databricks.com/dev-tools/bundles/deployment-modes.html.

    mode: development

    default: true

    workspace:

      host:

      root_path: /Workspace/Users/${workspace.current_user.userName}/.bundle/${bundle.name}/${bundle.target}

    variables:

      environnement: dev

      job_policy_id:

    permissions:

      - level: CAN_MANAGE

        user_name: ${workspace.current_user.userName}

      - level: CAN_VIEW

        service_principal_name: ${var.sp-cicd-dev-id}

      - level: CAN_MANAGE

        service_principal_name: ${var.sp-cicd-dev-id}

      - level: CAN_RUN

        user_name: ${workspace.current_user.userName}

      - level: CAN_VIEW

        user_name: ${workspace.current_user.userName}

    run_as:

      user_name: ${workspace.current_user.userName}

    sync:

      include:

        - config/

Join Us as a Local Community Builder!

Passionate about hosting events and connecting people? Help us grow a vibrant local community—sign up today to get started!

Sign Up Now