jonhieb
New Contributor III

Of course. In this example, I use the argument pipeline_task to reference a DLT pipeline that I created previously. This allows you to schedule your DLT pipeline inside your workflow.

# Job to orchestrate data_quality_pipelines DLT Pipeline.
resources:
  jobs:
    data_quality_pipelines_job:
      name: schedule_data_quality_job

      schedule:
        quartz_cron_expression: "0 0 8 ? * Mon" # At 8:00:00am, on Monday
        timezone_id: "America/Sao_Paulo"

      timeout_seconds: 3600  # 1 hour

      email_notifications:
        on_failure:
          - ${workspace.current_user.userName}
      webhook_notifications:
        on_failure:
          - id: ${var.webhook_id}

      tasks:
        - task_key: data_quality_task
          pipeline_task: 
            pipeline_id: ${var.input_tables_pipeline_id}
            full_refresh: false

        - task_key: output_data_quality_task
          pipeline_task:
            pipeline_id: ${var.output_tables_pipeline_id}
            full_refresh: false
        
        - task_key: notify_business_areas
          depends_on:
            - task_key: data_quality_task
            - task_key: output_data_quality_task
          run_job_task:
            job_id: ${var.send_notifications_job_id}
          
        - task_key: create_jira_tasks
          depends_on:
            - task_key: notify_business_areas
          run_job_task:
            job_id: ${var.create_jira_tasks_job_id}
          max_retries: 0

      run_as:
        user_name: xxxxxxxx@yyyyyyy
        
      parameters:
        - name: notification_conf_file
          default: ${var.notification_conf_file}