I have deployed a new databricks environment for development. I've copied a workflow from production to this environment with exactly the same compute configuration. Four notebooks that complete within minutes do not complete after 2 hours in development. I cannot find anything that would explain this phenomenon.

below is the json for the workflow

{
  "job_id": 568410428134074,
  "creator_user_name": "*",
  "run_as_user_name": "*",
  "run_as_owner": true,
  "settings": {
    "name": "gold flow",
    "email_notifications": {},
    "webhook_notifications": {},
    "timeout_seconds": 0,
    "max_concurrent_runs": 1,
    "tasks": [
      {
        "task_key": "tranform_master_prospect_customer",
        "run_if": "ALL_SUCCESS",
        "notebook_task": {
          "notebook_path": "/Workspace/live/transform_master_prospect_customer",
          "source": "WORKSPACE"
        },
        "job_cluster_key": "Parallel_Job",
        "timeout_seconds": 0,
        "email_notifications": {},
        "webhook_notifications": {}
      },
      {
        "task_key": "transform_cust_invoice_trans_cost_amount_ss",
        "run_if": "ALL_SUCCESS",
        "notebook_task": {
          "notebook_path": "/Workspace/live/transform_cust_invoice_trans_cost_amount_ss",
          "source": "WORKSPACE"
        },
        "job_cluster_key": "Parallel_Job",
        "timeout_seconds": 0,
        "email_notifications": {},
        "webhook_notifications": {}
      },
      {
        "task_key": "transform_dim_cost_center",
        "run_if": "ALL_SUCCESS",
        "notebook_task": {
          "notebook_path": "/Workspace/live/transform_dim_cost_center",
          "source": "WORKSPACE"
        },
        "job_cluster_key": "Parallel_Job",
        "timeout_seconds": 0,
        "email_notifications": {},
        "webhook_notifications": {}
      },
      {
        "task_key": "gold_dim_cost_center",
        "depends_on": [
          {
            "task_key": "transform_dim_cost_center"
          }
        ],
        "run_if": "ALL_SUCCESS",
        "notebook_task": {
          "notebook_path": "/Workspace/live/gold_dim_cost_center",
          "source": "WORKSPACE"
        },
        "job_cluster_key": "Parallel_Job",
        "timeout_seconds": 0,
        "email_notifications": {},
        "webhook_notifications": {}
      },
      {
        "task_key": "transform_dim_customer",
        "run_if": "ALL_SUCCESS",
        "notebook_task": {
          "notebook_path": "/Workspace/live/transform_dim_customer",
          "source": "WORKSPACE"
        },
        "job_cluster_key": "Parallel_Job",
        "timeout_seconds": 0,
        "email_notifications": {},
        "webhook_notifications": {}
      },
      {
        "task_key": "gold_dim_customer",
        "depends_on": [
          {
            "task_key": "transform_dim_customer"
          }
        ],
        "run_if": "ALL_SUCCESS",
        "notebook_task": {
          "notebook_path": "/Workspace/live/gold_dim_customer",
          "source": "WORKSPACE"
        },
        "job_cluster_key": "Parallel_Job",
        "timeout_seconds": 0,
        "email_notifications": {},
        "webhook_notifications": {}
      },
      {
        "task_key": "gold_dim_service_contract",
        "depends_on": [
          {
            "task_key": "transform_dim_customer"
          }
        ],
        "run_if": "ALL_SUCCESS",
        "notebook_task": {
          "notebook_path": "/Workspace/live/transform_dim_service_contract",
          "source": "WORKSPACE"
        },
        "job_cluster_key": "Parallel_Job",
        "timeout_seconds": 0,
        "email_notifications": {},
        "webhook_notifications": {}
      },
      {
        "task_key": "gold_dim_service_rework_status",
        "depends_on": [
          {
            "task_key": "transform_dim_customer"
          }
        ],
        "run_if": "ALL_SUCCESS",
        "notebook_task": {
          "notebook_path": "/Workspace/live/transform_dim_service_rework_status",
          "source": "WORKSPACE"
        },
        "job_cluster_key": "Parallel_Job",
        "timeout_seconds": 0,
        "email_notifications": {},
        "webhook_notifications": {}
      },
      {
        "task_key": "transform_dim_department",
        "run_if": "ALL_SUCCESS",
        "notebook_task": {
          "notebook_path": "/Workspace/live/transform_dim_department",
          "source": "WORKSPACE"
        },
        "job_cluster_key": "Parallel_Job",
        "timeout_seconds": 0,
        "email_notifications": {},
        "webhook_notifications": {}
      },
      {
        "task_key": "gold_dim_department",
        "depends_on": [
          {
            "task_key": "transform_dim_department"
          }
        ],
        "run_if": "ALL_SUCCESS",
        "notebook_task": {
          "notebook_path": "/Workspace/live/gold_dim_department",
          "source": "WORKSPACE"
        },
        "job_cluster_key": "Parallel_Job",
        "timeout_seconds": 0,
        "email_notifications": {},
        "webhook_notifications": {}
      },
      {
        "task_key": "transform_dim_division",
        "run_if": "ALL_SUCCESS",
        "notebook_task": {
          "notebook_path": "/Workspace/live/transform_dim_division",
          "source": "WORKSPACE"
        },
        "job_cluster_key": "Parallel_Job",
        "timeout_seconds": 0,
        "email_notifications": {},
        "webhook_notifications": {}
      },
      {
        "task_key": "gold_dim_division",
        "depends_on": [
          {
            "task_key": "transform_dim_division"
          }
        ],
        "run_if": "ALL_SUCCESS",
        "notebook_task": {
          "notebook_path": "/Workspace/live/gold_dim_division",
          "source": "WORKSPACE"
        },
        "job_cluster_key": "Parallel_Job",
        "timeout_seconds": 0,
        "email_notifications": {},
        "webhook_notifications": {}
      },
      {
        "task_key": "transform_dim_employee",
        "run_if": "ALL_SUCCESS",
        "notebook_task": {
          "notebook_path": "/Workspace/live/transform_dim_employee",
          "source": "WORKSPACE"
        },
        "job_cluster_key": "Parallel_Job",
        "timeout_seconds": 0,
        "email_notifications": {},
        "webhook_notifications": {}
      },
      {
        "task_key": "gold_dim_employee",
        "depends_on": [
          {
            "task_key": "transform_dim_employee"
          }
        ],
        "run_if": "ALL_SUCCESS",
        "notebook_task": {
          "notebook_path": "/Workspace/live/gold_dim_employee",
          "source": "WORKSPACE"
        },
        "job_cluster_key": "Parallel_Job",
        "timeout_seconds": 0,
        "email_notifications": {},
        "webhook_notifications": {}
      },
      {
        "task_key": "transform_dim_inventory_location",
        "run_if": "ALL_SUCCESS",
        "notebook_task": {
          "notebook_path": "/Workspace/live/transform_dim_inventory_location",
          "source": "WORKSPACE"
        },
        "job_cluster_key": "Parallel_Job",
        "timeout_seconds": 0,
        "email_notifications": {},
        "webhook_notifications": {}
      },
      {
        "task_key": "gold_dim_inventory_location",
        "depends_on": [
          {
            "task_key": "transform_dim_inventory_location"
          }
        ],
        "run_if": "ALL_SUCCESS",
        "notebook_task": {
          "notebook_path": "/Workspace/live/gold_dim_inventory_location",
          "source": "WORKSPACE"
        },
        "job_cluster_key": "Parallel_Job",
        "timeout_seconds": 0,
        "email_notifications": {},
        "webhook_notifications": {}
      },
      {
        "task_key": "transform_dim_item",
        "run_if": "ALL_SUCCESS",
        "notebook_task": {
          "notebook_path": "/Workspace/live/transform_dim_item",
          "source": "WORKSPACE"
        },
        "job_cluster_key": "Parallel_Job",
        "timeout_seconds": 0,
        "email_notifications": {},
        "webhook_notifications": {}
      },
      {
        "task_key": "gold_dim_item",
        "depends_on": [
          {
            "task_key": "transform_dim_item"
          }
        ],
        "run_if": "ALL_SUCCESS",
        "notebook_task": {
          "notebook_path": "/Workspace/live/gold_dim_item",
          "source": "WORKSPACE"
        },
        "job_cluster_key": "Parallel_Job",
        "timeout_seconds": 0,
        "email_notifications": {},
        "webhook_notifications": {}
      },
      {
        "task_key": "transform_dim_marketing_program",
        "run_if": "ALL_SUCCESS",
        "notebook_task": {
          "notebook_path": "/Workspace/live/transform_dim_marketing_program",
          "source": "WORKSPACE"
        },
        "job_cluster_key": "Parallel_Job",
        "timeout_seconds": 0,
        "email_notifications": {},
        "webhook_notifications": {}
      },
      {
        "task_key": "gold_dim_marketing_program",
        "depends_on": [
          {
            "task_key": "transform_dim_marketing_program"
          }
        ],
        "run_if": "ALL_SUCCESS",
        "notebook_task": {
          "notebook_path": "/Workspace/live/gold_dim_marketing_program",
          "source": "WORKSPACE"
        },
        "job_cluster_key": "Parallel_Job",
        "timeout_seconds": 0,
        "email_notifications": {},
        "webhook_notifications": {}
      },
      {
        "task_key": "transform_dim_order_class",
        "run_if": "ALL_SUCCESS",
        "notebook_task": {
          "notebook_path": "/Workspace/live/transform_dim_order_class",
          "source": "WORKSPACE"
        },
        "job_cluster_key": "Parallel_Job",
        "timeout_seconds": 0,
        "email_notifications": {},
        "webhook_notifications": {}
      },
      {
        "task_key": "gold_dim_order_class",
        "depends_on": [
          {
            "task_key": "transform_dim_order_class"
          }
        ],
        "run_if": "ALL_SUCCESS",
        "notebook_task": {
          "notebook_path": "/Workspace/live/gold_dim_order_class",
          "source": "WORKSPACE"
        },
        "job_cluster_key": "Parallel_Job",
        "timeout_seconds": 0,
        "email_notifications": {},
        "webhook_notifications": {}
      },
      {
        "task_key": "transform_dim_prospect_customer",
        "depends_on": [
          {
            "task_key": "tranform_master_prospect_customer"
          }
        ],
        "run_if": "ALL_SUCCESS",
        "notebook_task": {
          "notebook_path": "/Workspace/live/transform_dim_prospect_customer",
          "source": "WORKSPACE"
        },
        "job_cluster_key": "Parallel_Job",
        "timeout_seconds": 0,
        "email_notifications": {},
        "webhook_notifications": {}
      },
      {
        "task_key": "gold_dim_prospect_customer",
        "depends_on": [
          {
            "task_key": "transform_dim_prospect_customer"
          }
        ],
        "run_if": "ALL_SUCCESS",
        "notebook_task": {
          "notebook_path": "/Workspace/live/gold_dim_prospect_customer",
          "source": "WORKSPACE"
        },
        "job_cluster_key": "Parallel_Job",
        "timeout_seconds": 0,
        "email_notifications": {},
        "webhook_notifications": {}
      },
      {
        "task_key": "transform_dim_salesperson",
        "depends_on": [
          {
            "task_key": "transform_dim_employee"
          }
        ],
        "run_if": "ALL_SUCCESS",
        "notebook_task": {
          "notebook_path": "/Workspace/live/transform_dim_salesperson",
          "source": "WORKSPACE"
        },
        "job_cluster_key": "Parallel_Job",
        "timeout_seconds": 0,
        "email_notifications": {},
        "webhook_notifications": {}
      },
      {
        "task_key": "gold_dim_salesperson",
        "depends_on": [
          {
            "task_key": "transform_dim_salesperson"
          }
        ],
        "run_if": "ALL_SUCCESS",
        "notebook_task": {
          "notebook_path": "/Workspace/live/gold_dim_salesperson",
          "source": "WORKSPACE"
        },
        "job_cluster_key": "Parallel_Job",
        "timeout_seconds": 0,
        "email_notifications": {},
        "webhook_notifications": {}
      },
      {
        "task_key": "transform_dim_service_call_status_type",
        "run_if": "ALL_SUCCESS",
        "notebook_task": {
          "notebook_path": "/Workspace/live/transform_dim_service_call_status_type",
          "source": "WORKSPACE"
        },
        "job_cluster_key": "Parallel_Job",
        "timeout_seconds": 0,
        "email_notifications": {},
        "webhook_notifications": {}
      },
      {
        "task_key": "gold_dim_service_call_status_type",
        "depends_on": [
          {
            "task_key": "transform_dim_service_call_status_type"
          }
        ],
        "run_if": "ALL_SUCCESS",
        "notebook_task": {
          "notebook_path": "/Workspace/live/gold_dim_service_call_status_type",
          "source": "WORKSPACE"
        },
        "job_cluster_key": "Parallel_Job",
        "timeout_seconds": 0,
        "email_notifications": {},
        "webhook_notifications": {}
      },
      {
        "task_key": "transform_dim_service_call_type",
        "run_if": "ALL_SUCCESS",
        "notebook_task": {
          "notebook_path": "/Workspace/live/transform_dim_service_call_type",
          "source": "WORKSPACE"
        },
        "job_cluster_key": "Parallel_Job",
        "timeout_seconds": 0,
        "email_notifications": {},
        "webhook_notifications": {}
      },
      {
        "task_key": "gold_dim_service_call_type",
        "depends_on": [
          {
            "task_key": "transform_dim_service_call_type"
          }
        ],
        "run_if": "ALL_SUCCESS",
        "notebook_task": {
          "notebook_path": "/Workspace/live/gold_dim_service_call_type",
          "source": "WORKSPACE"
        },
        "job_cluster_key": "Parallel_Job",
        "timeout_seconds": 0,
        "email_notifications": {},
        "webhook_notifications": {}
      },
      {
        "task_key": "transform_dim_service_component",
        "run_if": "ALL_SUCCESS",
        "notebook_task": {
          "notebook_path": "/Workspace/live/transform_dim_service_component",
          "source": "WORKSPACE"
        },
        "job_cluster_key": "Parallel_Job",
        "timeout_seconds": 0,
        "email_notifications": {},
        "webhook_notifications": {}
      },
      {
        "task_key": "gold_dim_service_component",
        "depends_on": [
          {
            "task_key": "transform_dim_service_component"
          }
        ],
        "run_if": "ALL_SUCCESS",
        "notebook_task": {
          "notebook_path": "/Workspace/live/gold_dim_service_component",
          "source": "WORKSPACE"
        },
        "job_cluster_key": "Parallel_Job",
        "timeout_seconds": 0,
        "email_notifications": {},
        "webhook_notifications": {}
      },
      {
        "task_key": "transform_dim_service_contract",
        "run_if": "ALL_SUCCESS",
        "notebook_task": {
          "notebook_path": "/Workspace/live/transform_dim_service_contract",
          "source": "WORKSPACE"
        },
        "job_cluster_key": "Parallel_Job",
        "timeout_seconds": 0,
        "email_notifications": {},
        "webhook_notifications": {}
      },
      {
        "task_key": "transform_dim_service_job_code",
        "run_if": "ALL_SUCCESS",
        "notebook_task": {
          "notebook_path": "/Workspace/live/transform_dim_service_job_code",
          "source": "WORKSPACE"
        },
        "job_cluster_key": "Parallel_Job",
        "timeout_seconds": 0,
        "email_notifications": {},
        "webhook_notifications": {}
      },
      {
        "task_key": "gold_dim_service_job_code",
        "depends_on": [
          {
            "task_key": "transform_dim_service_job_code"
          }
        ],
        "run_if": "ALL_SUCCESS",
        "notebook_task": {
          "notebook_path": "/Workspace/live/gold_dim_service_job_code",
          "source": "WORKSPACE"
        },
        "job_cluster_key": "Parallel_Job",
        "timeout_seconds": 0,
        "email_notifications": {},
        "webhook_notifications": {}
      },
      {
        "task_key": "transform_dim_service_rework_status",
        "run_if": "ALL_SUCCESS",
        "notebook_task": {
          "notebook_path": "/Workspace/live/transform_dim_service_rework_status",
          "source": "WORKSPACE"
        },
        "job_cluster_key": "Parallel_Job",
        "timeout_seconds": 0,
        "email_notifications": {},
        "webhook_notifications": {}
      },
      {
        "task_key": "transform_dim_source_of_supply",
        "run_if": "ALL_SUCCESS",
        "notebook_task": {
          "notebook_path": "/Workspace/live/transform_dim_source_of_supply",
          "source": "WORKSPACE"
        },
        "job_cluster_key": "Parallel_Job",
        "timeout_seconds": 0,
        "email_notifications": {},
        "webhook_notifications": {}
      },
      {
        "task_key": "gold_dim_source_of_supply",
        "depends_on": [
          {
            "task_key": "transform_dim_source_of_supply"
          }
        ],
        "run_if": "ALL_SUCCESS",
        "notebook_task": {
          "notebook_path": "/Workspace/live/gold_dim_source_of_supply",
          "source": "WORKSPACE"
        },
        "job_cluster_key": "Parallel_Job",
        "timeout_seconds": 0,
        "email_notifications": {},
        "webhook_notifications": {}
      },
      {
        "task_key": "transform_dim_store",
        "run_if": "ALL_SUCCESS",
        "notebook_task": {
          "notebook_path": "/Workspace/live/transform_dim_store",
          "source": "WORKSPACE"
        },
        "job_cluster_key": "Parallel_Job",
        "timeout_seconds": 0,
        "email_notifications": {},
        "webhook_notifications": {}
      },
      {
        "task_key": "gold_dim_store",
        "depends_on": [
          {
            "task_key": "transform_dim_store"
          }
        ],
        "run_if": "ALL_SUCCESS",
        "notebook_task": {
          "notebook_path": "/Workspace/live/gold_dim_store",
          "source": "WORKSPACE"
        },
        "job_cluster_key": "Parallel_Job",
        "timeout_seconds": 0,
        "email_notifications": {},
        "webhook_notifications": {}
      },
      {
        "task_key": "transform_dim_vendor",
        "run_if": "ALL_SUCCESS",
        "notebook_task": {
          "notebook_path": "/Workspace/live/transform_dim_vendor",
          "source": "WORKSPACE"
        },
        "job_cluster_key": "Parallel_Job",
        "timeout_seconds": 0,
        "email_notifications": {},
        "webhook_notifications": {}
      },
      {
        "task_key": "gold_dim_vendor",
        "depends_on": [
          {
            "task_key": "transform_dim_vendor"
          }
        ],
        "run_if": "ALL_SUCCESS",
        "notebook_task": {
          "notebook_path": "/Workspace/live/gold_dim_vendor",
          "source": "WORKSPACE"
        },
        "job_cluster_key": "Parallel_Job",
        "timeout_seconds": 0,
        "email_notifications": {},
        "webhook_notifications": {}
      },
      {
        "task_key": "transform_fact_customer_invoice",
        "run_if": "ALL_SUCCESS",
        "notebook_task": {
          "notebook_path": "/Workspace/live/transform_fact_customer_invoice",
          "source": "WORKSPACE"
        },
        "job_cluster_key": "Parallel_Job",
        "timeout_seconds": 0,
        "email_notifications": {},
        "webhook_notifications": {}
      },
      {
        "task_key": "transform_fact_purchase_order",
        "run_if": "ALL_SUCCESS",
        "notebook_task": {
          "notebook_path": "/Workspace/live/transform_fact_purchase_order",
          "source": "WORKSPACE"
        },
        "job_cluster_key": "Parallel_Job",
        "timeout_seconds": 0,
        "email_notifications": {},
        "webhook_notifications": {}
      },
      {
        "task_key": "transform_fact_safety_incident",
        "run_if": "ALL_SUCCESS",
        "notebook_task": {
          "notebook_path": "/Workspace/live/transform_fact_safety_incident",
          "source": "WORKSPACE"
        },
        "job_cluster_key": "Parallel_Job",
        "timeout_seconds": 0,
        "email_notifications": {},
        "webhook_notifications": {}
      },
      {
        "task_key": "gold_fact_safety_incident",
        "depends_on": [
          {
            "task_key": "transform_fact_safety_incident"
          },
          {
            "task_key": "gold_dim_store"
          },
          {
            "task_key": "gold_dim_employee"
          }
        ],
        "run_if": "ALL_SUCCESS",
        "notebook_task": {
          "notebook_path": "/Workspace/live/gold_fact_safety_incident",
          "source": "WORKSPACE"
        },
        "job_cluster_key": "Parallel_Job",
        "timeout_seconds": 0,
        "email_notifications": {},
        "webhook_notifications": {}
      },
      {
        "task_key": "transform_fact_service_call_segment_transaction",
        "run_if": "ALL_SUCCESS",
        "notebook_task": {
          "notebook_path": "/Workspace/live/transform_fact_service_call_segment_transaction",
          "source": "WORKSPACE"
        },
        "job_cluster_key": "Parallel_Job",
        "timeout_seconds": 0,
        "email_notifications": {},
        "webhook_notifications": {}
      },
      {
        "task_key": "transform_stage_equipment_fleet_history_span",
        "run_if": "ALL_SUCCESS",
        "notebook_task": {
          "notebook_path": "/Workspace/live/transform_stage_equipment_fleet_history_span",
          "source": "WORKSPACE"
        },
        "job_cluster_key": "Parallel_Job",
        "timeout_seconds": 0,
        "email_notifications": {},
        "webhook_notifications": {}
      },
      {
        "task_key": "transform_dim_equipment",
        "depends_on": [
          {
            "task_key": "transform_stage_equipment_fleet_history_span"
          }
        ],
        "run_if": "ALL_SUCCESS",
        "notebook_task": {
          "notebook_path": "/Workspace/live/transform_dim_equipment",
          "source": "WORKSPACE"
        },
        "job_cluster_key": "Parallel_Job",
        "timeout_seconds": 0,
        "email_notifications": {},
        "webhook_notifications": {}
      },
      {
        "task_key": "gold_dim_equipment",
        "depends_on": [
          {
            "task_key": "transform_dim_equipment"
          }
        ],
        "run_if": "ALL_SUCCESS",
        "notebook_task": {
          "notebook_path": "/Workspace/live/gold_dim_equipment",
          "source": "WORKSPACE"
        },
        "job_cluster_key": "Parallel_Job",
        "timeout_seconds": 0,
        "email_notifications": {},
        "webhook_notifications": {}
      },
      {
        "task_key": "gold_fact_customer_invoice",
        "depends_on": [
          {
            "task_key": "gold_dim_order_class"
          },
          {
            "task_key": "gold_dim_store"
          },
          {
            "task_key": "gold_dim_marketing_program"
          },
          {
            "task_key": "transform_fact_customer_invoice"
          },
          {
            "task_key": "gold_dim_equipment"
          },
          {
            "task_key": "gold_dim_cost_center"
          },
          {
            "task_key": "gold_dim_division"
          },
          {
            "task_key": "gold_dim_department"
          },
          {
            "task_key": "gold_dim_salesperson"
          },
          {
            "task_key": "gold_dim_inventory_location"
          },
          {
            "task_key": "gold_dim_item"
          },
          {
            "task_key": "gold_dim_customer"
          },
          {
            "task_key": "gold_dim_employee"
          }
        ],
        "run_if": "ALL_SUCCESS",
        "notebook_task": {
          "notebook_path": "/Workspace/live/gold_fact_customer_invoice",
          "source": "WORKSPACE"
        },
        "job_cluster_key": "Parallel_Job",
        "timeout_seconds": 0,
        "email_notifications": {},
        "webhook_notifications": {}
      },
      {
        "task_key": "gold_fact_purchase_order",
        "depends_on": [
          {
            "task_key": "transform_fact_purchase_order"
          },
          {
            "task_key": "gold_dim_vendor"
          },
          {
            "task_key": "gold_dim_order_class"
          },
          {
            "task_key": "gold_dim_store"
          },
          {
            "task_key": "gold_dim_equipment"
          },
          {
            "task_key": "gold_dim_cost_center"
          },
          {
            "task_key": "gold_dim_division"
          },
          {
            "task_key": "gold_dim_department"
          },
          {
            "task_key": "gold_dim_prospect_customer"
          },
          {
            "task_key": "gold_dim_source_of_supply"
          },
          {
            "task_key": "gold_dim_inventory_location"
          },
          {
            "task_key": "gold_dim_item"
          },
          {
            "task_key": "gold_dim_employee"
          }
        ],
        "run_if": "ALL_SUCCESS",
        "notebook_task": {
          "notebook_path": "/Workspace/live/gold_fact_purchase_order",
          "source": "WORKSPACE"
        },
        "job_cluster_key": "Parallel_Job",
        "timeout_seconds": 0,
        "email_notifications": {},
        "webhook_notifications": {}
      },
      {
        "task_key": "gold_fact_service_call_segment_transaction",
        "depends_on": [
          {
            "task_key": "gold_dim_order_class"
          },
          {
            "task_key": "gold_dim_store"
          },
          {
            "task_key": "gold_dim_equipment"
          },
          {
            "task_key": "gold_dim_cost_center"
          },
          {
            "task_key": "gold_dim_service_contract"
          },
          {
            "task_key": "transform_fact_service_call_segment_transaction"
          },
          {
            "task_key": "gold_dim_division"
          },
          {
            "task_key": "gold_dim_service_job_code"
          },
          {
            "task_key": "gold_dim_department"
          },
          {
            "task_key": "gold_dim_service_rework_status"
          },
          {
            "task_key": "gold_dim_prospect_customer"
          },
          {
            "task_key": "gold_dim_service_component"
          },
          {
            "task_key": "gold_dim_service_call_status_type"
          },
          {
            "task_key": "gold_dim_inventory_location"
          },
          {
            "task_key": "gold_dim_item"
          },
          {
            "task_key": "gold_dim_employee"
          }
        ],
        "run_if": "ALL_SUCCESS",
        "notebook_task": {
          "notebook_path": "/Workspace/live/gold_fact_service_call_segment_transaction",
          "source": "WORKSPACE"
        },
        "job_cluster_key": "Parallel_Job",
        "timeout_seconds": 0,
        "email_notifications": {},
        "webhook_notifications": {}
      },
      {
        "task_key": "transform_stage_service_invoice_Labor",
        "run_if": "ALL_SUCCESS",
        "notebook_task": {
          "notebook_path": "/Workspace/live/transform_stage_service_invoice_Labor",
          "source": "WORKSPACE"
        },
        "job_cluster_key": "Parallel_Job",
        "timeout_seconds": 0,
        "email_notifications": {},
        "webhook_notifications": {}
      },
      {
        "task_key": "transform_stage_service_invoice_cost",
        "run_if": "ALL_SUCCESS",
        "notebook_task": {
          "notebook_path": "/Workspace/live/transform_stage_service_invoice_Cost",
          "source": "WORKSPACE"
        },
        "job_cluster_key": "Parallel_Job",
        "timeout_seconds": 0,
        "email_notifications": {},
        "webhook_notifications": {}
      },
      {
        "task_key": "transform_stage_service_invoice_item",
        "run_if": "ALL_SUCCESS",
        "notebook_task": {
          "notebook_path": "/Workspace/live/transform_stage_service_invoice_item",
          "source": "WORKSPACE"
        },
        "job_cluster_key": "Parallel_Job",
        "timeout_seconds": 0,
        "email_notifications": {},
        "webhook_notifications": {}
      },
      {
        "task_key": "transform_stage_service_invoice_revenue",
        "run_if": "ALL_SUCCESS",
        "notebook_task": {
          "notebook_path": "/Workspace/live/transform_stage_service_invoice_revenue",
          "source": "WORKSPACE"
        },
        "job_cluster_key": "Parallel_Job",
        "timeout_seconds": 0,
        "email_notifications": {},
        "webhook_notifications": {}
      },
      {
        "task_key": "transform_fact_service_invoice",
        "depends_on": [
          {
            "task_key": "transform_stage_service_invoice_Labor"
          },
          {
            "task_key": "transform_stage_service_invoice_cost"
          },
          {
            "task_key": "transform_stage_service_invoice_item"
          },
          {
            "task_key": "transform_stage_service_invoice_revenue"
          }
        ],
        "run_if": "ALL_SUCCESS",
        "notebook_task": {
          "notebook_path": "/Workspace/live/transform_fact_service_invoice",
          "source": "WORKSPACE"
        },
        "job_cluster_key": "Parallel_Job",
        "timeout_seconds": 0,
        "email_notifications": {},
        "webhook_notifications": {}
      },
      {
        "task_key": "gold_fact_service_invoice",
        "depends_on": [
          {
            "task_key": "gold_dim_order_class"
          },
          {
            "task_key": "transform_fact_service_invoice"
          },
          {
            "task_key": "gold_dim_store"
          },
          {
            "task_key": "gold_dim_marketing_program"
          },
          {
            "task_key": "gold_dim_equipment"
          },
          {
            "task_key": "gold_dim_service_call_type"
          },
          {
            "task_key": "gold_dim_cost_center"
          },
          {
            "task_key": "gold_dim_division"
          },
          {
            "task_key": "gold_dim_service_job_code"
          },
          {
            "task_key": "gold_dim_department"
          },
          {
            "task_key": "gold_dim_prospect_customer"
          },
          {
            "task_key": "gold_dim_service_component"
          },
          {
            "task_key": "gold_dim_service_call_status_type"
          },
          {
            "task_key": "gold_dim_salesperson"
          },
          {
            "task_key": "gold_dim_inventory_location"
          },
          {
            "task_key": "gold_dim_item"
          },
          {
            "task_key": "gold_dim_customer"
          },
          {
            "task_key": "gold_dim_employee"
          }
        ],
        "run_if": "ALL_SUCCESS",
        "notebook_task": {
          "notebook_path": "/Workspace/live/gold_fact_service_invoice",
          "source": "WORKSPACE"
        },
        "job_cluster_key": "Parallel_Job",
        "timeout_seconds": 0,
        "email_notifications": {},
        "webhook_notifications": {}
      }
    ],
    "job_clusters": [
      {
        "job_cluster_key": "Parallel_Job",
        "new_cluster": {
          "cluster_name": "",
          "spark_version": "15.4.x-scala2.12",
          "azure_attributes": {
            "first_on_demand": 1,
            "availability": "SPOT_WITH_FALLBACK_AZURE",
            "spot_bid_max_price": -1
          },
          "node_type_id": "Standard_D32ads_v5",
          "spark_env_vars": {
            "PYSPARK_PYTHON": "/databricks/python3/bin/python3"
          },
          "enable_elastic_disk": true,
          "data_security_mode": "SINGLE_USER",
          "runtime_engine": "PHOTON",
          "autoscale": {
            "min_workers": 2,
            "max_workers": 8
          }
        }
      }
    ],
    "format": "MULTI_TASK",
    "queue": {
      "enabled": true
    },
    "parameters": [
      {
        "name": "workspacePath",
        "default": "/Workspace/live"
      }
    ]
  },
  "created_time": 1730754786941
}

One of the more complex notebooks:

# Databricks notebook source
# MAGIC %sql
# MAGIC create or replace table gold.fact_service_call_segment_transaction as
# MAGIC select
# MAGIC   --begin dimsegment keys
# MAGIC   coalesce(
# MAGIC     dim_service_call_status_type_key.dim_service_call_status_type_key,
# MAGIC     - 1
# MAGIC   ) as dim_service_call_status_type_key,
# MAGIC   coalesce(
# MAGIC     dim_service_call_cost_center.dim_cost_center_key,
# MAGIC     - 1
# MAGIC   ) as dim_service_call_cost_center_key,
# MAGIC   coalesce(
# MAGIC     dim_service_call_prospect_customer.dim_prospect_customer_key,
# MAGIC     - 1
# MAGIC   ) as dim_service_call_prospect_customer_key,
# MAGIC   coalesce(
# MAGIC     dim_service_call_worker_employee.dim_employee_key,
# MAGIC     - 1
# MAGIC   ) as dim_service_call_worker_employee_key,
# MAGIC   coalesce(
# MAGIC     dim_service_call_order_class.dim_order_class_key,
# MAGIC     - 1
# MAGIC   ) as dim_service_call_order_class_key,
# MAGIC   coalesce(
# MAGIC     dim_service_call_fin_dim_cost_center.dim_cost_center_key,
# MAGIC     - 1
# MAGIC   ) as dim_service_call_fin_dim_cost_center_key,
# MAGIC   coalesce(
# MAGIC     dim_service_call_fin_dim_prospect_customer.dim_prospect_customer_key,
# MAGIC     - 1
# MAGIC   ) as dim_service_call_fin_dim_prospect_customer_key,
# MAGIC   coalesce(
# MAGIC     dim_service_call_fin_dim_department.dim_department_key,
# MAGIC     - 1
# MAGIC   ) as dim_service_call_fin_dim_department_key,
# MAGIC   coalesce(
# MAGIC     dim_service_call_fin_dim_division.dim_division_key,
# MAGIC     - 1
# MAGIC   ) as dim_service_call_fin_dim_division_key,
# MAGIC   coalesce(
# MAGIC     dim_service_call_fin_dim_equipment.dim_equipment_key,
# MAGIC     - 1
# MAGIC   ) as dim_service_call_fin_dim_equipment_key,
# MAGIC   coalesce(
# MAGIC     dim_service_call_fin_dim_store.dim_store_key,
# MAGIC     - 1
# MAGIC   ) as dim_service_call_fin_dim_store_key,
# MAGIC   coalesce(
# MAGIC     dim_service_call_warehouse_inventory_location.dim_inventory_location_key,
# MAGIC     - 1
# MAGIC   ) as dim_service_call_warehouse_inventory_location_key --begin dimsegment keys
# MAGIC ,
# MAGIC   coalesce(
# MAGIC     dim_service_job_code.dim_service_job_code_key,
# MAGIC     - 1
# MAGIC   ) as dim_segment_service_job_code_key,
# MAGIC   coalesce(
# MAGIC     dim_service_component.dim_service_component_key,
# MAGIC     - 1
# MAGIC   ) as dim_segment_service_component_key,
# MAGIC   coalesce(dim_segment_cost_center.dim_cost_center_key, - 1) as dim_segment_cost_center_key,
# MAGIC   coalesce(
# MAGIC     dim_segment_rework_status.dim_service_rework_status_key,
# MAGIC     - 1
# MAGIC   ) as dim_segment_rework_status_key,
# MAGIC   coalesce(
# MAGIC     dim_segment_rework_technician_employee.dim_employee_key,
# MAGIC     - 1
# MAGIC   ) as dim_segment_rework_technician_employee_key,
# MAGIC   coalesce(
# MAGIC     dim_segment_invoice_account_prospect_customer.dim_prospect_customer_key,
# MAGIC     - 1
# MAGIC   ) as dim_segment_invoice_account_prospect_customer_key --begin dim_transaction keys
# MAGIC ,
# MAGIC   coalesce(dim_item.dim_item_key, - 1) as dim_transaction_item_key,
# MAGIC   coalesce(
# MAGIC     dim_transaction_worker_employee_key.dim_employee_key,
# MAGIC     - 1
# MAGIC   ) as dim_transaction_worker_employee_key,
# MAGIC   coalesce(dim_equipment.dim_equipment_key, - 1) as dim_transaction_fin_dim_equipment_key,
# MAGIC   coalesce(
# MAGIC     dim_trans_dim_cost_center.dim_cost_center_key,
# MAGIC     - 1
# MAGIC   ) as dim_transaction_fin_dim_cost_center_key,
# MAGIC   coalesce(dim_department.dim_department_key, - 1) as dim_transaction_fin_dim_department_key,
# MAGIC   coalesce(dim_division.dim_division_key, - 1) as dim_transaction_fin_dim_division_key,
# MAGIC   coalesce(dim_store.dim_store_key, - 1) as dim_transaction_fin_dim_store_key,
# MAGIC   coalesce(
# MAGIC     dim_trans_dim_prospect_customer.dim_prospect_customer_key,
# MAGIC     - 1
# MAGIC   ) as dim_transaction_fin_dim_prospect_customer_key,
# MAGIC   coalesce(dim_contract.dim_service_contract_key, - 1) as dim_transaction_fin_dim_service_contract_key,
# MAGIC   coalesce(dim_employee.dim_employee_key, - 1) as dim_transaction_fin_dim_employee_key --begin fact servicecallsegmenttransaction fields
# MAGIC ,
# MAGIC   service_call_rec_id as bk_service_call_rec_id,
# MAGIC   segment_rec_id as bk_segment_rec_id,
# MAGIC   transaction_line_rec_id as bk_transaction_line_rec_id,
# MAGIC   transaction_line_trans_id as bk_transaction_line_trans_id,
# MAGIC   service_call_number,
# MAGIC   service_call_priority,
# MAGIC   service_call_open_date,
# MAGIC   service_call_close_date,
# MAGIC   service_call_promise_start_date,
# MAGIC   service_call_promise_end_date,
# MAGIC   service_call_first_labor_date,
# MAGIC   service_call_last_labor_date,
# MAGIC   service_call_service_zone_id,
# MAGIC   service_call_service_zone_description,
# MAGIC   service_call_smu,
# MAGIC   service_call_created_date,
# MAGIC   service_call_modified_date,
# MAGIC   project_id,
# MAGIC   segment_id,
# MAGIC   segment_first_labor_date,
# MAGIC   segment_last_labor_date,
# MAGIC   segment_description,
# MAGIC   segment_standard_job_id,
# MAGIC   segment_status,
# MAGIC   segment_sales_order_id,
# MAGIC   segment_estimated_duration,
# MAGIC   segment_target_duration,
# MAGIC   segment_rework_segment_id,
# MAGIC   segment_service_letter_id,
# MAGIC   segment_warranty_type_id,
# MAGIC   segment_warranty_type,
# MAGIC   transaction_line_source_table,
# MAGIC   transaction_line_actual_or_standard_job,
# MAGIC   transaction_line_type,
# MAGIC   transaction_line_project_date,
# MAGIC   transaction_line_project_id,
# MAGIC   transaction_line_category_group_id,
# MAGIC   transaction_line_category_id,
# MAGIC   transaction_line_description,
# MAGIC   transaction_line_quantity,
# MAGIC   transaction_line_cost_price_amount,
# MAGIC   transaction_line_cost_total_amount,
# MAGIC   transaction_line_sale_price_amount,
# MAGIC   transaction_line_sale_total_amount,
# MAGIC   transaction_line_property_id,
# MAGIC   transaction_line_status,
# MAGIC   transaction_line_funding_source,
# MAGIC   transaction_line_invoice_status,
# MAGIC   transaction_line_voucher
# MAGIC from
# MAGIC   transform.fact_service_call_segment_transaction f
# MAGIC   left join gold.dim_item dim_item on dim_item.bk_item_id = f.transaction_line_item_id
# MAGIC   left join gold.dim_equipment dim_equipment on dim_equipment.bk_equipment_id = f.transaction_line_fin_dim_equipment
# MAGIC   and f.transaction_line_project_date between dim_equipment.row_start_date
# MAGIC   and dim_equipment.row_end_date
# MAGIC   left join gold.dim_cost_center dim_service_call_cost_center on dim_service_call_cost_center.bk_cost_center_id = f.service_call_cost_center
# MAGIC   left join gold.dim_cost_center dim_segment_cost_center on dim_segment_cost_center.bk_cost_center_id = f.segment_cost_center
# MAGIC   left join gold.dim_cost_center dim_trans_dim_cost_center on dim_trans_dim_cost_center.bk_cost_center_id = f.transaction_line_fin_dim_cost_center
# MAGIC   left join gold.dim_department dim_department on dim_department.bk_department_id = f.transaction_line_fin_dim_department
# MAGIC   left join gold.dim_division dim_division on dim_division.bk_division_id = f.transaction_line_fin_dim_division
# MAGIC   left join gold.dim_store dim_store on dim_store.bk_store_id = f.transaction_line_fin_dim_store
# MAGIC   left join gold.dim_prospect_customer dim_service_call_prospect_customer on dim_service_call_prospect_customer.customer_account = f.service_call_customer_account
# MAGIC   and dim_service_call_prospect_customer.customer_account <> ''
# MAGIC   left join gold.dim_prospect_customer dim_segment_invoice_account_prospect_customer on dim_segment_invoice_account_prospect_customer.customer_account = f.segment_invoice_account
# MAGIC   and dim_segment_invoice_account_prospect_customer.customer_account <> ''
# MAGIC   left join gold.dim_prospect_customer dim_trans_dim_prospect_customer on dim_trans_dim_prospect_customer.customer_account = f.transaction_line_fin_dim_customer
# MAGIC   and dim_trans_dim_prospect_customer.customer_account <> ''
# MAGIC   left join gold.dim_service_contract dim_contract on dim_contract.bk_contract_id = f.transaction_line_fin_dim_contract
# MAGIC   left join gold.dim_employee dim_segment_rework_technician_employee on dim_segment_rework_technician_employee.bk_hcm_worker_rec_id = f.segment_rework_technician
# MAGIC   and f.segment_first_labor_date between dim_segment_rework_technician_employee.row_start_date
# MAGIC   and dim_segment_rework_technician_employee.row_end_date
# MAGIC   left join gold.dim_employee dim_employee on dim_employee.bk_personnel_number = f.transaction_line_fin_dim_employee
# MAGIC   and f.transaction_line_project_date between dim_employee.row_start_date
# MAGIC   and dim_employee.row_end_date
# MAGIC   and dim_employee.bk_personnel_number <> ''
# MAGIC   left join gold.dim_employee dim_transaction_worker_employee_key on dim_transaction_worker_employee_key.bk_hcm_worker_rec_id = f.transaction_line_worker_rec_id
# MAGIC   and f.transaction_line_project_date between dim_transaction_worker_employee_key.row_start_date
# MAGIC   and dim_transaction_worker_employee_key.row_end_date
# MAGIC   and dim_transaction_worker_employee_key.bk_personnel_number <> ''
# MAGIC   and dim_transaction_worker_employee_key.bk_hcm_worker_rec_id > 0
# MAGIC   left join gold.dim_service_job_code dim_service_job_code on dim_service_job_code.bk_service_job_code_rec_id = f.segment_job_code_rec_id
# MAGIC   left join gold.dim_service_component dim_service_component on dim_service_component.bk_service_component_rec_id = f.segment_component_code_rec_id
# MAGIC   left join gold.dim_service_rework_status dim_segment_rework_status on dim_segment_rework_status.bk_service_rework_status_rec_id = f.segment_rework_status_rec_id
# MAGIC   left join gold.dim_service_call_status_type dim_service_call_status_type_key on dim_service_call_status_type_key.bk_service_call_type_table_rec_id = f.service_call_type_rec_id
# MAGIC   and dim_service_call_status_type_key.bk_service_call_type_line_rec_id = f.service_call_status_rec_id
# MAGIC   left join gold.dim_employee dim_service_call_worker_employee on dim_service_call_worker_employee.bk_hcm_worker_rec_id = f.service_call_worker_rec_id
# MAGIC   and coalesce(
# MAGIC     f.service_call_first_labor_date,
# MAGIC     to_date('01/01/1900', 'MM/dd/yyyy')
# MAGIC   ) between dim_service_call_worker_employee.row_start_date
# MAGIC   and dim_service_call_worker_employee.row_end_date
# MAGIC   left join gold.dim_order_class dim_service_call_order_class on dim_service_call_order_class.bk_order_class_rec_id = f.service_call_order_class_rec_id
# MAGIC   left join gold.dim_cost_center dim_service_call_fin_dim_cost_center on dim_service_call_fin_dim_cost_center.bk_cost_center_id = f.service_call_fin_dim_cost_center
# MAGIC   left join gold.dim_prospect_customer dim_service_call_fin_dim_prospect_customer on dim_service_call_fin_dim_prospect_customer.customer_account = f.service_call_fin_dim_customer
# MAGIC   and dim_service_call_fin_dim_prospect_customer.customer_account <> ''
# MAGIC   left join gold.dim_department dim_service_call_fin_dim_department on dim_service_call_fin_dim_department.bk_department_id = f.service_call_fin_dim_department
# MAGIC   left join gold.dim_division dim_service_call_fin_dim_division on dim_service_call_fin_dim_division.bk_division_id = f.service_call_fin_dim_division
# MAGIC   left join gold.dim_equipment dim_service_call_fin_dim_equipment on dim_service_call_fin_dim_equipment.bk_equipment_id = f.service_call_fin_dim_equipment
# MAGIC   and f.service_call_first_labor_date between dim_service_call_fin_dim_equipment.row_start_date
# MAGIC   and dim_service_call_fin_dim_equipment.row_end_date
# MAGIC   left join gold.dim_store dim_service_call_fin_dim_store on dim_service_call_fin_dim_store.bk_store_id = f.service_call_fin_dim_store
# MAGIC   left join gold.dim_inventory_location dim_service_call_warehouse_inventory_location on dim_service_call_warehouse_inventory_location.bk_invent_location_id = f.service_call_warehouse_invent_location_id;

I'll accept any direction anyone has to offer.

Regards,

Trent