I am attempting to automate Jobs creation using Databrick Terraform provider. I have a number of task that will "depends_on" each other and am trying to use dynamic content to do this. Each task name is stored in a string array so looping over the task is what i'm trying to do. The first item in the array would not have a depends_on section. I have made numerous attempts, but the depends_on section errors with a null or empty string. Any suggestions?
Sample Code:
locals {
sorted_tasks = sort(var.tasks)
}
resource "databricks_job" "new_job" {
run_as {
user_name = "email@email.com"
}
max_concurrent_runs = 1
format = "MULTI_TASK"
name = "JOBNAME"
email_notifications {
on_failure = ["email@email.com"]
no_alert_for_skipped_runs = false
}
schedule {
quartz_cron_expression = "16 0 0 * * ?"
timezone_id = "America/Boise"
pause_status = "UNPAUSED"
}
dynamic task {
for_each = { for idx, task_key in local.sorted_tasks : idx => task_key }
content {
task_key = local.sorted_tasks[task.key]
depends_on {
task_key = task.key > 0 ? local.sorted_tasks[task.key-1] : []
}
notebook_task {
notebook_path = "/PATH/${local.sorted_tasks[task.key]}"
source = "WORKSPACE"
}
job_cluster_key = "CLUSTERNAME"
library {
pypi {
package = "snowflake-connector-python"
}
}
library {
pypi {
package = "arrow"
}
}
timeout_seconds = 0
}
}
job_cluster {
job_cluster_key = "CLUSTERNAME"
new_cluster {
cluster_name = ""
spark_version = "12.2.x-scala2.12"
spark_conf = {
"spark.databricks.delta.preview.enabled" = "true"
}
azure_attributes {
first_on_demand = 1
availability = "SPOT_WITH_FALLBACK_AZURE"
spot_bid_max_price = 100
}
node_type_id = "Standard_DS3_v2"
enable_elastic_disk = true
policy_id = "POLICYID"
data_security_mode = "LEGACY_SINGLE_USER_STANDARD"
runtime_engine = "PHOTON"
autoscale {
min_workers = 2
max_workers = 4
}
}
}
}