Options
- Mark as New
- Bookmark
- Subscribe
- Mute
- Subscribe to RSS Feed
- Permalink
- Report Inappropriate Content
01-21-2025 12:33 AM
you can handle this with databricks cli and adding null resource to your terraform.
add following to your devops pipeline:
# databricks cli is needed to run local-exec inside terraform
- script: |
python3.6 -m pip install databricks-cli --user
displayName: Install databricks cli
add following to your terraform ( match it to your terraform guidelines):
# Define the job names
locals {
job_names = ["job_name_1", "job_name_2"]
}
resource "databricks_token" "this" {
comment = "Terraform Provisioning"
}
resource "null_resource" "start_stop_existing_job" {
for_each = toset(local.job_names)
provisioner "local-exec" {
command = <<-EOT
echo "Running command to stop job: $JOB_ID"
run_id=$($HOME/.local/bin/databricks runs list --active-only --job-id $JOB_ID | cut -f1 -d' ')
echo "Found run_id: $run_id"
if [ -n "$run_id" ]; then
echo "Cancelling run with run_id: $run_id"
$HOME/.local/bin/databricks runs cancel --run-id $run_id
else
echo "No active runs found for job $JOB_ID"
fi
echo "Re-running job with job_id: $JOB_ID"
EOF
sleep 30
$HOME/.local/bin/databricks jobs run-now --job-id $JOB_ID
EOT
interpreter = ["bash", "-c"]
environment = {
DATABRICKS_HOST = "https://${data.azurerm_databricks_workspace.this.workspace_url}"
DATABRICKS_TOKEN = databricks_token.this.token_value
JOB_ID = databricks_job.this[each.key].id
}
}
triggers = {
always_run = timestamp()
}
depends_on = [
databricks_job.this,
databricks_token.this
]
}
TRY and let me know your results, Thanks
