cancel
Showing results forย 
Search instead forย 
Did you mean:ย 
Data Engineering
Join discussions on data engineering best practices, architectures, and optimization strategies within the Databricks Community. Exchange insights and solutions with fellow data engineers.
cancel
Showing results forย 
Search instead forย 
Did you mean:ย 

Parameter "expand_tasks" on List job runs request seams not to be working (databricsk api)

Mildred
New Contributor

I'm setting it as True, but it doesn't return the cluster_instance info. 

Here is the function I'm using:

def get_job_runs(job_id):
    """
    Fetches job runs for a specific job from Databricks Jobs API.
    """
    headers = {
        "Authorization": f"Bearer {DATABRICKS_TOKEN}",
        "Content-Type": "application/json"
    }

    url = f"{DATABRICKS_HOST}/api/2.2/jobs/runs/list"
    data = {
        "job_id": job_id,
        "expand_tasks": True
    }

    try:
        response = session.get(url, headers=headers, params=data, verify=False)
        response.raise_for_status()
        response_data = response.json()
        runs = response_data.get("runs", [])
        return runs

    except requests.exceptions.RequestException as e:
        print(f"Error during request: {e}")
        return []
    except json.JSONDecodeError as e:
        print(f"Error decoding JSON response: {e}")
        return []
    except Exception as e:
        print(f"An unexpected error occurred: {e}")
        return []

One of the returned items:

{'job_id': 960697773932569,
 'run_id': 605591465138074,
 'creator_user_name': '62fd43e8-4d50-4e21-963e-680b2f4859c4',
 'number_in_job': 605591465138074,
 'original_attempt_run_id': 605591465138074,
 'state': {'life_cycle_state': 'TERMINATED',
  'result_state': 'SUCCESS',
  'state_message': '',
  'user_cancelled_or_timedout': False},
 'job_parameters': [{'name': 'libraries',
   'default': '',
   'value': 'h2o-pysparkling-3.4==3.44.0.1.post1 kraken-bv==0.37.7'},
  {'name': 'wg_env', 'default': '', 'value': 'prd'},
  {'name': 'wg_safra', 'default': '', 'value': '202503'}],
 'start_time': 1747408029996,
 'setup_duration': 0,
 'execution_duration': 0,
 'cleanup_duration': 0,
 'end_time': 1747431182635,
 'run_duration': 23152639,
 'trigger': 'ONE_TIME',
 'run_name': 'job_krakenbricks-data-crmx-prop-leves-pap',
 'run_page_url': 'https://7589927382780461.1.gcp.databricks.com/?o=7589927382780461#job/960697773932569/run/605591465138074',
 'run_type': 'JOB_RUN',
 'format': 'MULTI_TASK',
 'status': {'state': 'TERMINATED',
  'termination_details': {'code': 'SUCCESS',
   'type': 'SUCCESS',
   'message': ''}},
 'job_run_id': 605591465138074}
1 REPLY 1

Krishna_S
Databricks Employee
Databricks Employee

Hi @Mildred 

The way you passed the data for the expand_tasks parameter is wrong:

data = {
data = {
"job_id": job_id,
"expand_tasks": "true"
}
It should not be passed as Python boolean values, but as a string "true" or "false"
Once you do that will see the entire 

[{'job_id': 154595207433406,
'run_id': 166719176114693,
'creator_user_name': '<EMAIL_REDACTED>',
'number_in_job': 166719176114693,
'original_attempt_run_id': 166719176114693,
'state': {'life_cycle_state': 'INTERNAL_ERROR',
'result_state': 'FAILED',
'state_message': 'Task Jar_task failed with message: Library installation failed for library due to user error for jar: "/Volumes/field_demos/demo_db_cyl/managed_volume_cyl/helloworld.jar"\n Error messages:\nLibrary installation attempted on the driver node of cluster 1003-154822-zzimx8ej and failed. Unity Catalog object not found. Verify catalog, schema, and volume exist. Error code: ERROR_UC_ASSET_NOT_FOUND, error message: java.util.concurrent.ExecutionException: org.apache.spark.sql.catalyst.analysis.NoSuchDatabaseException: [SCHEMA_NOT_FOUND] The schema `field_demos.demo_db_cyl` cannot be found. Verify the spelling and correctness of the schema and catalog.\nIf you did not qualify the name with a catalog, verify the current_schema() output, or qualify the name with the correct catalog.\nTo tolerate the error on drop use DROP SCHEMA IF EXISTS. SQLSTATE: 42704.',
'user_cancelled_or_timedout': False},
'job_parameters': [{'name': 'jar_task_param', 'default': 'jar_task_value'},
{'name': 'python_whl_param', 'default': 'python_whl_value'}],
'start_time': 1759506501515,
'setup_duration': 0,
'execution_duration': 0,
'cleanup_duration': 0,
'end_time': 1759506798349,
'run_duration': 296834,
'trigger': 'ONE_TIME',
'run_name': 'Multi-task job',
'run_page_url': 'https://<HOST_REDACTED>/?o=1444828305810485#job/154595207433406/run/166719176114693',
'run_type': 'JOB_RUN',
'tasks': [{'run_id': 21365227935982,
'task_key': 'clone_of_Print_args_jar_task',
'run_if': 'ALL_SUCCESS',
'spark_jar_task': {'jar_uri': '',
'main_class_name': 'PrintArgs',
'parameters': ['{{job.parameters.jar_task_param}}',
'{{job.parameters.jar_task_param}} -- Hello World'],
'run_as_repl': True},
'job_cluster_key': 'demo_job_cluster',
'libraries': [{'jar': '/Volumes/field_demos/demo_db_cyl/managed_volume_cyl/PrintArgs.jar'}],
'state': {'life_cycle_state': 'INTERNAL_ERROR',
'result_state': 'FAILED',
'state_message': 'Library installation failed for library due to user error for jar: "/Volumes/field_demos/demo_db_cyl/managed_volume_cyl/PrintArgs.jar"\n Error messages:\nLibrary installation attempted on the driver node of cluster 1003-154822-zzimx8ej and failed. Unity Catalog object not found. Verify catalog, schema, and volume exist. Error code: ERROR_UC_ASSET_NOT_FOUND, error message: java.util.concurrent.ExecutionException: org.apache.spark.sql.catalyst.analysis.NoSuchDatabaseException: [SCHEMA_NOT_FOUND] The schema `field_demos.demo_db_cyl` cannot be found. Verify the spelling and correctness of the schema and catalog.\nIf you did not qualify the name with a catalog, verify the current_schema() output, or qualify the name with the correct catalog.\nTo tolerate the error on drop use DROP SCHEMA IF EXISTS. SQLSTATE: 42704',
'user_cancelled_or_timedout': False},
'run_page_url': 'https://<HOST_REDACTED>/?o=1444828305810485#job/154595207433406/run/21365227935982',
'start_time': 1759506501563,
'setup_duration': 246000,
'execution_duration': 0,
'cleanup_duration': 40000,
'end_time': 1759506788003,
'cluster_instance': {'cluster_id': '1003-154822-zzimx8ej',
'spark_context_id': '3612902441764836731'},
'attempt_number': 0,
'status': {'state': 'TERMINATED',
'termination_details': {'code': 'LIBRARY_INSTALLATION_ERROR',
'type': 'CLIENT_ERROR',
'message': 'Library installation failed for library due to user error for jar: "/Volumes/field_demos/demo_db_cyl/managed_volume_cyl/PrintArgs.jar"\n Error messages:\nLibrary installation attempted on the driver node of cluster 1003-154822-zzimx8ej and failed. Unity Catalog object not found. Verify catalog, schema, and volume exist. Error code: ERROR_UC_ASSET_NOT_FOUND, error message: java.util.concurrent.ExecutionException: org.apache.spark.sql.catalyst.analysis.NoSuchDatabaseException: [SCHEMA_NOT_FOUND] The schema `field_demos.demo_db_cyl` cannot be found. Verify the spelling and correctness of the schema and catalog.\nIf you did not qualify the name with a catalog, verify the current_schema() output, or qualify the name with the correct catalog.\nTo tolerate the error on drop use DROP SCHEMA IF EXISTS. SQLSTATE: 42704'}}},
{'run_id': 850371806566397,
'task_key': 'Jar_task',
'run_if': 'ALL_SUCCESS',
'spark_jar_task': {'jar_uri': '',
'main_class_name': 'HelloWorld',
'parameters': ['{{job.parameters.jar_task_param}}'],
'run_as_repl': True},
'job_cluster_key': 'demo_job_cluster',
'libraries': [{'jar': '/Volumes/field_demos/demo_db_cyl/managed_volume_cyl/helloworld.jar'}],
'state': {'life_cycle_state': 'INTERNAL_ERROR',
'result_state': 'FAILED',
'state_message': 'Library installation failed for library due to user error for jar: "/Volumes/field_demos/demo_db_cyl/managed_volume_cyl/helloworld.jar"\n Error messages:\nLibrary installation attempted on the driver node of cluster 1003-154822-zzimx8ej and failed. Unity Catalog object not found. Verify catalog, schema, and volume exist. Error code: ERROR_UC_ASSET_NOT_FOUND, error message: java.util.concurrent.ExecutionException: org.apache.spark.sql.catalyst.analysis.NoSuchDatabaseException: [SCHEMA_NOT_FOUND] The schema `field_demos.demo_db_cyl` cannot be found. Verify the spelling and correctness of the schema and catalog.\nIf you did not qualify the name with a catalog, verify the current_schema() output, or qualify the name with the correct catalog.\nTo tolerate the error on drop use DROP SCHEMA IF EXISTS. SQLSTATE: 42704',
'user_cancelled_or_timedout': False},
'run_page_url': 'https://<HOST_REDACTED>/?o=1444828305810485#job/154595207433406/run/850371806566397',
'start_time': 1759506501561,
'setup_duration': 246000,
'execution_duration': 0,
'cleanup_duration': 30000,
'end_time': 1759506777925,
'cluster_instance': {'cluster_id': '1003-154822-zzimx8ej',
'spark_context_id': '3612902441764836731'},
'attempt_number': 0,
'status': {'state': 'TERMINATED',
'termination_details': {'code': 'LIBRARY_INSTALLATION_ERROR',
'type': 'CLIENT_ERROR',
'message': 'Library installation failed for library due to user error for jar: "/Volumes/field_demos/demo_db_cyl/managed_volume_cyl/helloworld.jar"\n Error messages:\nLibrary installation attempted on the driver node of cluster 1003-154822-zzimx8ej and failed. Unity Catalog object not found. Verify catalog, schema, and volume exist. Error code: ERROR_UC_ASSET_NOT_FOUND, error message: java.util.concurrent.ExecutionException: org.apache.spark.sql.catalyst.analysis.NoSuchDatabaseException: [SCHEMA_NOT_FOUND] The schema `field_demos.demo_db_cyl` cannot be found. Verify the spelling and correctness of the schema and catalog.\nIf you did not qualify the name with a catalog, verify the current_schema() output, or qualify the name with the correct catalog.\nTo tolerate the error on drop use DROP SCHEMA IF EXISTS. SQLSTATE: 42704'}}},
{'run_id': 712924030676166,
'task_key': 'Print_args_jar_task',
'run_if': 'ALL_SUCCESS',
'spark_jar_task': {'jar_uri': '',
'main_class_name': 'PrintArgs',
'parameters': ['{{job.parameters.jar_task_param}}',
'{{job.parameters.jar_task_param}} -- Hello World'],
'run_as_repl': True},
'job_cluster_key': 'demo_job_cluster',
'libraries': [{'jar': '/Volumes/field_demos/demo_db_cyl/managed_volume_cyl/PrintArgs.jar'}],
'state': {'life_cycle_state': 'INTERNAL_ERROR',
'result_state': 'FAILED',
'state_message': 'Library installation failed for library due to user error for jar: "/Volumes/field_demos/demo_db_cyl/managed_volume_cyl/PrintArgs.jar"\n Error messages:\nLibrary installation attempted on the driver node of cluster 1003-154822-zzimx8ej and failed. Unity Catalog object not found. Verify catalog, schema, and volume exist. Error code: ERROR_UC_ASSET_NOT_FOUND, error message: java.util.concurrent.ExecutionException: org.apache.spark.sql.catalyst.analysis.NoSuchDatabaseException: [SCHEMA_NOT_FOUND] The schema `field_demos.demo_db_cyl` cannot be found. Verify the spelling and correctness of the schema and catalog.\nIf you did not qualify the name with a catalog, verify the current_schema() output, or qualify the name with the correct catalog.\nTo tolerate the error on drop use DROP SCHEMA IF EXISTS. SQLSTATE: 42704',
'user_cancelled_or_timedout': False},
'run_page_url': 'https://<HOST_REDACTED>/?o=1444828305810485#job/154595207433406/run/712924030676166',
'start_time': 1759506501562,
'setup_duration': 246000,
'execution_duration': 0,
'cleanup_duration': 40000,
'end_time': 1759506787954,
'cluster_instance': {'cluster_id': '1003-154822-zzimx8ej',
'spark_context_id': '3612902441764836731'},
'attempt_number': 0,
'status': {'state': 'TERMINATED',
'termination_details': {'code': 'LIBRARY_INSTALLATION_ERROR',
'type': 'CLIENT_ERROR',
'message': 'Library installation failed for library due to user error for jar