resources:
jobs:
X:
name: X
tasks:
- task_key: X
for_each_task:
inputs: "{{job.parameters.input}}"
task:
task_key: X
existing_cluster_id: ${var.my_cluster_id}
spark_python_task:
python_file: ../../src/shared/XX.py
queue:
enabled: true
parameters:
- name: input
default: <How to format this correctly??>
for now all im getting is incorrect formatting errors.
when i do the same job but without the for each and inputs; the default value is a list or arguments that gets parsed
e.g ["--table_name=table1", "--table_schema=schema1"] ...
resources:
jobs:
X:
name: X
max_concurrent_runs: 50
tasks:
- task_key: X
existing_cluster_id: ${var.my_cluster_id}
spark_python_task:
python_file: ../../src/shared/XX.py
parameters: ["--table_name=table1", "--table_schema=schema1"]