Options
- Mark as New
- Bookmark
- Subscribe
- Mute
- Subscribe to RSS Feed
- Permalink
- Report Inappropriate Content
04-17-2023 02:13 AM
Hi @Suteja Kanuri
My pipeline is running DBR 12.1.
Is it because we're trying to do a merge with tables that are streaming live tables?
In case I'm missing something, here is my pipeline config:
{
"id": "**********************************",
"clusters": [
{
"label": "default",
"autoscale": {
"min_workers": 1,
"max_workers": 2,
"mode": "ENHANCED"
}
}
],
"development": true,
"continuous": true,
"channel": "PREVIEW",
"edition": "ADVANCED",
"photon": false,
"libraries": [
{
"notebook": {
"path": "/Repos/************/************/DLT/attendance/regPeriods/dlt_reg_periods_python"
}
},
.
.
.
.
}
],
"name": "CABS-Streaming-Processes_Dev",
"storage": "dbfs:/pipelines/************************",
"configuration": {
"pipelines.enableTrackHistory": "true",
"spark.databricks.secureVariableSubstitute.enabled": "false",
"spark.databaseUser": "{{secrets/***********/**************}}",
"spark.databasePassword": "{{secrets/**********/*******************}}",
"spark.databricks.delta.schema.autoMerge.enabled": "true"
},
"target": "CABS_dataProcessing_Dev"
}