Hi. I have upgraded to 14.3 recently and facing the same issue in production.
The below is my update statement
UPDATE table1
SET starttime = getArgument("time1"), endtime = null
WHERE ConnectionID=getArgument("ConnectionInfo") AND GroupName = getArgument("JobGroup")
AND EXISTS (SELECT date FROM table1 WHERE ConnectionID=getArgument("ConnectionInfo") AND GroupName = getArgument("JobGroup") AND endtime IS NOT NULL AND getArgument("RunType") = 'start')
AND IsSyncRequired = true
below is the error we are facing :
"runError": "ConcurrentAppendException: Files were added to partition [ConnectionID=con1, GroupName=group2] by a concurrent update. Please try the operation again.
{
"timestamp": 1719876118108,
"userId": "7939453779672853",
"userName": "x",
"operation": "UPDATE",
"operationParameters": {
"predicate": [
"(((ConnectionID#99116459 = getArgument(ConnectionInfo)) AND (GroupName#99116455 = getArgument(JobGroup))) AND (isnotnull(scalar-subquery#99116271 []) AND IsSyncRequired#99116460))"
]
},
"job": {
"jobId": "x",
"jobName": "x",
"runId": "x",
"jobOwnerId": "x",
"triggerType": "manual"
},
"notebook": {
"notebookId": "x"
},
"clusterId": "x",
"readVersion": 1513710,
"isolationLevel": "WriteSerializable",
"isBlindAppend": false,
"operationMetrics": {
"numRemovedFiles": "1",
"numRemovedBytes": "1732",
"numCopiedRows": "0",
"numDeletionVectorsAdded": "0",
"numDeletionVectorsRemoved": "0",
"numAddedChangeFiles": "0",
"executionTimeMs": "9837",
"numDeletionVectorsUpdated": "0",
"scanTimeMs": "7487",
"numAddedFiles": "1",
"numUpdatedRows": "1",
"numAddedBytes": "1631",
"rewriteTimeMs": "2309"
},
"tags": {
"noRowsCopied": "false",
"delta.rowTracking.preserved": "false",
"restoresDeletedRows": "false"
},
"engineInfo": "Databricks-Runtime/14.3.x-scala2.12",
"txnId": "xx"
}
Please note that my group1 update is failing with error on group2 conflicting commit when ideally both of them should go to different partitions.
The partition columns are both ConnectionID and GroupName.
Can someone please help me understand when my where clause specifies both, why is it still giving me ConcurrentAppendException?
