Databricks Community

thomas-totter · 3 weeks ago

Since at least two weeks (but probably even longer) our DLT pipeline posts error messages to log4j (driver logs) like the one below. I tried with both channels (preview, current), switched between serverless and classic compute and started the pipeline in triggered as well as well as in continuous mode. However, the error messages keep coming in at a very high frequency (probably after streaming updates or similar triggers). Any leads as to why this happens (and maybe even how to solve it) would be very much appreciated!

There are no corresponding messages in DLTs event log and the pipeline execution is also not negatively affected by it, at least not that i could tell so far. Any leads as to why this suddenly happens (or how to prevent it) would be much appreciated!

Shortened error message from log4j:

25/08/29 10:21:45 ERROR NativeADLGen2RequestComparisonHandler: Error in request comparison
java.lang.NumberFormatException: For input string: "Fri, 29 Aug 2025 09:02:07 GMT"

Full version:

Spoiler

25/08/29 10:21:45 ERROR NativeADLGen2RequestComparisonHandler: Error in request comparison

java.lang.NumberFormatException: For input string: "Fri, 29 Aug 2025 09:02:07 GMT"

at java.base/java.lang.NumberFormatException.forInputString(NumberFormatException.java:67)

at java.base/java.lang.Long.parseLong(Long.java:711)

at java.base/java.lang.Long.parseLong(Long.java:836)

at scala.collection.immutable.StringLike.toLong(StringLike.scala:309)

at scala.collection.immutable.StringLike.toLong$(StringLike.scala:309)

at scala.collection.immutable.StringOps.toLong(StringOps.scala:33)

at com.databricks.sql.io.NativeADLGen2RequestComparisonHandler.doHandle(NativeADLGen2RequestComparisonHandler.scala:94)

at com.databricks.sql.io.NativeADLGen2RequestComparisonHandler.beforeAttempt(NativeADLGen2RequestComparisonHandler.scala:155)

at shaded.databricks.azurebfs.org.apache.hadoop.fs.azurebfs.services.AbfsRestOperation.executeHttpOperation(AbfsRestOperation.java:396)

at shaded.databricks.azurebfs.org.apache.hadoop.fs.azurebfs.services.AbfsRestOperation.completeExecute(AbfsRestOperation.java:284)

at shaded.databricks.azurebfs.org.apache.hadoop.fs.azurebfs.services.AbfsRestOperation.lambda$execute$0(AbfsRestOperation.java:251)

at org.apache.hadoop.fs.statistics.impl.IOStatisticsBinding.measureDurationOfInvocation(IOStatisticsBinding.java:494)

at org.apache.hadoop.fs.statistics.impl.IOStatisticsBinding.trackDurationOfInvocation(IOStatisticsBinding.java:465)

at shaded.databricks.azurebfs.org.apache.hadoop.fs.azurebfs.services.AbfsRestOperation.execute(AbfsRestOperation.java:249)

at shaded.databricks.azurebfs.org.apache.hadoop.fs.azurebfs.services.AbfsClient.read(AbfsClient.java:1221)

at shaded.databricks.azurebfs.org.apache.hadoop.fs.azurebfs.services.AbfsInputStream.readRemote(AbfsInputStream.java:670)

at shaded.databricks.azurebfs.org.apache.hadoop.fs.azurebfs.services.AbfsInputStream.readInternal(AbfsInputStream.java:633)

at shaded.databricks.azurebfs.org.apache.hadoop.fs.azurebfs.services.AbfsInputStream.readOneBlock(AbfsInputStream.java:423)

at shaded.databricks.azurebfs.org.apache.hadoop.fs.azurebfs.services.AbfsInputStream.read(AbfsInputStream.java:360)

at java.base/java.io.DataInputStream.read(DataInputStream.java:151)

at com.databricks.common.filesystem.LokiAbfsInputStream.$anonfun$read$3(LokiABFS.scala:210)

at scala.runtime.java8.JFunction0$mcI$sp.apply(JFunction0$mcI$sp.java:23)

at com.databricks.common.filesystem.LokiAbfsInputStream.withExceptionRewrites(LokiABFS.scala:200)

at com.databricks.common.filesystem.LokiAbfsInputStream.read(LokiABFS.scala:210)

at java.base/java.io.DataInputStream.read(DataInputStream.java:151)

at java.base/sun.nio.cs.StreamDecoder.readBytes(StreamDecoder.java:281)

at java.base/sun.nio.cs.StreamDecoder.implRead(StreamDecoder.java:324)

at java.base/sun.nio.cs.StreamDecoder.read(StreamDecoder.java:189)

at java.base/java.io.InputStreamReader.read(InputStreamReader.java:177)

at java.base/java.io.BufferedReader.fill(BufferedReader.java:162)

at java.base/java.io.BufferedReader.readLine(BufferedReader.java:329)

at java.base/java.io.BufferedReader.readLine(BufferedReader.java:396)

at com.databricks.sql.transaction.tahoe.storage.LineClosableIterator.hasNext(LineClosableIterator.scala:50)

at scala.collection.Iterator.foreach(Iterator.scala:943)

at scala.collection.Iterator.foreach$(Iterator.scala:943)

at com.databricks.sql.transaction.tahoe.storage.LineClosableIterator.foreach(LineClosableIterator.scala:31)

at scala.collection.generic.Growable.$plus$plus$eq(Growable.scala:62)

at scala.collection.generic.Growable.$plus$plus$eq$(Growable.scala:53)

at scala.collection.immutable.VectorBuilder.$plus$plus$eq(Vector.scala:668)

at scala.collection.immutable.VectorBuilder.$plus$plus$eq(Vector.scala:645)

at scala.collection.TraversableOnce.to(TraversableOnce.scala:366)

at scala.collection.TraversableOnce.to$(TraversableOnce.scala:364)

at com.databricks.sql.transaction.tahoe.storage.LineClosableIterator.to(LineClosableIterator.scala:31)

at scala.collection.TraversableOnce.toIndexedSeq(TraversableOnce.scala:356)

at scala.collection.TraversableOnce.toIndexedSeq$(TraversableOnce.scala:356)

at com.databricks.sql.transaction.tahoe.storage.LineClosableIterator.toIndexedSeq(LineClosableIterator.scala:31)

at com.databricks.sql.transaction.tahoe.storage.LogStore.read(LogStore.scala:86)

at com.databricks.sql.transaction.tahoe.storage.LogStore.read$(LogStore.scala:83)

at com.databricks.tahoe.store.DelegatingLogStore.read(DelegatingLogStore.scala:38)

at com.databricks.sql.transaction.tahoe.sources.DeltaSource$.actions$lzycompute$1(DeltaSource.scala:1812)

at com.databricks.sql.transaction.tahoe.sources.DeltaSource$.actions$1(DeltaSource.scala:1811)

at com.databricks.sql.transaction.tahoe.sources.DeltaSource$.com$databricks$sql$transaction$tahoe$sources$DeltaSource$$createClosableIterator$1(DeltaSource.scala:1819)

at com.databricks.sql.transaction.tahoe.sources.DeltaSource$$anon$3.<init>(DeltaSource.scala:1827)

at com.databricks.sql.transaction.tahoe.sources.DeltaSource$.createRewindableActionIterator(DeltaSource.scala:1826)

at com.databricks.sql.transaction.tahoe.sources.DeltaSourceMetadataEvolutionSupport.$anonfun$collectActions$2(DeltaSourceMetadataEvolutionSupport.scala:209)

at com.databricks.sql.transaction.tahoe.storage.ClosableIterator$IteratorFlatMapCloseOp$$anon$2.hasNext(ClosableIterator.scala:89)

at scala.collection.Iterator$$anon$13.hasNext(Iterator.scala:585)

at scala.collection.Iterator.toStream(Iterator.scala:1417)

at scala.collection.Iterator.toStream$(Iterator.scala:1416)

at scala.collection.AbstractIterator.toStream(Iterator.scala:1431)

at scala.collection.TraversableOnce.toSeq(TraversableOnce.scala:354)

at scala.collection.TraversableOnce.toSeq$(TraversableOnce.scala:354)

at scala.collection.AbstractIterator.toSeq(Iterator.scala:1431)

at com.databricks.sql.transaction.tahoe.sources.DeltaSourceMetadataEvolutionSupport.$anonfun$collectMetadataActions$1(DeltaSourceMetadataEvolutionSupport.scala:289)

at com.databricks.sql.transaction.tahoe.storage.ClosableIterator.processAndClose(ClosableIterator.scala:43)

at com.databricks.sql.transaction.tahoe.storage.ClosableIterator.processAndClose$(ClosableIterator.scala:41)

at com.databricks.sql.transaction.tahoe.storage.ClosableIterator$IteratorFlatMapCloseOp$$anon$2.processAndClose(ClosableIterator.scala:78)

at com.databricks.sql.transaction.tahoe.sources.DeltaSourceMetadataEvolutionSupport.collectMetadataActions(DeltaSourceMetadataEvolutionSupport.scala:288)

at com.databricks.sql.transaction.tahoe.sources.DeltaSourceMetadataEvolutionSupport.collectMetadataActions$(DeltaSourceMetadataEvolutionSupport.scala:285)

at com.databricks.sql.transaction.tahoe.sources.DeltaSource.collectMetadataActions(DeltaSource.scala:804)

at com.databricks.sql.transaction.tahoe.sources.DeltaSourceBase.$anonfun$checkReadIncompatibleSchemaChangeOnStreamStartOnce$3(DeltaSource.scala:627)

at scala.runtime.java8.JFunction1$mcVJ$sp.apply(JFunction1$mcVJ$sp.java:23)

at scala.Option.foreach(Option.scala:407)

at com.databricks.sql.transaction.tahoe.sources.DeltaSourceBase.$anonfun$checkReadIncompatibleSchemaChangeOnStreamStartOnce$2(DeltaSource.scala:626)

at com.databricks.sql.transaction.tahoe.sources.DeltaSourceBase.$anonfun$checkReadIncompatibleSchemaChangeOnStreamStartOnce$2$adapted(DeltaSource.scala:603)

at scala.Option.foreach(Option.scala:407)

at com.databricks.sql.transaction.tahoe.sources.DeltaSourceBase.checkReadIncompatibleSchemaChangeOnStreamStartOnce(DeltaSource.scala:603)

at com.databricks.sql.transaction.tahoe.sources.DeltaSourceBase.checkReadIncompatibleSchemaChangeOnStreamStartOnce$(DeltaSource.scala:582)

at com.databricks.sql.transaction.tahoe.sources.DeltaSource.checkReadIncompatibleSchemaChangeOnStreamStartOnce(DeltaSource.scala:804)

at com.databricks.sql.transaction.tahoe.sources.DeltaSource.validateAndInitMetadataLogForPlannedBatchesDuringStreamStart(DeltaSource.scala:1464)

at com.databricks.sql.transaction.tahoe.sources.DeltaSource.$anonfun$getBatch$1(DeltaSource.scala:1362)

at com.databricks.sql.transaction.tahoe.metering.DeltaLogging.withOperationTypeTag(DeltaLogging.scala:325)

at com.databricks.sql.transaction.tahoe.metering.DeltaLogging.withOperationTypeTag$(DeltaLogging.scala:312)

at com.databricks.sql.transaction.tahoe.sources.DeltaSource.withOperationTypeTag(DeltaSource.scala:804)

at com.databricks.sql.transaction.tahoe.metering.DeltaLogging.$anonfun$recordDeltaOperationInternal$2(DeltaLogging.scala:178)

at com.databricks.spark.util.FrameProfiler$.record(FrameProfiler.scala:94)

at com.databricks.sql.transaction.tahoe.metering.DeltaLogging.recordFrameProfile(DeltaLogging.scala:418)

at com.databricks.sql.transaction.tahoe.metering.DeltaLogging.recordFrameProfile$(DeltaLogging.scala:416)

at com.databricks.sql.transaction.tahoe.sources.DeltaSource.recordFrameProfile(DeltaSource.scala:804)

at com.databricks.sql.transaction.tahoe.metering.DeltaLogging.$anonfun$recordDeltaOperationInternal$1(DeltaLogging.scala:177)

at com.databricks.logging.UsageLogging.$anonfun$recordOperation$1(UsageLogging.scala:510)

at com.databricks.logging.UsageLogging.executeThunkAndCaptureResultTags$1(UsageLogging.scala:616)

at com.databricks.logging.UsageLogging.$anonfun$recordOperationWithResultTags$4(UsageLogging.scala:643)

at com.databricks.logging.AttributionContextTracing.$anonfun$withAttributionContext$1(AttributionContextTracing.scala:49)

at com.databricks.logging.AttributionContext$.$anonfun$withValue$1(AttributionContext.scala:293)

at scala.util.DynamicVariable.withValue(DynamicVariable.scala:62)

at com.databricks.logging.AttributionContext$.withValue(AttributionContext.scala:289)

at com.databricks.logging.AttributionContextTracing.withAttributionContext(AttributionContextTracing.scala:47)

at com.databricks.logging.AttributionContextTracing.withAttributionContext$(AttributionContextTracing.scala:44)

at com.databricks.spark.util.PublicDBLogging.withAttributionContext(DatabricksSparkUsageLogger.scala:30)

at com.databricks.logging.AttributionContextTracing.withAttributionTags(AttributionContextTracing.scala:96)

at com.databricks.logging.AttributionContextTracing.withAttributionTags$(AttributionContextTracing.scala:77)

at com.databricks.spark.util.PublicDBLogging.withAttributionTags(DatabricksSparkUsageLogger.scala:30)

at com.databricks.logging.UsageLogging.recordOperationWithResultTags(UsageLogging.scala:611)

at com.databricks.logging.UsageLogging.recordOperationWithResultTags$(UsageLogging.scala:519)

at com.databricks.spark.util.PublicDBLogging.recordOperationWithResultTags(DatabricksSparkUsageLogger.scala:30)

at com.databricks.logging.UsageLogging.recordOperation(UsageLogging.scala:511)

at com.databricks.logging.UsageLogging.recordOperation$(UsageLogging.scala:475)

at com.databricks.spark.util.PublicDBLogging.recordOperation(DatabricksSparkUsageLogger.scala:30)

at com.databricks.spark.util.PublicDBLogging.recordOperation0(DatabricksSparkUsageLogger.scala:120)

at com.databricks.spark.util.DatabricksSparkUsageLogger.recordOperation(DatabricksSparkUsageLogger.scala:210)

at com.databricks.spark.util.UsageLogger.recordOperation(UsageLogger.scala:78)

at com.databricks.spark.util.UsageLogger.recordOperation$(UsageLogger.scala:65)

at com.databricks.spark.util.DatabricksSparkUsageLogger.recordOperation(DatabricksSparkUsageLogger.scala:169)

at com.databricks.spark.util.UsageLogging.recordOperation(UsageLogger.scala:537)

at com.databricks.spark.util.UsageLogging.recordOperation$(UsageLogger.scala:516)

at com.databricks.sql.transaction.tahoe.sources.DeltaSource.recordOperation(DeltaSource.scala:804)

at com.databricks.sql.transaction.tahoe.metering.DeltaLogging.recordDeltaOperationInternal(DeltaLogging.scala:176)

at com.databricks.sql.transaction.tahoe.metering.DeltaLogging.recordDeltaOperation(DeltaLogging.scala:166)

at com.databricks.sql.transaction.tahoe.metering.DeltaLogging.recordDeltaOperation$(DeltaLogging.scala:155)

at com.databricks.sql.transaction.tahoe.sources.DeltaSource.recordDeltaOperation(DeltaSource.scala:804)

at com.databricks.sql.transaction.tahoe.sources.DeltaSource.getBatch(DeltaSource.scala:1314)

at org.apache.spark.sql.execution.streaming.MicroBatchExecution.$anonfun$populateStartOffsets$4(MicroBatchExecution.scala:989)

at scala.collection.Iterator.foreach(Iterator.scala:943)

at scala.collection.Iterator.foreach$(Iterator.scala:943)

at scala.collection.AbstractIterator.foreach(Iterator.scala:1431)

at scala.collection.IterableLike.foreach(IterableLike.scala:74)

at scala.collection.IterableLike.foreach$(IterableLike.scala:73)

at org.apache.spark.sql.execution.streaming.StreamProgress.foreach(StreamProgress.scala:27)

at org.apache.spark.sql.execution.streaming.MicroBatchExecution.populateStartOffsets(MicroBatchExecution.scala:986)

at org.apache.spark.sql.execution.streaming.MultiBatchRollbackSupport.populateStartOffsetsWithRollbackHandling(MultiBatchRollbackSupport.scala:125)

at org.apache.spark.sql.execution.streaming.MultiBatchRollbackSupport.populateStartOffsetsWithRollbackHandling$(MultiBatchRollbackSupport.scala:85)

at org.apache.spark.sql.execution.streaming.MicroBatchExecution.populateStartOffsetsWithRollbackHandling(MicroBatchExecution.scala:81)

at org.apache.spark.sql.execution.streaming.MicroBatchExecution.initializeExecution(MicroBatchExecution.scala:559)

at org.apache.spark.sql.execution.streaming.MicroBatchExecution.runActivatedStreamWithListener(MicroBatchExecution.scala:698)

at org.apache.spark.sql.execution.streaming.MicroBatchExecution.runActivatedStream(MicroBatchExecution.scala:475)

at org.apache.spark.sql.execution.streaming.StreamExecution.$anonfun$runStream$2(StreamExecution.scala:450)

at scala.runtime.java8.JFunction0$mcV$sp.apply(JFunction0$mcV$sp.java:23)

at org.apache.spark.sql.SparkSession.withActive(SparkSession.scala:1462)

at org.apache.spark.sql.execution.streaming.StreamExecution.$anonfun$runStream$1(StreamExecution.scala:389)