Hi, I am working as azure support engineer

I found this error while I am checking the pipeline failure, and showing below error

"org.apache.spark.SparkException: Job aborted due to stage failure: Task 0 in stage 72403.0 failed 4 times, most recent failure: Lost task 0.3 in stage 72403.0 (TID 801507, 10.139.64.5, executor 169): org.apache.spark.memory.SparkOutOfMemoryError: Unable to acquire 65536 bytes of memory, got 0"

Py4JJavaError Traceback (most recent call last)

<command-2313153849666105> in create_destination(location)

154 try:

--> 155 sql_df = spark.sql(sql_query)

156 break

/databricks/spark/python/pyspark/sql/session.py in sql(self, sqlQuery)

708 """

--> 709 return DataFrame(self._jsparkSession.sql(sqlQuery), self._wrapped)

710

/databricks/spark/python/lib/py4j-0.10.9-src.zip/py4j/java_gateway.py in __call__(self, *args)

1304 return_value = get_return_value(

-> 1305 answer, self.gateway_client, self.target_id, self.name)

1306

org.apache.spark.memory.TaskMemoryManager.allocatePage(TaskMemoryManager.java:289)

at org.apache.spark.memory.MemoryConsumer.allocatePage(MemoryConsumer.java:116)

at org.apache.spark.util.collection.unsafe.sort.UnsafeExternalSorter.acquireNewPageIfNecessary(UnsafeExternalSorter.java:419)

at org.apache.spark.util.collection.unsafe.sort.UnsafeExternalSorter.insertRecord(UnsafeExternalSorter.java:443)

at org.apache.spark.sql.execution.UnsafeExternalRowSorter.insertRow(UnsafeExternalRowSorter.java:138)

at org.apache.spark.sql.execution.UnsafeExternalRowSorter.sort(UnsafeExternalRowSorter.java:241)

at org.apache.spark.sql.execution.SortExec$$anon$2.sortedIterator(SortExec.scala:133)

at org.apache.spark.sql.execution.SortExec$$anon$2.hasNext(SortExec.scala:147)

at org.apache.spark.sql.execution.window.WindowExec$$anon$1.fetchNextRow(WindowExec.scala:185)

at org.apache.spark.sql.execution.window.WindowExec$$anon$1.<init>(WindowExec.scala:194)

at org.apache.spark.sql.execution.window.WindowExec.$anonfun$doExecute$3(WindowExec.scala:168)

at org.apache.spark.sql.execution.window.WindowExec.$anonfun$doExecute$3$adapted(WindowExec.scala:167)

at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsWithIndexInternal$2(RDD.scala:866)

at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsWithIndexInternal$2$adapted(RDD.scala:866)

at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:60)

at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:356)

at org.apache.spark.rdd.RDD.iterator(RDD.scala:320)

at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:60)

at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:356)

at org.apache.spark.rdd.RDD.iterator(RDD.scala:320)

at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:60)

at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:356)

at org.apache.spark.rdd.RDD.$anonfun$getOrCompute$1(RDD.scala:369)

at org.apache.spark.storage.BlockManager.$anonfun$doPutIterator$6(BlockManager.scala:1414)

at org.apache.spark.storage.BlockManager.$anonfun$doPutIterator$6$adapted(BlockManager.scala:1412)

at org.apache.spark.storage.DiskStore.put(DiskStore.scala:70)

at org.apache.spark.storage.BlockManager.$anonfun$doPutIterator$1(BlockManager.scala:1412)