INTERNAL_ERROR occurred while converting Iceberg format table to Delta format using Spark

hskimskydd — Thu, 12 Mar 2026 08:53:38 GMT

I used Apache Spark to write an iceberg table to Amazon S3.
I then ran the code below to convert the iceberg table to delta, and the following exception occurred:

```python
spark.sql('convert to delta iceberg.`s3a://BUCKET/path/to/table_name/`')
```

```text
pyspark.errors.exceptions.connect.SparkException: [INTERNAL_ERROR] Eagerly executed command failed. You encountered a bug in Spark or the Spark plugins you use. Please report this bug to the corresponding community or vendor and provide the full stack trace. SQLSTATE: XX000
```

I'm leaving this message because they told me to contact the community or vendor.

```text

>>> result = spark.sql('convert to delta iceberg.`s3a://BUCKET/path/to/iceberg_table/`')

Traceback (most recent call last):

File "<stdin>", line 1, in <module>

File "/usr/local/spark/spark-4.1.1-bin-hadoop3-connect/python/pyspark/sql/connect/session.py", line 828, in sql

data, properties, ei = self.client.execute_command(cmd.command(self._client))

^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^

File "/usr/local/spark/spark-4.1.1-bin-hadoop3-connect/python/pyspark/sql/connect/client/core.py", line 1195, in execute_command

data, _, metrics, observed_metrics, properties = self._execute_and_fetch(

^^^^^^^^^^^^^^^^^^^^^^^^

File "/usr/local/spark/spark-4.1.1-bin-hadoop3-connect/python/pyspark/sql/connect/client/core.py", line 1697, in _execute_and_fetch

for response in self._execute_and_fetch_as_iterator(

^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^

File "/usr/local/spark/spark-4.1.1-bin-hadoop3-connect/python/pyspark/sql/connect/client/core.py", line 1674, in _execute_and_fetch_as_iterator

self._handle_error(error)

File "/usr/local/spark/spark-4.1.1-bin-hadoop3-connect/python/pyspark/sql/connect/client/core.py", line 1982, in _handle_error

self._handle_rpc_error(error)

File "/usr/local/spark/spark-4.1.1-bin-hadoop3-connect/python/pyspark/sql/connect/client/core.py", line 2066, in _handle_rpc_error

raise convert_exception(

pyspark.errors.exceptions.connect.SparkException: [INTERNAL_ERROR] Eagerly executed command failed. You hit a bug in Spark or the Spark plugins you use. Please, report this bug to the corresponding communities or vendors, and provide the full stack trace. SQLSTATE: XX000

JVM stacktrace:

org.apache.spark.SparkException

at org.apache.spark.SparkException$.internalError(SparkException.scala:107)

at org.apache.spark.sql.execution.QueryExecution$.toInternalError(QueryExecution.scala:706)

at org.apache.spark.sql.execution.QueryExecution$.withInternalError(QueryExecution.scala:719)

at org.apache.spark.sql.execution.QueryExecution.org$apache$spark$sql$execution$QueryExecution$$eagerlyExecute$1(QueryExecution.scala:184)

at org.apache.spark.sql.execution.QueryExecution$$anonfun$eagerlyExecuteCommands$3.applyOrElse(QueryExecution.scala:201)

at org.apache.spark.sql.execution.QueryExecution$$anonfun$eagerlyExecuteCommands$3.applyOrElse(QueryExecution.scala:194)

at org.apache.spark.sql.catalyst.trees.TreeNode.$anonfun$transformDownWithPruning$1(TreeNode.scala:491)

at org.apache.spark.sql.catalyst.trees.CurrentOrigin$.withOrigin(origin.scala:107)

at org.apache.spark.sql.catalyst.trees.TreeNode.transformDownWithPruning(TreeNode.scala:491)

at org.apache.spark.sql.catalyst.plans.logical.LogicalPlan.org$apache$spark$sql$catalyst$plans$logical$AnalysisHelper$$super$transformDownWithPruning(LogicalPlan.scala:37)

at org.apache.spark.sql.catalyst.plans.logical.AnalysisHelper.transformDownWithPruning(AnalysisHelper.scala:360)

at org.apache.spark.sql.catalyst.plans.logical.AnalysisHelper.transformDownWithPruning$(AnalysisHelper.scala:356)

at org.apache.spark.sql.catalyst.plans.logical.LogicalPlan.transformDownWithPruning(LogicalPlan.scala:37)

at org.apache.spark.sql.catalyst.trees.TreeNode.transformDown(TreeNode.scala:467)

at org.apache.spark.sql.execution.QueryExecution.eagerlyExecuteCommands(QueryExecution.scala:194)

at org.apache.spark.sql.execution.QueryExecution.$anonfun$lazyCommandExecuted$1(QueryExecution.scala:155)

at scala.util.Try$.apply(Try.scala:217)

at org.apache.spark.util.Utils$.doTryWithCallerStacktrace(Utils.scala:1392)

at org.apache.spark.util.Utils$.getTryWithCallerStacktrace(Utils.scala:1453)

at org.apache.spark.util.LazyTry.get(LazyTry.scala:58)

at org.apache.spark.sql.execution.QueryExecution.commandExecuted(QueryExecution.scala:160)

at org.apache.spark.sql.classic.Dataset.<init>(Dataset.scala:276)

at org.apache.spark.sql.classic.Dataset$.$anonfun$ofRows$5(Dataset.scala:139)

at org.apache.spark.sql.SparkSession.withActive(SparkSession.scala:804)

at org.apache.spark.sql.classic.Dataset$.ofRows(Dataset.scala:135)

at org.apache.spark.sql.classic.SparkSession.$anonfun$sql$4(SparkSession.scala:584)

at org.apache.spark.sql.SparkSession.withActive(SparkSession.scala:804)

at org.apache.spark.sql.classic.SparkSession.sql(SparkSession.scala:561)

at org.apache.spark.sql.connect.planner.SparkConnectPlanner.executeSQL(SparkConnectPlanner.scala:3148)

at org.apache.spark.sql.connect.planner.SparkConnectPlanner.handleSqlCommand(SparkConnectPlanner.scala:2996)

at org.apache.spark.sql.connect.planner.SparkConnectPlanner.process(SparkConnectPlanner.scala:2830)

at org.apache.spark.sql.connect.execution.SparkConnectPlanExecution.handlePlan(SparkConnectPlanExecution.scala:96)

at org.apache.spark.sql.connect.execution.ExecuteThreadRunner.$anonfun$executeInternal$1(ExecuteThreadRunner.scala:225)

at org.apache.spark.sql.connect.execution.ExecuteThreadRunner.$anonfun$executeInternal$1$adapted(ExecuteThreadRunner.scala:197)

at org.apache.spark.sql.connect.service.SessionHolder.$anonfun$withSession$2(SessionHolder.scala:396)

at org.apache.spark.sql.SparkSession.withActive(SparkSession.scala:804)

at org.apache.spark.sql.connect.service.SessionHolder.$anonfun$withSession$1(SessionHolder.scala:396)

at org.apache.spark.JobArtifactSet$.withActiveJobArtifactState(JobArtifactSet.scala:94)

at org.apache.spark.sql.artifact.ArtifactManager.$anonfun$withResources$1(ArtifactManager.scala:112)

at org.apache.spark.util.Utils$.withContextClassLoader(Utils.scala:185)

at org.apache.spark.sql.artifact.ArtifactManager.withClassLoaderIfNeeded(ArtifactManager.scala:102)

at org.apache.spark.sql.artifact.ArtifactManager.withResources(ArtifactManager.scala:111)

at org.apache.spark.sql.connect.service.SessionHolder.withSession(SessionHolder.scala:395)

at org.apache.spark.sql.connect.execution.ExecuteThreadRunner.executeInternal(ExecuteThreadRunner.scala:197)

at org.apache.spark.sql.connect.execution.ExecuteThreadRunner.org$apache$spark$sql$connect$execution$ExecuteThreadRunner$$execute(ExecuteThreadRunner.scala:126)

at org.apache.spark.sql.connect.execution.ExecuteThreadRunner$ExecutionThread.run(ExecuteThreadRunner.scala:334)

Caused by: java.lang.NullPointerException: Cannot invoke "org.apache.spark.sql.connector.catalog.CatalogPlugin.name()" because "this.delegate" is null

at org.apache.spark.sql.connector.catalog.DelegatingCatalogExtension.name(DelegatingCatalogExtension.java:50)

at org.apache.spark.sql.catalyst.analysis.RelationResolution.toCacheKey(RelationResolution.scala:310)

at org.apache.spark.sql.catalyst.analysis.RelationResolution.$anonfun$resolveRelation$2(RelationResolution.scala:119)

at scala.Option.orElse(Option.scala:477)

at org.apache.spark.sql.catalyst.analysis.RelationResolution.resolveRelation(RelationResolution.scala:117)

at org.apache.spark.sql.catalyst.analysis.Analyzer$ResolveRelations$.resolveRelation(Analyzer.scala:1323)

at org.apache.spark.sql.catalyst.analysis.Analyzer$ResolveRelations$$anonfun$apply$12.applyOrElse(Analyzer.scala:1244)

at org.apache.spark.sql.catalyst.analysis.Analyzer$ResolveRelations$$anonfun$apply$12.applyOrElse(Analyzer.scala:1205)

at org.apache.spark.sql.catalyst.plans.logical.AnalysisHelper.$anonfun$resolveOperatorsUpWithPruning$3(AnalysisHelper.scala:139)

at org.apache.spark.sql.catalyst.trees.CurrentOrigin$.withOrigin(origin.scala:107)

at org.apache.spark.sql.catalyst.plans.logical.AnalysisHelper.$anonfun$resolveOperatorsUpWithPruning$1(AnalysisHelper.scala:139)

at org.apache.spark.sql.catalyst.plans.logical.AnalysisHelper$.allowInvokingTransformsInAnalyzer(AnalysisHelper.scala:416)

at org.apache.spark.sql.catalyst.plans.logical.AnalysisHelper.resolveOperatorsUpWithPruning(AnalysisHelper.scala:135)

at org.apache.spark.sql.catalyst.plans.logical.AnalysisHelper.resolveOperatorsUpWithPruning$(AnalysisHelper.scala:131)

at org.apache.spark.sql.catalyst.plans.logical.LogicalPlan.resolveOperatorsUpWithPruning(LogicalPlan.scala:37)

at org.apache.spark.sql.catalyst.analysis.Analyzer$ResolveRelations$.apply(Analyzer.scala:1205)

at org.apache.spark.sql.catalyst.analysis.Analyzer$ResolveRelations$.apply(Analyzer.scala:1167)

at org.apache.spark.sql.catalyst.rules.RuleExecutor.$anonfun$execute$2(RuleExecutor.scala:248)

at scala.collection.LinearSeqOps.foldLeft(LinearSeq.scala:183)

at scala.collection.LinearSeqOps.foldLeft$(LinearSeq.scala:179)

at scala.collection.immutable.List.foldLeft(List.scala:79)

at org.apache.spark.sql.catalyst.rules.RuleExecutor.$anonfun$execute$1(RuleExecutor.scala:245)

at org.apache.spark.sql.catalyst.rules.RuleExecutor.$anonfun$execute$1$adapted(RuleExecutor.scala:237)

at scala.collection.immutable.List.foreach(List.scala:323)

at org.apache.spark.sql.catalyst.rules.RuleExecutor.execute(RuleExecutor.scala:237)

at org.apache.spark.sql.catalyst.analysis.Analyzer.org$apache$spark$sql$catalyst$analysis$Analyzer$$executeSameContext(Analyzer.scala:343)

at org.apache.spark.sql.catalyst.analysis.Analyzer.$anonfun$execute$1(Analyzer.scala:339)

at org.apache.spark.sql.catalyst.analysis.AnalysisContext$.withNewAnalysisContext(Analyzer.scala:224)

at org.apache.spark.sql.catalyst.analysis.Analyzer.execute(Analyzer.scala:339)

at org.apache.spark.sql.delta.commands.DeltaCommand.resolveIdentifier(DeltaCommand.scala:187)

at org.apache.spark.sql.delta.commands.DeltaCommand.resolveIdentifier$(DeltaCommand.scala:186)

at org.apache.spark.sql.delta.commands.ConvertToDeltaCommandBase.resolveIdentifier(ConvertToDeltaCommand.scala:70)

at org.apache.spark.sql.delta.commands.DeltaCommand.isCatalogTable(DeltaCommand.scala:199)

at org.apache.spark.sql.delta.commands.DeltaCommand.isCatalogTable$(DeltaCommand.scala:197)

at org.apache.spark.sql.delta.commands.ConvertToDeltaCommandBase.isCatalogTable(ConvertToDeltaCommand.scala:199)

at org.apache.spark.sql.delta.commands.ConvertToDeltaCommandBase.resolveConvertTarget(ConvertToDeltaCommand.scala:125)

at org.apache.spark.sql.delta.commands.ConvertToDeltaCommandBase.run(ConvertToDeltaCommand.scala:97)

at org.apache.spark.sql.execution.command.ExecutedCommandExec.sideEffectResult$lzycompute(commands.scala:79)

at org.apache.spark.sql.execution.command.ExecutedCommandExec.sideEffectResult(commands.scala:77)

at org.apache.spark.sql.execution.command.ExecutedCommandExec.executeCollect(commands.scala:88)

at org.apache.spark.sql.execution.QueryExecution.$anonfun$eagerlyExecuteCommands$2(QueryExecution.scala:185)

at org.apache.spark.sql.execution.SQLExecution$.$anonfun$withNewExecutionId0$8(SQLExecution.scala:177)

at org.apache.spark.sql.execution.SQLExecution$.withSessionTagsApplied(SQLExecution.scala:285)

at org.apache.spark.sql.execution.SQLExecution$.$anonfun$withNewExecutionId0$7(SQLExecution.scala:139)

at org.apache.spark.JobArtifactSet$.withActiveJobArtifactState(JobArtifactSet.scala:94)

at org.apache.spark.sql.artifact.ArtifactManager.$anonfun$withResources$1(ArtifactManager.scala:112)

at org.apache.spark.util.Utils$.withContextClassLoader(Utils.scala:185)

at org.apache.spark.sql.artifact.ArtifactManager.withClassLoaderIfNeeded(ArtifactManager.scala:102)

at org.apache.spark.sql.artifact.ArtifactManager.withResources(ArtifactManager.scala:111)

at org.apache.spark.sql.execution.SQLExecution$.$anonfun$withNewExecutionId0$6(SQLExecution.scala:139)

at org.apache.spark.sql.execution.SQLExecution$.withSQLConfPropagated(SQLExecution.scala:308)

at org.apache.spark.sql.execution.SQLExecution$.$anonfun$withNewExecutionId0$1(SQLExecution.scala:138)

at org.apache.spark.sql.SparkSession.withActive(SparkSession.scala:804)

at org.apache.spark.sql.execution.SQLExecution$.withNewExecutionId0(SQLExecution.scala:92)

at org.apache.spark.sql.execution.SQLExecution$.withNewExecutionId(SQLExecution.scala:250)

at org.apache.spark.sql.execution.QueryExecution.$anonfun$eagerlyExecuteCommands$1(QueryExecution.scala:185)

at org.apache.spark.sql.execution.QueryExecution$.withInternalError(QueryExecution.scala:717)