<?xml version="1.0" encoding="UTF-8"?>
<rss xmlns:content="http://purl.org/rss/1.0/modules/content/" xmlns:dc="http://purl.org/dc/elements/1.1/" xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#" xmlns:taxo="http://purl.org/rss/1.0/modules/taxonomy/" version="2.0">
  <channel>
    <title>topic Auto Loader: addNewColumns can still stop your stream in Data Engineering</title>
    <link>https://community.databricks.com/t5/data-engineering/auto-loader-addnewcolumns-can-still-stop-your-stream/m-p/176758#M57199</link>
    <description>&lt;P&gt;A useful detail in the Auto Loader docs: addNewColumns does not mean the stream keeps running when a new column arrives.&lt;/P&gt;&lt;P&gt;Auto Loader updates its saved schema, then stops with UnknownFieldException. Restarting the stream picks up the new schema. Databricks recommends using Lakeflow Jobs to restart automatically.&lt;/P&gt;&lt;P&gt;For production, I’d also monitor _rescued_data and check that the target Delta table allows the intended schema change. Reader schema evolution and target table evolution need to line up.&lt;/P&gt;&lt;P&gt;Docs: &lt;A href="https://docs.databricks.com/aws/en/ingestion/cloud-object-storage/auto-loader/schema" target="_blank"&gt;https://docs.databricks.com/aws/en/ingestion/cloud-object-storage/auto-loader/schema&lt;/A&gt;&lt;BR /&gt;&lt;A href="https://docs.databricks.com/aws/en/data-engineering/schema-evolution" target="_blank"&gt;https://docs.databricks.com/aws/en/data-engineering/schema-evolution&lt;/A&gt;&lt;/P&gt;</description>
    <pubDate>Wed, 18 Nov 2026 12:21:08 GMT</pubDate>
    <dc:creator>karanvichare</dc:creator>
    <dc:date>2026-11-18T12:21:08Z</dc:date>
    <item>
      <title>Auto Loader: addNewColumns can still stop your stream</title>
      <link>https://community.databricks.com/t5/data-engineering/auto-loader-addnewcolumns-can-still-stop-your-stream/m-p/176758#M57199</link>
      <description>&lt;P&gt;A useful detail in the Auto Loader docs: addNewColumns does not mean the stream keeps running when a new column arrives.&lt;/P&gt;&lt;P&gt;Auto Loader updates its saved schema, then stops with UnknownFieldException. Restarting the stream picks up the new schema. Databricks recommends using Lakeflow Jobs to restart automatically.&lt;/P&gt;&lt;P&gt;For production, I’d also monitor _rescued_data and check that the target Delta table allows the intended schema change. Reader schema evolution and target table evolution need to line up.&lt;/P&gt;&lt;P&gt;Docs: &lt;A href="https://docs.databricks.com/aws/en/ingestion/cloud-object-storage/auto-loader/schema" target="_blank"&gt;https://docs.databricks.com/aws/en/ingestion/cloud-object-storage/auto-loader/schema&lt;/A&gt;&lt;BR /&gt;&lt;A href="https://docs.databricks.com/aws/en/data-engineering/schema-evolution" target="_blank"&gt;https://docs.databricks.com/aws/en/data-engineering/schema-evolution&lt;/A&gt;&lt;/P&gt;</description>
      <pubDate>Wed, 18 Nov 2026 12:21:08 GMT</pubDate>
      <guid>https://community.databricks.com/t5/data-engineering/auto-loader-addnewcolumns-can-still-stop-your-stream/m-p/176758#M57199</guid>
      <dc:creator>karanvichare</dc:creator>
      <dc:date>2026-11-18T12:21:08Z</dc:date>
    </item>
  </channel>
</rss>

