Databricks Community

slloyd · ‎08-13-2024

I am using the Databricks SQL Driver for Node.js to create an endpoint that queries a Databricks database following the guide here Databricks SQL Driver for Node.js | Databricks on AWS . This code was working previously but now I am getting a TypeError: Cannot read properties of undefined (reading '0') when executing

session = await client.openSession(); I have verified my environment variables are correct.
api/databricks/route.ts

export async function POST(req: NextRequest) {  
  console.log("initialized");  
  const body = await req.json();  
  const { query } = body;  
  console.log("Received Query:", query);  
  
  if (token == "" || serverHostname == "" || httpPath == "") {  
    return NextResponse.json(  
      {  
        error:  
          "Cannot find Server Hostname, HTTP Path, or personal access token. " +  
          "Check the environment variables DATABRICKS_SERVER_HOSTNAME, " +  
          "DATABRICKS_HTTP_PATH, and DATABRICKS_TOKEN.",  
      },  
      { status: 500 }  
    );  
  }  
  
  const client = new DBSQLClient();  
  const connectOptions = {  
    host: serverHostname,  
    token: token,
    path: httpPath,
  };  
  
  console.log("Connect Options:", connectOptions);  
  
  const timeout = 30000; // Timeout in milliseconds (30 seconds)  
  try {  
    await client.connect(connectOptions);  
    console.log("Connected to Databricks");

    let session: IDBSQLSession;
    try {  
      session = await client.openSession();  
      console.log("Session opened");  
    } catch (openSessionError) {  
      console.error("Error opening session:", openSessionError);  
      throw openSessionError; // Rethrow to be caught by the outer catch  
    } 
  
    const statements = splitSQLStatements(query);  
    let result: any[] = [];  
  
    for (const statement of statements) {  
      console.log("Executing Statement:", statement);  
      const queryOperationPromise: IOperation = await session.executeStatement(  
        statement,  
        {  
          runAsync: true,  
          maxRows: 500, // This option enables the direct results feature.  
        }  
      );  
  
      let queryOperation: IOperation;  
      try {  
        queryOperation = (await Promise.race([  
          queryOperationPromise,  
          createTimeoutPromise(timeout),  
        ])) as IOperation;  
      } catch (error) {  
        if (error instanceof Error && error.message.includes("timed out")) {  
          console.error("Error: Query execution timed out.");  
          return NextResponse.json(  
            { error: "Query execution timed out." },  
            { status: 500 }  
          );  
        } else {  
          throw error;  
        }  
      }  
  
      const statementResult = await queryOperation.fetchAll();  
      console.log(  
        "Result for Statement:",  
        util.inspect(statementResult, { depth: null })  
      );  
  
      // Check if the result is not empty and merge the results  
      if (statementResult.length > 0) {  
        result = result.concat(statementResult);  
      } else {  
        console.warn(`No data returned for statement: ${statement}`);  
      }  
      await queryOperation.close();  
    }  
  
    await session.close();  
    client.close();  
    console.log("Final Result:", util.inspect(result, { depth: null }));  
  
    // Ensure the result is serializable  
    return NextResponse.json(result);  
  } catch (error) {  
    if (error instanceof Error) {  
      console.error("Error executing query: ", error);  
      return NextResponse.json(  
        { error: "Failed to execute query: ", details: error.message },  
        { status: 500 }  
      );  
    }  
  }  
}

console:

POST /api/chat 200 in 13099ms
initialized
Received Query: use catalog iedatalakeprd;
use schema ds_cmdb;

SELECT
    win.Name AS Server_Name,
    win.OperationalStatus AS Operational_Status,
    loc.Name AS Location_Name,
    dep.Name AS Department_Name
FROM
    silver_cmdbciwinserver AS win
LEFT JOIN
    silver_cmnlocation AS loc ON win.Location_Id = loc.Id
LEFT JOIN
    silver_cmndepartment AS dep ON win.ElmoDepartment = dep.Id;
{"level":"info","message":"Created DBSQLClient"}
Connect Options: {
  host: 'adb-6894947688059139.19.azuredatabricks.net',
  token: 'dapid12947cad444f67eb7d44ff57f4148b3',
  path: '/sql/1.0/warehouses/e3042c2a5aa08bc9'
}
Connected to Databricks
Error opening session: TypeError: Cannot read properties of undefined (reading '0')
    at isInsideNodeModules (node:internal/util:508:17)
    at showFlaggedDeprecation (node:buffer:178:8)
    at new Buffer (node:buffer:266:3)
    at new module.exports (webpack-internal:///(rsc)/./node_modules/node-int64/Int64.js:65:34)
    at DBSQLClient.eval (webpack-internal:///(rsc)/./node_modules/@databricks/sql/dist/DBSQLClient.js:197:111)
    at Generator.next (<anonymous>)
    at eval (webpack-internal:///(rsc)/./node_modules/@databricks/sql/dist/DBSQLClient.js:31:71)
    at new Promise (<anonymous>)
    at __awaiter (webpack-internal:///(rsc)/./node_modules/@databricks/sql/dist/DBSQLClient.js:27:12)
    at DBSQLClient.openSession (webpack-internal:///(rsc)/./node_modules/@databricks/sql/dist/DBSQLClient.js:196:16)
    at POST (webpack-internal:///(rsc)/./app/api/databricks/route.ts:73:36)
    at process.processTicksAndRejections (node:internal/process/task_queues:95:5)
    at async /home/slloyd/database_ai_assistant/node_modules/next/dist/compiled/next-server/app-route.runtime.dev.js:6:53446
    at async e_.execute (/home/slloyd/database_ai_assistant/node_modules/next/dist/compiled/next-server/app-route.runtime.dev.js:6:44747)
    at async e_.handle (/home/slloyd/database_ai_assistant/node_modules/next/dist/compiled/next-server/app-route.runtime.dev.js:6:54700)
    at async doRender (/home/slloyd/database_ai_assistant/node_modules/next/dist/server/base-server.js:1377:42)
    at async cacheEntry.responseCache.get.routeKind (/home/slloyd/database_ai_assistant/node_modules/next/dist/server/base-server.js:1599:28)
    at async DevServer.renderToResponseWithComponentsImpl (/home/slloyd/database_ai_assistant/node_modules/next/dist/server/base-server.js:1507:28)
    at async DevServer.renderPageComponent (/home/slloyd/database_ai_assistant/node_modules/next/dist/server/base-server.js:1924:24)
    at async DevServer.renderToResponseImpl (/home/slloyd/database_ai_assistant/node_modules/next/dist/server/base-server.js:1962:32)
    at async DevServer.pipeImpl (/home/slloyd/database_ai_assistant/node_modules/next/dist/server/base-server.js:920:25)
    at async NextNodeServer.handleCatchallRenderRequest (/home/slloyd/database_ai_assistant/node_modules/next/dist/server/next-server.js:272:17)
    at async DevServer.handleRequestImpl (/home/slloyd/database_ai_assistant/node_modules/next/dist/server/base-server.js:816:17)
    at async /home/slloyd/database_ai_assistant/node_modules/next/dist/server/dev/next-dev-server.js:339:20
    at async Span.traceAsyncFn (/home/slloyd/database_ai_assistant/node_modules/next/dist/trace/trace.js:154:20)
    at async DevServer.handleRequest (/home/slloyd/database_ai_assistant/node_modules/next/dist/server/dev/next-dev-server.js:336:24)
    at async invokeRender (/home/slloyd/database_ai_assistant/node_modules/next/dist/server/lib/router-server.js:174:21)
    at async handleRequest (/home/slloyd/database_ai_assistant/node_modules/next/dist/server/lib/router-server.js:353:24)
    at async requestHandlerImpl (/home/slloyd/database_ai_assistant/node_modules/next/dist/server/lib/router-server.js:377:13)
    at async Server.requestListener (/home/slloyd/database_ai_assistant/node_modules/next/dist/server/lib/start-server.js:141:13)
Error executing query:  TypeError: Cannot read properties of undefined (reading '0')
    at isInsideNodeModules (node:internal/util:508:17)
    at showFlaggedDeprecation (node:buffer:178:8)
    at new Buffer (node:buffer:266:3)
    at new module.exports (webpack-internal:///(rsc)/./node_modules/node-int64/Int64.js:65:34)
    at DBSQLClient.eval (webpack-internal:///(rsc)/./node_modules/@databricks/sql/dist/DBSQLClient.js:197:111)
    at Generator.next (<anonymous>)
    at eval (webpack-internal:///(rsc)/./node_modules/@databricks/sql/dist/DBSQLClient.js:31:71)
    at new Promise (<anonymous>)
    at __awaiter (webpack-internal:///(rsc)/./node_modules/@databricks/sql/dist/DBSQLClient.js:27:12)
    at DBSQLClient.openSession (webpack-internal:///(rsc)/./node_modules/@databricks/sql/dist/DBSQLClient.js:196:16)
    at POST (webpack-internal:///(rsc)/./app/api/databricks/route.ts:73:36)
    at process.processTicksAndRejections (node:internal/process/task_queues:95:5)
    at async /home/slloyd/database_ai_assistant/node_modules/next/dist/compiled/next-server/app-route.runtime.dev.js:6:53446
    at async e_.execute (/home/slloyd/database_ai_assistant/node_modules/next/dist/compiled/next-server/app-route.runtime.dev.js:6:44747)
    at async e_.handle (/home/slloyd/database_ai_assistant/node_modules/next/dist/compiled/next-server/app-route.runtime.dev.js:6:54700)
    at async doRender (/home/slloyd/database_ai_assistant/node_modules/next/dist/server/base-server.js:1377:42)
    at async cacheEntry.responseCache.get.routeKind (/home/slloyd/database_ai_assistant/node_modules/next/dist/server/base-server.js:1599:28)
    at async DevServer.renderToResponseWithComponentsImpl (/home/slloyd/database_ai_assistant/node_modules/next/dist/server/base-server.js:1507:28)
    at async DevServer.renderPageComponent (/home/slloyd/database_ai_assistant/node_modules/next/dist/server/base-server.js:1924:24)
    at async DevServer.renderToResponseImpl (/home/slloyd/database_ai_assistant/node_modules/next/dist/server/base-server.js:1962:32)
    at async DevServer.pipeImpl (/home/slloyd/database_ai_assistant/node_modules/next/dist/server/base-server.js:920:25)
    at async NextNodeServer.handleCatchallRenderRequest (/home/slloyd/database_ai_assistant/node_modules/next/dist/server/next-server.js:272:17)
    at async DevServer.handleRequestImpl (/home/slloyd/database_ai_assistant/node_modules/next/dist/server/base-server.js:816:17)
    at async /home/slloyd/database_ai_assistant/node_modules/next/dist/server/dev/next-dev-server.js:339:20
    at async Span.traceAsyncFn (/home/slloyd/database_ai_assistant/node_modules/next/dist/trace/trace.js:154:20)
    at async DevServer.handleRequest (/home/slloyd/database_ai_assistant/node_modules/next/dist/server/dev/next-dev-server.js:336:24)
    at async invokeRender (/home/slloyd/database_ai_assistant/node_modules/next/dist/server/lib/router-server.js:174:21)
    at async handleRequest (/home/slloyd/database_ai_assistant/node_modules/next/dist/server/lib/router-server.js:353:24)
    at async requestHandlerImpl (/home/slloyd/database_ai_assistant/node_modules/next/dist/server/lib/router-server.js:377:13)
    at async Server.requestListener (/home/slloyd/database_ai_assistant/node_modules/next/dist/server/lib/start-server.js:141:13)
 POST /api/databricks 500 in 189ms

mark_ott · Wednesday

Your TypeError: Cannot read properties of undefined (reading '0') at session = await client.openSession() typically indicates an unexpected change or regression inside the Databricks SQL Node.js driver or the environment, even if your environment variables appear correct.

Key Points From Your Output and Latest Docs

You have valid-looking connect options.
You connect, and get "Connected to Databricks" before the session fails.
Error is thrown inside the openSession method, in the DBSQLClient.js at a point that calls Buffer or Int64.

Root Causes Based on Databricks SQL Driver Docs & Your Stack Trace

1. Breaking Change: Node.js Buffer Deprecation and Environment

The error is tripped inside Buffer and/or related module code (new Buffer(...)), which in newer Node.js has been deprecated and may behave unexpectedly, especially in environments like Vercel, Next.js, or serverless contexts.
Recent updates to Node.js or the @DataBricks/sql driver can combine badly if you rely on a deprecated or missing Buffer API.

2. Driver Dependency or Upgrade Issue

Check for a recent upgrade or dependency change in either Node.js itself or @DataBricks/sql or its dependencies, especially if it was working until recently.
Node 20+ enforces stricter module boundaries and buffer APIs.

3. Databricks Driver Regression or Input Bug

The error comes from inside the driver and/or one of its dependencies (likely node-int64 package).
If the driver receives bad or incomplete data from the Databricks backend, or the backend response format changed, it could result in undefined values feeding into a Buffer constructor.

4. Workspace Permissions, Path, or Compatibility

If Databricks updated their API, broke backward compatibility, or now enforces stricter checks on openSession, a formerly-working call may now fail without a clear error message. (Check if you have any workspace, warehouse, or path changes recently.)

Steps to Diagnose and Work Around the Error

A. Update your code for Node.js Buffer Compatibility

If you are using a Node.js version >= 14, prefer Buffer.from, not new Buffer:

This is a driver-internal issue, but check your code and dependencies for new Buffer and prefer Buffer.from().
If possible, downgrade Node.js to 18.x LTS (if you're on 20+ only for experimentation) to check for version-specific issues.
Try pinning your @DataBricks/sql to a version that last worked, or updating to the latest available release if you are not on it.

B. Clean and Validate Node Modules

Run npm ls @DataBricks/sql and npm ls node-int64 to check for duplicates or mismatches.
Run npm ci or rm -rf node_modules package-lock.json && npm install to clear out stale/broken dependencies.

C. Validate Databricks Workspace and Warehouse

Ensure the httpPath and warehouse/cluster ID are valid.
Test your credentials and SQL warehouse by connecting via Databricks SQL CLI or any other client to rule out backend changes.

D. Add Detailed Logging of the Connection and Session Response

Right before and after await client.openSession(), log the client and connection objects in detail. Sometimes "undefined" errors are due to non-array or malformed response objects—knowing what the driver is working with helps.

E. Test with a Minimal Standalone Script

Isolate your connection/session code:

js

const { DBSQLClient } = require('@databricks/sql');
const client = new DBSQLClient();
const connectOptions = { host: ..., token: ..., path: ... };

client.connect(connectOptions).then(async (cli) => {
  try {
    const session = await cli.openSession();
    console.log("Session opened", session);
    await session.close();
  } catch (e) {
    console.error("Failed in session:", e);
  } finally {
    await cli.close();
  }
});

If this fails in the same way, it is certainly not the Next.js/route layer.

F. Check with the Latest Driver and Backport if Needed

Query: did you recently update @DataBricks/sql? Try both the latest and the last known working version.

If All Else Fails: Downgrade or Raise an Issue

Try downgrading @DataBricks/sql to the specific version before the error started, especially if there's no environmental change on your side.
If the issue persists, file a bug with Databricks, including your stack trace and package versions: it may be a breaking change on their side or in an upstream dependency (node-int64).
Consult the Databricks SQL Driver for Node.js repository for new issues and see if others report the bug.

Summary Table

Possible Issue	Steps/Checks
Node.js version/buffer breaking change	Use Node.js 18.x LTS, check for Buffer API
Databricks driver regression/dependency mismatch	Pin/downgrade/upgrade driver, clean modules
Bad env vars/warehouse/path	Validate all credentials & path/workspace
Backend contract change/permissions	Test with Databricks SQL CLI or DBeaver
Application-level regression	Isolate to pure Node.js code, test minimal
Known issues on GitHub or Databricks forums	Check and monitor for open bug reports

Summary:
This error most commonly reflects either a Node.js version/Buffer API mismatch, a regression in the Databricks Node.js SQL driver (or a dependency), or a backend response change. Try downgrading Node.js, pinning/rolling back @DataBricks/sql, cleaning dependencies, and validating with a minimal script. Add maximum logging around openSession and client states. Check Databricks forums for breaking driver issues.