I'm developing an API in Flask that interacts with Databricks to upload files to a Unity Catalog volume, but I'm encountering the following error:
From my API in Flask, I am trying to upload a file using the Databricks API. Below I am sharing the code for app.py, where I first store the file temporarily and then upload it to the volume in Unity Catalog
import io
import os
import base64
import requests
from flask import Flask, request, render_template, jsonify
from databricks.sdk import WorkspaceClient
# Remove environment variables that may cause conflicts
os.environ.pop('DATABRICKS_HOST', None)
os.environ.pop('DATABRICKS_CLIENT_ID', None)
os.environ.pop('DATABRICKS_CLIENT_SECRET', None)
# Configure WorkspaceClient with explicit authentication using only the token
w = WorkspaceClient(
token="my_token"
)
# Inicializar Flask
app = Flask(__name__)
def index():
return render_template('index.html')
@app.route('/upload', methods=['POST']) def upload_file():
"""Handles uploading files to a volume in Unity Catalog."""
if 'file' not in request.files:
return "Error: No file selected", 400
file = request.files['file']
if file.filename == '':
return "Error: No file selected", 400
try:
# Save the file temporarily to the local file system
temp_file_path = f"/tmp/{file.filename}"
file.save(temp_file_path)
# Read the file and encode it in base64
with open(temp_file_path, 'rb') as f:
file_data = f.read()
file_data_base64 = base64.b64encode(file_data).decode('utf-8')
# Upload the file to the volume in Unity Catalog
response = requests.post(
f"{w.config.host}/api/2.0/unity-catalog/volumes/upload",
headers={"Authorization": f"Bearer {w.config.token}"},
json={
"catalog_name": "your_catalog_name",
"schema_name": "your_schema_name",
"volume_name": "your_volume_name",
"file_name": file.filename,
"file_contents": file_data_base64
}
)
if response.status_code != 200:
raise Exception(f"Error uploading file: {response.text}")
return f"file'{file.filename}' successfully uploaded to Unity Catalog"
except Exception as e:
return f"Error uploading file: {e}", 500
def list_files():
try:
response = requests.get(
f"{w.config.host}/api/2.0/unity-catalog/volumes/list",
headers={"Authorization": f"Bearer {w.config.token}"},
params={
"catalog_name": "your_catalog_name",
"schema_name": "your_schema_name",
"volume_name": "your_volume_name"
}
)
if response.status_code != 200:
raise Exception(f"Error listing files: {response.text}")
file_list = response.json().get("files", [])
return jsonify({"files": file_list})
except Exception as e:
return f"Error listing files: {e}", 500
if __name__ == '__main__':
app.run(host='0.0.0.0', port=5000, debug=True)
The error is giving me when uploading the file to the volume. I am doing it through the unity catalog since DBFS mounts are an obsolete pattern and are not recommended by Databricks.
- Is this the correct way to upload files to a Unity Catalog volume from a Flask API?
- Is a different endpoint in the Databricks API required to interact with volumes?
- Has anyone faced a similar issue?
Any help is welcome! Thanks in advance.
