Options
- Mark as New
- Bookmark
- Subscribe
- Mute
- Subscribe to RSS Feed
- Permalink
- Report Inappropriate Content
10-17-2024 11:54 AM
You can do something like this using databricks-sdk:
Here's the saved query
Pyspark code:
import re
from pyspark.sql import DataFrame
from databricks.sdk import WorkspaceClient
def run_saved_query(query_id:str, **params) -> DataFrame:
query_body, args = get_query_by_id(query_id)
query = replace_double_curly(query_body)
df = spark.sql(query, **params)
return df
def get_query_by_id(query_id):
w = WorkspaceClient()
q = w.queries.get(query_id)
query_body = q.query
query_params = [p.name for p in q.options.parameters]
return query_body, query_params
def replace_double_curly(text):
# Adjust regex to match the outermost double curly braces without capturing nested ones
def strip_outer_whitespace(match):
inner_text = match.group(1)
return f'{{{inner_text.strip()}}}'
# Match only the outermost {{...}} braces
result = re.sub(r'\{\{\s*([^{}]*?)\s*\}\}', strip_outer_whitespace, text)
return result
params = {"year": 2016}
df = run_saved_query('d045b229-e5d0-4de0-bcbc-bec6cbba61d7', **params)
df.display()