I am having an issue with getting a display upon interaction with the ipywidgets dropdown menu. Once I've selected an option from the dropdown, nothing happens. I am inclined to believe it has to do with how I've structure my on_select function and not the .observe function for the dropdown. Any suggestions?
import pyspark.sql.functions as F
from pyspark.sql import SparkSession
import ipywidgets as widgets
from IPython.display import HTML, display
# Create the SparkSession object
appName = "hive_pyspark"
master = "local"
spark = SparkSession.builder.appName(appName).enableHiveSupport().getOrCreate()
# Read in the data dictionary table
df = spark.table("data_dictionary")
# Define a function for handling table name selections
def on_select(name, value):
print(f"Selected Value: {value}")
# Clear the previously displayed HTML table
display(HTML("<div id='table-data'></div>"))
# Filter down to the selected table
selected_table = df.filter(F.col("table_name") == value)
# Display the table data in an HTML table format
display(HTML(selected_table.toPandas().to_html(index=False)))
# Define and display a dropdown menu for selecting a table name
table_names = df.select(F.col("table_name")).distinct().rdd.flatMap(lambda x: x).collect()
table_dropdown = widgets.Dropdown(
options=table_names,
description='Select Table Name:'
)
table_dropdown.observe(on_select, names='value')
display(table_dropdown)
