Viewing logged User input and LLM response for a Deployed Chatbot

[Tee_O]

Hi All, I deployed a simple chatbot on Databricks Free Edition and I am trying to find the location where the user input and llm response are stored (logged) when a user asks a question (I assumed this would be in the payload table created but couldn't find them there).

When querying the llm in a notebook I can see the question and response under Experiments on databricks (with mlflow enabled) but when the chatbot is deployed I am unable to locate the logged question and llm response (I could note upload the notebook or html here so I pasted some the code). Any Ideas?

%pip install databricks-langchain langchain-core mlflow[databricks]

dbutils.library.restartPython()

#Install libraries

def wait_for_model_serving_endpoint_to_be_ready(ep_name😞

from databricks.sdk import WorkspaceClient

from databricks.sdk.service.serving import EndpointStateReady, EndpointStateConfigUpdate

import time

#Define Utility Functions

# TODO make the endpoint name as a param

# Wait for it to be ready

w = WorkspaceClient()

state = ""

for i in range(200😞

state = w.serving_endpoints.get(ep_name).state

if state.config_update == EndpointStateConfigUpdate.IN_PROGRESS:

if i % 40 == 0:

print(f"Waiting for endpoint to deploy {ep_name}. Current state: {state}")

time.sleep(10)

elif state.ready == EndpointStateReady.READY:

print('endpoint ready.')

return

else:

break

raise Exception(f"Couldn't start the endpoint, timeout, please check your endpoint for more details: {state}")

def extract_user_query_string(chat_messages_array😞

return chat_messages_array[-1]["content"]

#Create Chain and test

from langchain_core.prompts import ChatPromptTemplate

from databricks_langchain.chat_models import ChatDatabricks

from langchain_core.output_parsers import StrOutputParser

from operator import itemgetter

from langchain.schema.runnable import RunnableLambda

prompt = ChatPromptTemplate.from_messages(

[

("system", """You are an assistant that answers questions. """), # Contains the instructions from the configuration

("user", "{question}") #user's questions

]

)

# Our foundation model answering the final prompt

model = ChatDatabricks(

endpoint="databricks-meta-llama-3-3-70b-instruct",

extra_params={"temperature": 0.01, "max_tokens": 500}

)

# RAG Chain

chain = (

{

"question": itemgetter("messages")

| RunnableLambda(extract_user_query_string)

}

| prompt

| model

| StrOutputParser()

)

input_example = {"messages": [ {"role": "user", "content": "What is Retrieval-augmented Generation?"}]}

answer = chain.invoke(input_example)

print(answer)

import mlflow

from mlflow.models.resources import DatabricksVectorSearchIndex, DatabricksServingEndpoint

## Enable MLflow Tracing

mlflow.langchain.autolog()

input_example = {"messages": [ {"role": "user", "content": "What is Retrieval-augmented Generation?"}]}

# Log the model to MLflow

with mlflow.start_run(run_name="simple_rag_bot"😞

logged_chain_info = mlflow.langchain.log_model(

lc_model=chain,

artifact_path="chain",

input_example=input_example,

resources=[

DatabricksServingEndpoint(endpoint_name="databricks-meta-llama-3-3-70b-instruct")

]

)

# Let's give it a try:

answer = chain.invoke(input_example)

print(answer)

##Register Model

catalog = "main"

db = "default"

MODEL_NAME = "simple_rag_demo"

MODEL_NAME_FQN = f"{catalog}.{db}.{MODEL_NAME}"

# Register to UC

uc_registered_model_info = mlflow.register_model(model_uri=logged_chain_info.model_uri, name=MODEL_NAME_FQN)

##Deploy Chatbot

from databricks import agents

# Deploy to enable the Review APP and create an API endpoint

# Note: scaling down to zero will provide unexpected behavior for the chat app. Set it to false for a prod-ready application.

deployment_info = agents.deploy(MODEL_NAME_FQN, model_version=uc_registered_model_info.version, scale_to_zero=True,

input_schema = input_example

)

instructions_to_reviewer = f"""## Instructions for Testing the Assistant chatbot

Your inputs are invaluable for the development team. By providing detailed feedback and corrections, you help us fix issues and improve the overall quality of the application. We rely on your expertise to identify any gaps or areas needing enhancement."""

# Add the user-facing instructions to the Review App

agents.set_review_instructions(MODEL_NAME_FQN, instructions_to_reviewer)

wait_for_model_serving_endpoint_to_be_ready(deployment_info.endpoint_name)