Not able to query serving endpoint thr UI consisting of logged pyfunc model calling DBRX endpoint

ram0021 — Mon, 22 Jul 2024 14:01:09 GMT

Hi All,

I need some urgent help. I have created an serving endpoint through below code. The code logs in pyfunc model that internally calls DBRX endpoint. However when i query using below input i always get same error. Can someone please help me with it. Its really urgent.

code to create endpoint:

import mlflow

import pandas as pd

from mlflow.models.signature import infer_signature

import json

# define a custom model

class MyModel(mlflow.pyfunc.PythonModel😞

def predict(self, context, model_input, params=None😞

print('input predict', model_input)

print('type input predict', type(model_input))

print('output predict', self.my_custom_function(model_input, params))

print('type output predict', type(self.my_custom_function(model_input, params)))

return self.my_custom_function(model_input, params)

def my_custom_function(self, model_input, params=None😞

# Convert DataFrame to a list of dictionaries

model_input_list = model_input.to_dict(orient='records')

# Convert list of dictionaries to JSON string

model_input_json = json.dumps(model_input_list)

print(type(model_input_json))

print('model_input_json:',model_input_json)

client = get_deploy_client("databricks")

response = client.predict(

endpoint="openai",

inputs={

"messages": [

{"role": "user", "content": model_input_json},

)

print('output1',response)

print('type', type(response))

json_string = json.dumps(response, indent=4)

print('json_string',json_string)

print('type json_string',type(json_string))

return pd.DataFrame({'output': [json_string]})

# return pd.DataFrame({'predictions': ['Hi']})

# Convert some_input to a pandas DataFrame

some_input = pd.DataFrame(['how are you'])

# save the model

with mlflow.start_run():

model = MyModel()

input_example = some_input

signature = infer_signature(input_example, model.my_custom_function(input_example))

model_info = mlflow.pyfunc.log_model(

artifact_path="model",

python_model=model,

signature=signature,

input_example=input_example

)

# Register the model

catalog = "aclr_u"

schema = "chrgbck_s"

model_name = "new_pyfunc"

mlflow.set_registry_uri("databricks-uc")

mlflow.register_model(model_info.model_uri, f"{catalog}.{schema}.{model_name}")

print('/n model registered/n')

error: {"error_code": "BAD_REQUEST", "message": "Encountered an unexpected error while evaluating the model. Verify that the input is compatible with the model for inference. Error 'Expecting value: line 1 column 1 (char 0)'", "stack_trace": "Traceback (most recent call last):\n File \"/opt/conda/envs/mlflow-env/lib/python3.10/site-packages/requests/models.py\", line 974, in json\n return complexjson.loads(self.text, **kwargs)\n File \"/opt/conda/envs/mlflow-env/lib/python3.10/json/__init__.py\", line 346, in loads\n return _default_decoder.decode(s)\n File \"/opt/conda/envs/mlflow-env/lib/python3.10/json/decoder.py\", line 337, in decode\n obj, end = self.raw_decode(s, idx=_w(s, 0).end())\n File \"/opt/conda/envs/mlflow-env/lib/python3.10/json/decoder.py\", line 355, in raw_decode\n raise JSONDecodeError(\"Expecting value\", s, err.value) from None\njson.decoder.JSONDecodeError: Expecting value: line 1 column 1 (char 0)\n\nDuring handling of the above exception, another exception occurred:\n\nTraceback (most recent call last):\n File \"/opt/conda/envs/mlflow-env/lib/python3.10/site-packages/mlflowserving/scoring_server/__init__.py\", line 627, in transformation\n (raw_predictions, databricks_output, fs_metrics) = _score_model(\n File \"/opt/conda/envs/mlflow-env/lib/python3.10/site-packages/mlflowserving/scoring_server/__init__.py\", line 368, in _score_model\n prediction, fs_metrics = score_model_maybe_with_fs_metrics(model, data, params)\n File \"/opt/conda/envs/mlflow-env/lib/python3.10/site-packages/mlflowserving/scoring_server/scoring_server_utils.py\", line 595, in score_model_maybe_with_fs_metrics\n return (score_pyfunc_model(model, data, params), None)\n File \"/opt/conda/envs/mlflow-env/lib/python3.10/site-packages/mlflowserving/scoring_server/scoring_server_utils.py\", line 600, in score_pyfunc_model\n return model.predict(data, params=params)\n File \"/opt/conda/envs/mlflow-env/lib/python3.10/site-packages/mlflow/pyfunc/__init__.py\", line 492, in predict\n return _predict()\n File \"/opt/conda/envs/mlflow-env/lib/python3.10/site-packages/mlflow/pyfunc/__init__.py\", line 478, in _predict\n return self._predict_fn(data, params=params)\n File \"/opt/conda/envs/mlflow-env/lib/python3.10/site-packages/mlflow/pyfunc/model.py\", line 469, in predict\n return self.python_model.predict(\n File \"/root/.ipykernel/8563/command-2484865133215772-2987730514\", line 16, in predict\n File \"/root/.ipykernel/8563/command-2484865133215772-2987730514\", line 54, in my_custom_function\n File \"/opt/conda/envs/mlflow-env/lib/python3.10/site-packages/requests/models.py\", line 978, in json\n raise RequestsJSONDecodeError(e.msg, e.doc, e.pos)\nrequests.exceptions.JSONDecodeError: Expecting value: line 1 column 1 (char 0)\n"}

input given to serving UI in databricks:

{
"dataframe_split": {
"data": [
[
"how are you"
]
]
}
}

topic Not able to query serving endpoint thr UI consisting of logged pyfunc model calling DBRX endpoint in Generative AI

Not able to query serving endpoint thr UI consisting of logged pyfunc model calling DBRX endpoint