Options
- Mark as New
- Bookmark
- Subscribe
- Mute
- Subscribe to RSS Feed
- Permalink
- Report Inappropriate Content
11-13-2023 11:51 PM
UDF registry file:
# Databricks notebook source
# MAGIC %pip install mlflow
# COMMAND ----------
import dlt
# COMMAND ----------
import mlflow
from pyspark.sql.functions import struct, col
logged_model = 'runs:/xxxx/model_overstort_all_devices'
# Load model as a Spark UDF. Override result_type if the model does not return double values.
loaded_model = mlflow.pyfunc.spark_udf(spark, model_uri=logged_model, result_type='double')
spark.udf.register("detect_anomaly", loaded_model)
Actual pipeline code:
CREATE OR REFRESH STREAMING LIVE TABLE iot_overstort_predict
COMMENT "Predicting anomalies in measurement data using the Isolation Forest model"
TBLPROPERTIES ("quality" = "gold")
AS SELECT
utc_timestamp,
measurement_value_float,
topic,
YEAR(utc_timestamp) as year,
MONTH(utc_timestamp) as month,
DAY(utc_timestamp) as day,
hour(utc_timestamp) as hour,
detect_anomaly(measurement_value_float)
FROM STREAM(aqf_tda.test.xxx)
Actual pipeline:
CREATE OR REFRESH STREAMING LIVE TABLE iot_overstort_predict
COMMENT "Predicting anomalies in measurement data using the Isolation Forest model"
TBLPROPERTIES ("quality" = "gold")
AS SELECT
utc_timestamp,
measurement_value_float,
topic,
YEAR(utc_timestamp) as year,
MONTH(utc_timestamp) as month,
DAY(utc_timestamp) as day,
hour(utc_timestamp) as hour,
detect_anomaly(measurement_value_float)
FROM STREAM(aqf_tda.test.iot_overstort_sample_v1)