Hi @michelleliu 

For DLT pipeline monitoring, you have several options depending on what you want to achieve:

Built-in DLT Monitoring: Try this

# Add this table to your existing DLT pipeline
@dlt.table(
    comment="Pipeline performance metrics",
    table_properties={"quality": "bronze"}
)
def pipeline_performance_metrics():
    import time
    from datetime import datetime
    
    def collect_metrics():
        executor_infos = spark.sparkContext.statusTracker().getExecutorInfos()
        
        return [{
            "timestamp": datetime.now(),
            "pipeline_id": spark.conf.get("spark.databricks.pipelines.pipelineId"),
            "update_id": spark.conf.get("spark.databricks.pipelines.updateId"),
            "active_executors": len([e for e in executor_infos if e.executorId != "driver"]),
            "total_memory_mb": sum([e.maxMemory for e in executor_infos if e.executorId != "driver"]) / 1024 / 1024,
            "used_memory_mb": sum([e.memoryUsed for e in executor_infos if e.executorId != "driver"]) / 1024 / 1024,
            "memory_utilization": sum([e.memoryUsed for e in executor_infos if e.executorId != "driver"]) / max(sum([e.maxMemory for e in executor_infos if e.executorId != "driver"]), 1)
        }]
    
    return spark.createDataFrame(collect_metrics())