Problem I am solving:

• Reads the raw sports data  IPL CSV → bronze layer

• Cleans and aggregates → silver layer

• Summarizes team stats → gold layer

• Prepares ML-ready features and trains a Random Forest classifier to predict match winners

Getting error: [PARSE_SYNTAX_ERROR] Syntax error at or near end of input. SQLSTATE: 42601  when i run code: 

import pandas as pd

import numpy as np

from sklearn.ensemble import RandomForestClassifier

from sklearn.metrics import accuracy_score, confusion_matrix

from sklearn.model_selection import train_test_split

# Ensure 'features' is array<float> and collect to Pandas (classic Spark API)

pdf = (

    df_ml

    .select("features", "label")

    .limit(10000)  # Optional: limit for performance

    .collect()

)

# Expand features into proper numeric columns

X = pd.DataFrame([

    np.array(f) if f is not None else np.zeros(len(pdf["features"][0]))

    for f in pdf["features"]

])

y = pdf["label"].astype(int)

# Train/test split

X_train, X_test, y_train, y_test = train_test_split(

    X, y,

    test_size=0.2,

    random_state=42,

    stratify=y

)

# Train Random Forest (scikit-learn)

rf = RandomForestClassifier(

    n_estimators=50,

    max_depth=10,

    random_state=42,

    n_jobs=-1

)

rf.fit(X_train, y_train)

# Predictions & evaluation

y_pred = rf.predict(X_test)

acc = accuracy_score(y_test, y_pred)

display(f"✅ RandomForest trained successfully. Test Accuracy = {acc:.2f}")

# Confusion matrix

cm = confusion_matrix(y_test, y_pred)

display(cm)

# Map back to team names

index_to_team = {idx: team for team, idx in team_to_index.items()}

pred_vs_actual = pd.DataFrame({

    "actual": y_test.map(index_to_team).reset_index(drop=True),

    "predicted": pd.Series(y_pred).map(index_to_team)

})

display(pred_vs_actual.head(10))