Model Evaluation, Validation & Improvement

# =====================================================================
# DevAscent Academy: Model Evaluation & Hyperparameter Tuning Pipeline
# Requires: numpy, pandas, scikit-learn
# =====================================================================

import numpy as np
import pandas as pd
from sklearn.datasets import make_classification
from sklearn.model_selection import (
    train_test_split, StratifiedKFold, cross_val_score,
    GridSearchCV, RandomizedSearchCV
)
from sklearn.preprocessing import StandardScaler
from sklearn.ensemble import RandomForestClassifier
from sklearn.linear_model import LogisticRegression
from sklearn.inspection import permutation_importance
from sklearn.metrics import classification_report, roc_auc_score, f1_score

def execute_model_improvement_pipeline():
    # -----------------------------------------------------------------
    # 1. Synthesize Imbalanced Fraud-Like Dataset (N=1500, Ratio=90:10)
    # -----------------------------------------------------------------
    X_raw, y = make_classification(
        n_samples=1500,
        n_features=15,
        n_informative=8,
        n_redundant=4,
        weights=[0.90, 0.10],
        random_state=42
    )

    feature_names = [f"sensor_metric_{i+1}" for i in range(15)]
    df = pd.DataFrame(X_raw, columns=feature_names)
    df['is_fraud'] = y

    print(f"[Dataset] Shape: {df.shape} | Class Balance: "
          f"{np.bincount(y)[0]} Normal vs {np.bincount(y)[1]} Fraud ({np.mean(y)*100:.1f}%)")

    # -----------------------------------------------------------------
    # 2. Strict Train-Validation-Test Splitting (Firewall Verification)
    # -----------------------------------------------------------------
    X = df.drop(columns=['is_fraud'])
    y = df['is_fraud']

    X_train, X_test, y_train, y_test = train_test_split(
        X, y, test_size=0.20, stratify=y, random_state=42
    )

    scaler = StandardScaler()
    X_train_scaled = scaler.fit_transform(X_train)
    X_test_scaled = scaler.transform(X_test)  # Transform test using train parameters

    # -----------------------------------------------------------------
    # 3. Stratified 5-Fold Cross-Validation Baseline
    # -----------------------------------------------------------------
    print("\n--- [Phase 1] Stratified 5-Fold Cross-Validation ---")
    cv_strategy = StratifiedKFold(n_splits=5, shuffle=True, random_state=42)
    
    # Baseline Unweighted Model
    baseline_clf = RandomForestClassifier(n_estimators=100, random_state=42)
    cv_roc = cross_val_score(baseline_clf, X_train_scaled, y_train, cv=cv_strategy, scoring='roc_auc')
    cv_f1 = cross_val_score(baseline_clf, X_train_scaled, y_train, cv=cv_strategy, scoring='f1')

    print(f"Baseline RF (Unweighted) | 5-Fold Mean ROC-AUC: {cv_roc.mean():.4f} (±{cv_roc.std():.4f})")
    print(f"Baseline RF (Unweighted) | 5-Fold Mean F1-Score: {cv_f1.mean():.4f} (±{cv_f1.std():.4f})")

    # -----------------------------------------------------------------
    # 4. Hyperparameter Tuning: RandomizedSearchCV vs GridSearchCV
    # -----------------------------------------------------------------
    print("\n--- [Phase 2] Hyperparameter Optimization & Cost-Weighting ---")
    
    param_distributions = {
        'n_estimators': [50, 100, 150, 200],
        'max_depth': [5, 10, 15, None],
        'min_samples_split': [2, 5, 10],
        'class_weight': ['balanced', 'balanced_subsample', None]
    }

    # Randomized Search across 10 iterations
    rand_search = RandomizedSearchCV(
        estimator=RandomForestClassifier(random_state=42),
        param_distributions=param_distributions,
        n_iter=10,
        scoring='roc_auc',
        cv=cv_strategy,
        random_state=42,
        n_jobs=-1
    )
    rand_search.fit(X_train_scaled, y_train)

    print(f"Best Randomized Search Hyperparameters: {rand_search.best_params_}")
    print(f"Optimized Cross-Validation ROC-AUC: {rand_search.best_score_:.4f}")

    best_model = rand_search.best_estimator_

    # -----------------------------------------------------------------
    # 5. Out-of-Sample Test Evaluation & Audit
    # -----------------------------------------------------------------
    print("\n--- [Phase 3] Final Test Set Generalization Audit ---")
    y_pred = best_model.predict(X_test_scaled)
    y_prob = best_model.predict_proba(X_test_scaled)[:, 1]

    print(f"Test ROC-AUC Score: {roc_auc_score(y_test, y_prob):.4f}")
    print(f"Test Macro F1-Score: {f1_score(y_test, y_pred, average='macro'):.4f}")
    print("\nDetailed Classification Report:")
    print(classification_report(y_test, y_pred, target_names=['Normal', 'Fraud']))

    # -----------------------------------------------------------------
    # 6. Feature Importance: MDI vs Permutation Importance
    # -----------------------------------------------------------------
    print("--- [Phase 4] Feature Importance Comparison ---")
    
    # 1. Tree MDI (Mean Decrease Impurity)
    mdi_importances = best_model.feature_importances_
    
    # 2. Permutation Importance on Test Set
    perm_result = permutation_importance(
        best_model, X_test_scaled, y_test, scoring='roc_auc', n_repeats=10, random_state=42
    )

    importance_df = pd.DataFrame({
        'Feature': feature_names,
        'MDI_Importance': mdi_importances,
        'Permutation_Drop_Mean': perm_result.importances_mean,
        'Permutation_Drop_Std': perm_result.importances_std
    }).sort_values(by='Permutation_Drop_Mean', ascending=False)

    print(importance_df.head(6).to_string(index=False))

if __name__ == '__main__':
    execute_model_improvement_pipeline()