Classification in Practice

Run the script. It builds 6,000 bookings with about 2% no-shows, implements a minimal SMOTE, and cross-validates a random forest twice: with SMOTE applied to all rows before the folds (scored on original rows only) and with SMOTE inside each training fold. Then:

1. Explain why the first number is so high although validation rows are all original. Which rows leaked what?
2. A constant score would reach AP 0.02. How much of the honest AP of 0.078 is signal?
3. Block B shows the mean probability after SMOTE to 50/50. What would the cost threshold of 0.059 do with it?

import numpy as np
from sklearn.ensemble import RandomForestClassifier
from sklearn.linear_model import LogisticRegression
from sklearn.metrics import average_precision_score
from sklearn.model_selection import StratifiedKFold
from sklearn.neighbors import NearestNeighbors

rng = np.random.default_rng(13)
n = 6000                                          # synthetic bookings, 2% no-shows, a weak signal in 6 features
X = rng.normal(size=(n, 6))
y = rng.random(n) < 1 / (1 + np.exp(-(X[:, :3].sum(axis=1) * 0.5 - 4.4)))

def smote(Xp, n_new, k=5, seed=0):                # minimal SMOTE: interpolate between a minority row and one of its k neighbours
    r = np.random.default_rng(seed)
    nb = NearestNeighbors(n_neighbors=k + 1).fit(Xp).kneighbors(Xp, return_distance=False)[:, 1:]
    i = r.integers(0, len(Xp), n_new)
    j = nb[i, r.integers(0, k, n_new)]
    return Xp[i] + r.random((n_new, 1)) * (Xp[j] - Xp[i])

def balance(Xa, ya, seed=0):                      # oversample positives to 50/50 with SMOTE
    new = smote(Xa[ya], (~ya).sum() - ya.sum(), seed=seed)
    return np.vstack([Xa, new]), np.r_[ya, np.ones(len(new), bool)]

def cv_ap(make_model, outside):
    scores = []
    folds = StratifiedKFold(5, shuffle=True, random_state=0)
    if outside:                                   # WRONG: synthetic rows are created from all positives before the folds
        Xs, ys = balance(X, y)
        orig = np.r_[np.ones(n, bool), np.zeros(len(ys) - n, bool)]
        for tr, te in folds.split(Xs, ys):
            te = te[orig[te]]                     # score only on original rows, for a fair comparison
            m = make_model().fit(Xs[tr], ys[tr])
            scores.append(average_precision_score(ys[te], m.predict_proba(Xs[te])[:, 1]))
    else:                                         # RIGHT: oversample inside each training fold only
        for tr, te in folds.split(X, y):
            Xb, yb = balance(X[tr], y[tr])
            m = make_model().fit(Xb, yb)
            scores.append(average_precision_score(y[te], m.predict_proba(X[te])[:, 1]))
    return np.mean(scores)

rf = lambda: RandomForestClassifier(100, random_state=0, n_jobs=-1)
print(f"A) prevalence {y.mean():.1%}; AP of a constant score = {y.mean():.3f}")
print(f"A) SMOTE before CV (leak):   AP {cv_ap(rf, True):.3f}")
print(f"A) SMOTE inside each fold:   AP {cv_ap(rf, False):.3f}")
# B) what SMOTE does to probabilities: the model now believes positives are common
Xb, yb = balance(X, y)
lr = LogisticRegression().fit(Xb, yb)
print(f"B) mean predicted probability after SMOTE to 50/50: {lr.predict_proba(X)[:, 1].mean():.3f} (actual rate {y.mean():.3f})")
print(f"B) plain logistic regression: {LogisticRegression().fit(X, y).predict_proba(X)[:, 1].mean():.3f}")

A teammate wants to use SMOTE on a 0.5% defect-rate dataset and report the F1 score from cross-validation. How do you respond?