Review a churn model with a 0.97 AUC

A telecom operator (simulated) wants a churn model to prioritise retention calls. A colleague (simulated) opens a pull request with this description: "Churn model v1. Snapshot 2024-03-31, label = churned between 2024-04-01 and 2024-06-30. ROC-AUC 0.97 on a 20% holdout. Ready to ship." The data are synthetic.

import pandas as pd
from sklearn.model_selection import train_test_split
from sklearn.preprocessing import StandardScaler
from sklearn.linear_model import LogisticRegression
from sklearn.metrics import roc_auc_score

customers = pd.read_parquet("customers.parquet")  # 1 row per customer_id; tenure_months, churned_q2 (0/1)
usage = pd.read_parquet("usage.parquet")          # 1 row per customer_id x month, 2023-01 .. 2024-06
tickets = pd.read_parquet("tickets.parquet")      # 1 row per support ticket: customer_id, ticket_id, opened_at, severity

df = (customers
      .merge(usage, on="customer_id", how="left")
      .merge(tickets[["customer_id", "ticket_id", "severity"]], on="customer_id", how="left"))

feats = df.groupby("customer_id").agg(
    avg_minutes=("minutes", "mean"),
    min_data_gb=("data_gb", "min"),
    n_tickets=("ticket_id", "count"),
    max_severity=("severity", "max"),
).reset_index()
df = df.merge(feats, on="customer_id")

X = df[["avg_minutes", "min_data_gb", "n_tickets", "max_severity", "tenure_months"]].fillna(0)
y = df["churned_q2"]
X = StandardScaler().fit_transform(X)
X_tr, X_te, y_tr, y_te = train_test_split(X, y, test_size=0.2, random_state=0)
model = LogisticRegression(max_iter=1000).fit(X_tr, y_tr)
print(roc_auc_score(y_te, model.predict_proba(X_te)[:, 1]))  # 0.97