교차 검증(Cross-Validation): K-Fold, Stratified, Time Series Split

from sklearn.model_selection import train_test_split
from sklearn.ensemble import RandomForestClassifier
from sklearn.metrics import accuracy_score

X_train, X_test, y_train, y_test = train_test_split(
    X, y, test_size=0.2, random_state=42
)

model = RandomForestClassifier(n_estimators=100, random_state=42)
model.fit(X_train, y_train)
accuracy = accuracy_score(y_test, model.predict(X_test))

scores = []
for seed in range(10):
    X_tr, X_te, y_tr, y_te = train_test_split(X, y, test_size=0.2, random_state=seed)
    model = RandomForestClassifier(n_estimators=100, random_state=42)
    model.fit(X_tr, y_tr)
    scores.append(accuracy_score(y_te, model.predict(X_te)))

print(f"Min: {min(scores):.4f}, Max: {max(scores):.4f}, Std: {np.std(scores):.4f}")

K=5 예시:

Fold 1: [검증] [훈련] [훈련] [훈련] [훈련]  → Score₁
Fold 2: [훈련] [검증] [훈련] [훈련] [훈련]  → Score₂
Fold 3: [훈련] [훈련] [검증] [훈련] [훈련]  → Score₃
Fold 4: [훈련] [훈련] [훈련] [검증] [훈련]  → Score₄
Fold 5: [훈련] [훈련] [훈련] [훈련] [검증]  → Score₅

최종 점수 = (Score₁ + Score₂ + ... + Score₅) / 5

전체 데이터: 양성 10%, 음성 90%

기본 K-Fold (fold별 양성 비율이 다를 수 있음):
  Fold 1: 양성 8%   Fold 2: 양성 14%  Fold 3: 양성 7%  ...

Stratified K-Fold (fold별 양성 비율 유지):
  Fold 1: 양성 10%  Fold 2: 양성 10%  Fold 3: 양성 10%  ...

from sklearn.model_selection import StratifiedKFold

skf = StratifiedKFold(n_splits=5, shuffle=True, random_state=42)

for fold, (train_idx, val_idx) in enumerate(skf.split(X, y)):
    print(f"Fold {fold}: 훈련 양성 비율 = {y[train_idx].mean():.3f}, "
          f"검증 양성 비율 = {y[val_idx].mean():.3f}")

from sklearn.model_selection import LeaveOneOut, cross_val_score

loo = LeaveOneOut()
scores = cross_val_score(model, X, y, cv=loo, scoring='accuracy')
print(f"LOO 평균 정확도: {scores.mean():.4f}")

from sklearn.model_selection import RepeatedStratifiedKFold

rskf = RepeatedStratifiedKFold(n_splits=5, n_repeats=10, random_state=42)
scores = cross_val_score(model, X, y, cv=rskf, scoring='f1')
print(f"Repeated 5-Fold (10회): {scores.mean():.4f} (+/- {scores.std():.4f})")

일반 K-Fold의 문제 (시계열에서):

시간 →  [1월] [2월] [3월] [4월] [5월]

Fold 3:  [훈련] [훈련] [검증] [훈련] [훈련]
          ↑ 3월을 예측하는데 4월, 5월 데이터로 학습?!

Time Series Split (5 splits):

Split 1: [훈련]  [검증]
Split 2: [훈련]  [훈련]  [검증]
Split 3: [훈련]  [훈련]  [훈련]  [검증]
Split 4: [훈련]  [훈련]  [훈련]  [훈련]  [검증]
Split 5: [훈련]  [훈련]  [훈련]  [훈련]  [훈련]  [검증]

from sklearn.model_selection import TimeSeriesSplit

tscv = TimeSeriesSplit(n_splits=5)

for fold, (train_idx, val_idx) in enumerate(tscv.split(X)):
    print(f"Split {fold}: 훈련 {len(train_idx)}개 (idx {train_idx[0]}-{train_idx[-1]}), "
          f"검증 {len(val_idx)}개 (idx {val_idx[0]}-{val_idx[-1]})")

전체 데이터
├── 훈련셋(Training Set)     → 모델 학습
├── 검증셋(Validation Set)   → 하이퍼파라미터 튜닝, 모델 선택
└── 테스트셋(Test Set)       → 최종 성능 보고 (딱 한 번만 사용)

# 실전 워크플로우
# 1단계: 테스트셋 분리 (처음에 한 번)
X_dev, X_test, y_dev, y_test = train_test_split(X, y, test_size=0.15, random_state=42)

# 2단계: 개발셋에서 교차 검증으로 모델/하이퍼파라미터 선택
cv_scores = cross_val_score(model, X_dev, y_dev, cv=5, scoring='f1')
print(f"CV F1: {cv_scores.mean():.4f}")

# 3단계: 최종 모델을 전체 개발셋으로 재학습
best_model.fit(X_dev, y_dev)

# 4단계: 테스트셋으로 최종 성능 보고 (단 한 번!)
test_score = f1_score(y_test, best_model.predict(X_test))
print(f"Test F1: {test_score:.4f}")

from sklearn.preprocessing import StandardScaler

# 전체 데이터로 스케일링 (누수!)
scaler = StandardScaler()
X_scaled = scaler.fit_transform(X)  # 전체 X의 평균/표준편차 사용

# 스케일링된 데이터로 교차 검증
scores = cross_val_score(model, X_scaled, y, cv=5)

from sklearn.pipeline import Pipeline
from sklearn.preprocessing import StandardScaler
from sklearn.linear_model import LogisticRegression

# Pipeline: 전처리가 각 fold 안에서 실행됨
pipe = Pipeline([
    ('scaler', StandardScaler()),
    ('clf', LogisticRegression())
])

# cross_val_score가 각 fold에서 pipe.fit() → pipe.predict()를 호출
# → 각 fold의 훈련 데이터만으로 fit_transform, 검증 데이터는 transform만
scores = cross_val_score(pipe, X, y, cv=5, scoring='accuracy')

from sklearn.model_selection import cross_val_score

scores = cross_val_score(
    estimator=model,
    X=X, y=y,
    cv=5,                    # int → StratifiedKFold (분류) 또는 KFold (회귀)
    scoring='accuracy',      # 평가 지표
    n_jobs=-1                # 병렬 처리
)

print(f"Accuracy: {scores.mean():.4f} (+/- {scores.std():.4f})")

from sklearn.model_selection import cross_validate

results = cross_validate(
    estimator=model,
    X=X, y=y,
    cv=5,
    scoring=['accuracy', 'f1', 'roc_auc'],  # 여러 지표 동시에
    return_train_score=True,                  # 훈련 점수도 반환
    n_jobs=-1
)

print(f"Test Accuracy:  {results['test_accuracy'].mean():.4f}")
print(f"Train Accuracy: {results['train_accuracy'].mean():.4f}")
print(f"Test F1:        {results['test_f1'].mean():.4f}")
print(f"Fit time:       {results['fit_time'].mean():.2f}s")

from sklearn.model_selection import StratifiedKFold, RepeatedStratifiedKFold

# 기본 Stratified K-Fold
skf = StratifiedKFold(n_splits=10, shuffle=True, random_state=42)
scores = cross_val_score(model, X, y, cv=skf)

# Repeated Stratified K-Fold
rskf = RepeatedStratifiedKFold(n_splits=5, n_repeats=10, random_state=42)
scores = cross_val_score(model, X, y, cv=rskf)

# Time Series Split
tscv = TimeSeriesSplit(n_splits=5)
scores = cross_val_score(model, X, y, cv=tscv)

from sklearn.linear_model import LogisticRegression
from sklearn.ensemble import RandomForestClassifier, GradientBoostingClassifier
from sklearn.neural_network import MLPClassifier
from sklearn.preprocessing import StandardScaler
from sklearn.pipeline import Pipeline
from sklearn.model_selection import cross_validate, StratifiedKFold
import pandas as pd

# 동일한 CV 전략
cv = StratifiedKFold(n_splits=5, shuffle=True, random_state=42)

models = {
    'LogisticRegression': Pipeline([
        ('scaler', StandardScaler()),
        ('clf', LogisticRegression(max_iter=1000))
    ]),
    'RandomForest': RandomForestClassifier(n_estimators=100, random_state=42),
    'GBM': GradientBoostingClassifier(n_estimators=100, random_state=42),
    'MLP': Pipeline([
        ('scaler', StandardScaler()),
        ('clf', MLPClassifier(hidden_layer_sizes=(64, 32), max_iter=500, random_state=42))
    ]),
}

results = {}
for name, model in models.items():
    cv_result = cross_validate(
        model, X, y, cv=cv,
        scoring=['accuracy', 'f1'],
        return_train_score=True,
        n_jobs=-1
    )
    results[name] = {
        'Test Acc': f"{cv_result['test_accuracy'].mean():.4f} (+/- {cv_result['test_accuracy'].std():.4f})",
        'Test F1':  f"{cv_result['test_f1'].mean():.4f} (+/- {cv_result['test_f1'].std():.4f})",
        'Train Acc': f"{cv_result['train_accuracy'].mean():.4f}",
        'Fit Time': f"{cv_result['fit_time'].mean():.2f}s",
    }

df = pd.DataFrame(results).T
print(df)

from sklearn.model_selection import GroupKFold

groups = patient_ids  # 각 샘플이 속한 그룹 (환자 ID 등)
gkf = GroupKFold(n_splits=5)
scores = cross_val_score(model, X, y, cv=gkf, groups=groups)