경사하강법(Gradient Descent): 비용 함수의 바닥을 찾아가는 알고리즘

J(w, b) = (1/n) × Σᵢ(ŷᵢ − yᵢ)²
         = (1/n) × Σᵢ(wxᵢ + b − yᵢ)²

∂J/∂w = (2/n) × Σᵢ(wxᵢ + b − yᵢ) × xᵢ
∂J/∂b = (2/n) × Σᵢ(wxᵢ + b − yᵢ)

import numpy as np

def compute_gradients(X, y, w, b):
    n = len(y)
    y_pred = w * X + b
    error = y_pred - y             # 각 포인트의 오차
    dw = (2/n) * np.sum(error * X)  # w에 대한 기울기
    db = (2/n) * np.sum(error)      # b에 대한 기울기
    return dw, db

import numpy as np

# 데이터 (선형 회귀 글과 동일)
X = np.array([60, 75, 85, 95, 110, 120, 140, 155], dtype=float)
y = np.array([2.1, 2.8, 3.2, 3.6, 4.1, 4.5, 5.2, 5.8])

# 초기값
w = 0.0
b = 0.0
learning_rate = 0.00001  # 학습률
epochs = 5000            # 반복 횟수

# 학습 기록
history = []

for epoch in range(epochs):
    # 1. 예측
    y_pred = w * X + b

    # 2. 비용 계산
    cost = np.mean((y_pred - y) ** 2)

    # 3. 기울기 계산
    n = len(y)
    dw = (2/n) * np.sum((y_pred - y) * X)
    db = (2/n) * np.sum(y_pred - y)

    # 4. 파라미터 업데이트
    w = w - learning_rate * dw
    b = b - learning_rate * db

    if epoch % 1000 == 0:
        history.append((epoch, cost, w, b))
        print(f"Epoch {epoch:5d} | Cost: {cost:.6f} | w: {w:.6f} | b: {b:.6f}")

print(f"\n최종 결과: w = {w:.4f}, b = {b:.4f}")

Epoch     0 | Cost: 16.648750 | w: 0.008920 | b: 0.000078
Epoch  1000 | Cost: 0.003062 | w: 0.037320 | b: 0.000201
Epoch  2000 | Cost: 0.003061 | w: 0.037322 | b: 0.000074
Epoch  3000 | Cost: 0.003059 | w: 0.037323 | b: -0.000053
Epoch  4000 | Cost: 0.003057 | w: 0.037324 | b: -0.000179

최종 결과: w = 0.0373, b = -0.0003

learning_rates = [0.000001, 0.00001, 0.0001]

for lr in learning_rates:
    w, b = 0.0, 0.0
    for _ in range(3000):
        y_pred = w * X + b
        dw = (2/len(y)) * np.sum((y_pred - y) * X)
        db = (2/len(y)) * np.sum(y_pred - y)
        w -= lr * dw
        b -= lr * db
    cost = np.mean((w * X + b - y) ** 2)
    print(f"lr={lr:.6f} → w={w:.4f}, b={b:.4f}, cost={cost:.6f}")

lr=0.0000002 → w=0.0373, b=0.0003, cost=0.003064
lr=0.0000100 → w=0.0373, b=-0.0001, cost=0.003059
lr=0.0001000 → w=nan, b=nan, cost=nan

# 미니배치 경사하강법 예시
batch_size = 4

for epoch in range(1000):
    # 데이터를 랜덤하게 셔플
    indices = np.random.permutation(len(y))

    for start in range(0, len(y), batch_size):
        batch_idx = indices[start:start + batch_size]
        X_batch = X[batch_idx]
        y_batch = y[batch_idx]

        y_pred = w * X_batch + b
        dw = (2/len(y_batch)) * np.sum((y_pred - y_batch) * X_batch)
        db = (2/len(y_batch)) * np.sum(y_pred - y_batch)
        w -= learning_rate * dw
        b -= learning_rate * db

tolerance = 1e-7
prev_cost = float('inf')

for epoch in range(100000):
    y_pred = w * X + b
    cost = np.mean((y_pred - y) ** 2)

    # 수렴 판정
    if abs(prev_cost - cost) < tolerance:
        print(f"수렴! Epoch {epoch}, Cost: {cost:.8f}")
        break

    prev_cost = cost
    # ... (기울기 계산 및 업데이트)