L2 懲罰將所有係數向零收縮，但永遠不會精確歸零
L1 懲罰可將係數強制歸零：自動特徵選擇
重點：正則化路徑圖（係數收縮 vs lambda）、RidgeCV/LassoCV 最佳 alpha 選擇、係數比較表、預測評估。設計用於 Google Colab。
import numpy as np
import pandas as pd
from sklearn.linear_model import LinearRegression, Ridge, Lasso, RidgeCV, LassoCV
from sklearn.model_selection import train_test_split, cross_val_score, KFold
from sklearn.metrics import mean_squared_error, r2_score
from sklearn.preprocessing import StandardScaler
from itertools import combinations
import warnings
warnings.filterwarnings('ignore')
# 1. Generate synthetic data
np.random.seed(42)
n, p = 200, 30
# Only 5 features matter (sparse true model)
beta_true = np.zeros(p)
beta_true[[0, 1, 4, 9, 15]] = [3.0, -2.0, 1.5, 0.8, -1.2]
X = np.random.randn(n, p)
X[:, 5:] += 0.3 * X[:, :5].sum(axis=1, keepdims=True) / 5
y = X @ beta_true + np.random.randn(n) * 1.5
X_train, X_test, y_train, y_test = train_test_split(
X, y, test_size=0.3, random_state=1)
scaler = StandardScaler()
X_train_scaled = scaler.fit_transform(X_train)
X_test_scaled = scaler.transform(X_test)
print("Data:", X_train.shape, X_test.shape)
baseline = mean_squared_error(y_test, np.full_like(y_test, y_train.mean()))
print("Baseline MSE:", baseline)
# 2. Best Subset Selection (k=5)
print("\n--- Best Subset (k=5) ---")
best_score = float('inf')
best_combo = None
for combo in combinations(range(p), 5):
lr = LinearRegression().fit(X_train_scaled[:, combo], y_train)
mse = mean_squared_error(y_test, lr.predict(X_test_scaled[:, combo]))
if mse < best_score:
best_score = mse
best_combo = combo
true_important = {0, 1, 4, 9, 15}
found = set(best_combo[:5])
print(f"Best 5-feature subset: {sorted(best_combo)}")
print(f"Test MSE: {best_score:.4f}")
print(f"True signal recall: {len(true_important & found)}/5")
# 3. Ridge Regression
print("\n--- Ridge ---")
ridge_cv = RidgeCV(alphas=np.logspace(-3, 3, 50))
ridge_cv.fit(X_train_scaled, y_train)
y_pred_ridge = ridge_cv.predict(X_test_scaled)
print(f"Best alpha: {ridge_cv.alpha_:.4f}")
print(f"Test MSE: {mean_squared_error(y_test, y_pred_ridge):.4f}")
# 4. Lasso Regression
print("\n--- Lasso ---")
lasso_cv = LassoCV(alphas=np.logspace(-3, 1, 50), cv=5,
max_iter=10000, random_state=42)
lasso_cv.fit(X_train_scaled, y_train)
y_pred_lasso = lasso_cv.predict(X_test_scaled)
nz = np.sum(lasso_cv.coef_ != 0)
selected = np.where(lasso_cv.coef_ != 0)[0]
print(f"Best alpha: {lasso_cv.alpha_:.4f}, Non-zero coefs: {nz}")
print(f"Selected: {sorted(selected)}")
recall = len(true_important & set(selected)) / len(true_important)
print(f"Recall: {recall:.0%}")
print(f"Test MSE: {mean_squared_error(y_test, y_pred_lasso):.4f}")
# 5. 5-Fold CV Comparison
print("\n--- 5-Fold CV ---")
models = {
'OLS': LinearRegression(),
'Ridge': Ridge(alpha=ridge_cv.alpha_),
'Lasso': Lasso(alpha=lasso_cv.alpha_, max_iter=10000)
}
kf = KFold(n_splits=5, shuffle=True, random_state=42)
for name, model in models.items():
scores = cross_val_score(model, X_train_scaled, y_train,
cv=kf, scoring='neg_mean_squared_error')
print(f"{name}: CV MSE = {-scores.mean():.4f} (+/- {scores.std():.4f})")
# 6. High-Dimensional: p > n
print("\n--- High-Dim: p=100, n=50 ---")
np.random.seed(0)
beta_big = np.zeros(100)
beta_big[[0, 1, 2]] = [5.0, -3.0, 2.0]
Xb = np.random.randn(50, 100)
yb = Xb @ beta_big + np.random.randn(50) * 0.5
Xb_tr, Xb_te, yb_tr, yb_te = train_test_split(
Xb, yb, test_size=15, random_state=1)
scb = StandardScaler()
Xb_tr_s = scb.fit_transform(Xb_tr)
Xb_te_s = scb.transform(Xb_te)
ridge_big = RidgeCV(alphas=np.logspace(-2, 2, 30)).fit(Xb_tr_s, yb_tr)
lasso_big = LassoCV(alphas=np.logspace(-2, 1, 30), cv=5,
max_iter=10000, random_state=42).fit(Xb_tr_s, yb_tr)
print(f"Ridge (p>n): MSE={mean_squared_error(yb_te, ridge_big.predict(Xb_te_s)):.4f}")
print(f"Lasso (p>n): MSE={mean_squared_error(yb_te, lasso_big.predict(Xb_te_s)):.4f}")
print(f"Lasso selected: {np.sum(lasso_big.coef_ != 0)}/100")
print("\nALL DONE")展示真正的訊號特徵（粗線）如何在正則化中存活，而雜訊特徵（細灰線）在低 alpha 時被消滅。
import matplotlib.pyplot as plt
alphas = np.logspace(-3, 1, 100)
coefs = []
for a in alphas:
lasso = Lasso(alpha=a, max_iter=10000)
lasso.fit(X_train_scaled, y_train)
coefs.append(lasso.coef_)
coefs = np.array(coefs)
plt.figure(figsize=(12, 6))
for i in [0, 1, 4, 9, 15]:
plt.plot(alphas, coefs[:, i], linewidth=2, label=f'Feature {i} (true)')
for i in [2, 3, 5, 6, 7, 8, 10, 11, 12]:
plt.plot(alphas, coefs[:, i], alpha=0.3, linewidth=0.8, color='gray')
plt.xscale('log')
plt.axvline(lasso_cv.alpha_, color='red', linestyle='--',
label=f'CV best alpha={lasso_cv.alpha_:.3f}')
plt.xlabel('Alpha (regularization strength)')
plt.ylabel('Coefficient value')
plt.title('Lasso Coefficient Paths: True Signals Survive Longer')
plt.legend(fontsize=8)
plt.tight_layout()
plt.show()| 方法 | 特徵選擇 | 速度 (p=30) | p > n 可用 | 可解釋性 |
|---|---|---|---|---|
| 最佳子集 | 明確選 k 個最佳 | 慢：C(p,k) | 否 | 高 |
| 向前逐步 | 貪婪路徑 | 快 | 否 | 中 |
| Ridge | 無（全部保留） | 快 | 是 | 低 |
| Lasso | 自動歸零 | 快 | 是（最多 n 個） | 高 |
| PCR/PLS | 降維 | 中 | 是 | 低 |
Lasso 是標準工具：自動選出與疾病真正相關的少數基因。Ridge 會保留全部 20000 個基因，每個係數極小（無法解釋）。最佳子集在計算上不可行（C(20000, k) 是天文數字）。
交叉驗證的 Ridge 在這裡通常表現最好，因為所有因子可能都有一點影響（總體經濟力量），且 n > p 代表我們撐得住保留全部因子。向前逐步在金融文獻中很受歡迎，因其可解釋性（Fama-MacBeth 傳統）。
對 Hermes 的啟發：這堂課呼應我們自己的 agent 架構辯論。Lasso 風格「歸零不相關特徵」對應技能修剪（哪些技能真正重要？）。Ridge 風格「全部保留但收縮」對應 RAG 檢索（所有記憶都有貢獻，但以重要性加權）。第六章的關鍵洞察：沒有普遍最佳的方法——正確的選擇完全取決於你的資料結構和你的目標（預測 vs 可解釋性、稠密真相 vs 稀疏真相）。Agent 記憶系統同理：沒有任何單一架構能在所有場景中稱霸（arXiv 2606.24775 本週也得到相同結論）。