9.6 Lab: 支援向量機

📦 環境設定與基礎匯入

本 Lab 使用 sklearn.svm.SVC 來展示支援向量分類器與支援向量機。我們從必要的匯入開始：

# 9.6 Lab: Support Vector Machines — 環境設定
import numpy as np
import matplotlib
matplotlib.use('Agg')
from matplotlib.pyplot import subplots, cm, savefig
import sklearn.model_selection as skm
from ISLP import load_data, confusion_table
from sklearn.svm import SVC
from ISLP.svm import plot as plot_svm
from sklearn.metrics import RocCurveDisplay

# ROC 曲線繪圖縮寫
roc_curve = RocCurveDisplay.from_estimator
print("匯入完成 ✓")

9.6.1 支援向量分類器（SVC）

我們從模擬一個二維二分類資料集開始。先用線性核心的 SVC 擬合，觀察不同懲罰參數 \(C\) 對邊界和支援向量數量的影響。

# 9.6.1 SVC — 線性核心，模擬資料與 C 值的影響
import numpy as np
import matplotlib
matplotlib.use('Agg')
from matplotlib.pyplot import subplots, cm, savefig
import sklearn.model_selection as skm
from ISLP import confusion_table
from sklearn.svm import SVC
from ISLP.svm import plot as plot_svm

# --- 生成資料 ---
rng = np.random.default_rng(1)
X = rng.standard_normal((50, 2))
y = np.array([-1]*25 + [1]*25)
X[y==1] += 1

# --- 擬合 C=10 ---
svm_linear = SVC(C=10, kernel='linear')
svm_linear.fit(X, y)

# --- 繪圖 ---
fig, ax = subplots(figsize=(8,8))
plot_svm(X, y, svm_linear, ax=ax)
ax.set_title("SVC (C=10, linear kernel)")
savefig('/tmp/svm_linear_c10.png', dpi=100)
print(f"係數: {svm_linear.coef_}")

# --- 較小的 C ---
svm_linear_small = SVC(C=0.1, kernel='linear')
svm_linear_small.fit(X, y)
fig, ax = subplots(figsize=(8,8))
plot_svm(X, y, svm_linear_small, ax=ax)
ax.set_title("SVC (C=0.1, linear kernel)")
savefig('/tmp/svm_linear_c01.png', dpi=100)
print("C=10 vs C=0.1 比較完成 ✓")

\(C=10\) 時邊界較窄、支援向量較少；\(C=0.1\) 時邊界變寬、支援向量增多（標記為 + 的點）。這直觀展示了 SVM 的偏誤-變異權衡：\(C\) 大 → 低偏誤高變異（可能過擬合），\(C\) 小 → 高偏誤低變異（更強的正則化）。

交叉驗證選擇 C

# SVC 交叉驗證調整 C
import numpy as np
import matplotlib
matplotlib.use('Agg')
from sklearn.svm import SVC
import sklearn.model_selection as skm
from ISLP import confusion_table

# --- 重建資料 ---
rng = np.random.default_rng(1)
X = rng.standard_normal((50, 2))
y = np.array([-1]*25 + [1]*25)
X[y==1] += 1

# --- 5-fold CV + GridSearch ---
kfold = skm.KFold(5, random_state=0, shuffle=True)
grid = skm.GridSearchCV(
    SVC(C=10, kernel='linear'),
    {'C': [0.001, 0.01, 0.1, 1, 5, 10, 100]},
    refit=True, cv=kfold, scoring='accuracy'
)
grid.fit(X, y)
print(f"最佳 C: {grid.best_params_['C']}")
print(f"各 C 的 CV 準確率: {grid.cv_results_['mean_test_score']}")

# --- 測試集評估 ---
X_test = rng.standard_normal((20, 2))
y_test = np.array([-1]*10 + [1]*10)
X_test[y_test==1] += 1

best_ = grid.best_estimator_
y_test_hat = best_.predict(X_test)
print(f"\n測試集混亂矩陣 (C={grid.best_params_['C']}):")
print(confusion_table(y_test_hat, y_test))

# --- 對照：C=0.001 ---
svm_weak = SVC(C=0.001, kernel='linear').fit(X, y)
y_weak = svm_weak.predict(X_test)
print(f"\n測試集混亂矩陣 (C=0.001):")
print(confusion_table(y_weak, y_test))

線性可分資料的極端案例

當類別剛好線性可分時，設定極大 \(C\)（如 \(10^5\)）會產生「硬邊界」— 僅 3 個支援向量定義整個邊界。試試看：

# 線性可分案例 — 對比 C=1e5 vs C=0.1
import numpy as np
import matplotlib
matplotlib.use('Agg')
from matplotlib.pyplot import subplots, cm, savefig
from sklearn.svm import SVC
from ISLP import confusion_table
from ISLP.svm import plot as plot_svm

# --- 讓資料線性可分 ---
rng = np.random.default_rng(1)
X = rng.standard_normal((50, 2))
y = np.array([-1]*25 + [1]*25)
X[y==1] += 1
X[y==1] += 1.9  # 進一步分離

# --- C=1e5（硬邊界）---
svm_hard = SVC(C=1e5, kernel='linear').fit(X, y)
y_hard = svm_hard.predict(X)
print(f"C=1e5 訓練錯誤數: {(y_hard != y).sum()}")
print(confusion_table(y_hard, y))

fig, ax = subplots(figsize=(8,8))
plot_svm(X, y, svm_hard, ax=ax)
ax.set_title("SVC (C=1e5, 硬邊界)")
savefig('/tmp/svm_hard_margin.png', dpi=100)

# --- C=0.1（軟邊界）---
svm_soft = SVC(C=0.1, kernel='linear').fit(X, y)
y_soft = svm_soft.predict(X)
print(f"\nC=0.1 訓練錯誤數: {(y_soft != y).sum()}")
print(confusion_table(y_soft, y))

fig, ax = subplots(figsize=(8,8))
plot_svm(X, y, svm_soft, ax=ax)
ax.set_title("SVC (C=0.1, 軟邊界)")
savefig('/tmp/svm_soft_margin.png', dpi=100)
print("\n完成 ✓")

9.6.2 支援向量機（非線性核心）

現在我們使用 RBF（徑向基底函數）核心來處理非線性決策邊界。核心的概念類似「把資料投影到更高的維度，在那裡找到線性的分割面，再投影回來」——有點像摺紙：平面上無法分開的兩色點，把紙摺起來（增加第三維）就可以一刀切開了。

# 9.6.2 SVM — RBF 核心處理非線性邊界
import numpy as np
import matplotlib
matplotlib.use('Agg')
from matplotlib.pyplot import subplots, cm, savefig
import sklearn.model_selection as skm
from sklearn.svm import SVC
from ISLP import confusion_table
from ISLP.svm import plot as plot_svm

# --- 生成非線性邊界資料 ---
rng = np.random.default_rng(2)
X = rng.standard_normal((200, 2))
X[:100] += 2
X[100:150] -= 2
y = np.array([1]*150 + [2]*50)

# --- 分割訓練/測試 ---
(X_train, X_test,
 y_train, y_test) = skm.train_test_split(
    X, y, test_size=0.5, random_state=0)

# --- RBF SVM, gamma=1, C=1 ---
svm_rbf = SVC(kernel="rbf", gamma=1, C=1)
svm_rbf.fit(X_train, y_train)

fig, ax = subplots(figsize=(8,8))
plot_svm(X_train, y_train, svm_rbf, ax=ax)
ax.set_title("RBF SVM (gamma=1, C=1)")
savefig('/tmp/svm_rbf_g1_c1.png', dpi=100)

y_pred = svm_rbf.predict(X_train)
print(f"訓練錯誤數 (gamma=1, C=1): {(y_pred != y_train).sum()}/100")

# --- C=1e5，更複雜邊界 ---
svm_rbf_highC = SVC(kernel="rbf", gamma=1, C=1e5)
svm_rbf_highC.fit(X_train, y_train)

fig, ax = subplots(figsize=(8,8))
plot_svm(X_train, y_train, svm_rbf_highC, ax=ax)
ax.set_title("RBF SVM (gamma=1, C=1e5)")
savefig('/tmp/svm_rbf_g1_c1e5.png', dpi=100)

y_pred_hc = svm_rbf_highC.predict(X_train)
print(f"訓練錯誤數 (gamma=1, C=1e5): {(y_pred_hc != y_train).sum()}/100")
print("完成 ✓")

網格搜索選擇 \(\gamma\) 與 \(C\)

# RBF SVM — 5-fold CV 網格搜索 (gamma, C)
import numpy as np
import matplotlib
matplotlib.use('Agg')
from matplotlib.pyplot import subplots, savefig
import sklearn.model_selection as skm
from sklearn.svm import SVC
from ISLP import confusion_table
from ISLP.svm import plot as plot_svm

# --- 重建資料並分割 ---
rng = np.random.default_rng(2)
X = rng.standard_normal((200, 2))
X[:100] += 2; X[100:150] -= 2
y = np.array([1]*150 + [2]*50)
(X_train, X_test, y_train, y_test) = skm.train_test_split(
    X, y, test_size=0.5, random_state=0)

# --- 網格搜索 ---
kfold = skm.KFold(5, random_state=0, shuffle=True)
grid = skm.GridSearchCV(
    SVC(kernel="rbf"),
    {'C': [0.1, 1, 10, 100, 1000],
     'gamma': [0.5, 1, 2, 3, 4]},
    refit=True, cv=kfold, scoring='accuracy'
)
grid.fit(X_train, y_train)
print(f"最佳參數: {grid.best_params_}")

# --- 最佳模型預測 ---
best_svm = grid.best_estimator_
y_hat_test = best_svm.predict(X_test)
print(f"\n測試集混亂矩陣:\n{confusion_table(y_hat_test, y_test)}")

er = (y_hat_test != y_test).sum()
print(f"測試錯誤: {er}/{len(y_test)} = {er/len(y_test)*100:.1f}%")

# --- 繪製最佳模型邊界 ---
fig, ax = subplots(figsize=(8,8))
plot_svm(X_train, y_train, best_svm, ax=ax)
ax.set_title(f"Best RBF SVM: {grid.best_params_}")
savefig('/tmp/svm_rbf_best.png', dpi=100)
print("完成 ✓")

9.6.3 ROC 曲線

SVM 的 decision_function() 回傳每個觀測值的「擬合值」（fitted value）——正負號決定類別，數值大小反映信心程度。把分類閾值從 0 調整到不同值，我們就可以繪製 ROC 曲線來權衡真陽率和偽陽率。

# 9.6.3 ROC 曲線 — 訓練 vs 測試
import numpy as np
import matplotlib
matplotlib.use('Agg')
from matplotlib.pyplot import subplots, savefig
import sklearn.model_selection as skm
from sklearn.svm import SVC
from sklearn.metrics import RocCurveDisplay

# --- 重建資料與最佳模型 ---
rng = np.random.default_rng(2)
X = rng.standard_normal((200, 2))
X[:100] += 2; X[100:150] -= 2
y = np.array([1]*150 + [2]*50)
(X_train, X_test, y_train, y_test) = skm.train_test_split(
    X, y, test_size=0.5, random_state=0)

best_svm = SVC(kernel="rbf", C=100, gamma=1)
best_svm.fit(X_train, y_train)

# --- ROC: 訓練集（紅色）---
fig, ax = subplots(figsize=(8,8))
RocCurveDisplay.from_estimator(best_svm, X_train, y_train,
    name='Training (CV-tuned)', color='r', ax=ax)

# --- 彈性模型 γ=50 ---
svm_flex = SVC(kernel="rbf", gamma=50, C=1)
svm_flex.fit(X_train, y_train)
RocCurveDisplay.from_estimator(svm_flex, X_train, y_train,
    name='Training gamma=50', color='orange', ax=ax)

# --- ROC: 測試集（藍色）---
RocCurveDisplay.from_estimator(svm_flex, X_test, y_test,
    name='Test gamma=50', color='b', ax=ax)
RocCurveDisplay.from_estimator(best_svm, X_test, y_test,
    name='Test (CV-tuned)', color='c', ax=ax)

ax.set_title("ROC Curves: Training vs Test")
savefig('/tmp/svm_roc.png', dpi=100)
print("ROC 曲線繪製完成 ✓")

9.6.4 多類別 SVM

當類別超過兩個時，SVC 使用一對一（one-versus-one, ovo）或一對多（one-versus-rest, ovr）策略。我們用模擬的三類別資料展示：

# 9.6.4 多類別 SVM（ovo 策略）
import numpy as np
import matplotlib
matplotlib.use('Agg')
from matplotlib.pyplot import subplots, cm, savefig
from sklearn.svm import SVC
from ISLP.svm import plot as plot_svm

# --- 擴展為三類別 ---
rng = np.random.default_rng(123)
X = rng.standard_normal((200, 2))
X[:100] += 2
X[100:150] -= 2
y = np.array([1]*150 + [2]*50)

# 加入第三類
X = np.vstack([X, rng.standard_normal((50, 2))])
y = np.hstack([y, [0]*50])
X[y==0, 1] += 2

# --- 擬合 RBF SVM，ovo ---
svm_rbf_3 = SVC(kernel="rbf", C=10, gamma=1,
                decision_function_shape='ovo')
svm_rbf_3.fit(X, y)

fig, ax = subplots(figsize=(8,8))
plot_svm(X, y, svm_rbf_3, scatter_cmap=cm.tab10, ax=ax)
ax.set_title("3-Class SVM (RBF, ovo)")
savefig('/tmp/svm_multiclass.png', dpi=100)
print(f"類別數: {len(np.unique(y))}, 支援向量數: {svm_rbf_3.n_support_.sum()}")
print("完成 ✓")

9.6.5 基因表達資料實戰

Khan 資料集：63 個訓練樣本 + 20 個測試樣本，每個樣本有 2,308 個基因表達值。目標是預測 四種兒童腫瘤類型。特徵數 (2,308) ≫ 樣本數 (63)——這正是 SVM 的拿手好戲。

# 9.6.5 Khan 基因表達資料 — 線性 SVM
import numpy as np
from sklearn.svm import SVC
from ISLP import load_data, confusion_table

# --- 載入資料 ---
Khan = load_data('Khan')
X_train = Khan['xtrain']
y_train = Khan['ytrain'].values.ravel() if hasattr(Khan['ytrain'], 'values') else Khan['ytrain']
X_test = Khan['xtest']
y_test = Khan['ytest'].values.ravel() if hasattr(Khan['ytest'], 'values') else Khan['ytest']

print(f"訓練集: {X_train.shape} (n={X_train.shape[0]}, p={X_train.shape[1]})")
print(f"測試集: {X_test.shape} (n={X_test.shape[0]}, p={X_test.shape[1]})")

# --- 線性 SVM（高維度不需要 RBF）---
khan_linear = SVC(kernel='linear', C=10)
khan_linear.fit(X_train, y_train)

# --- 訓練集混淆矩陣 ---
y_tr_pred = khan_linear.predict(X_train)
print(f"\n訓練集混淆矩陣:")
print(confusion_table(y_tr_pred, y_train))

# --- 測試集混淆矩陣 ---
y_te_pred = khan_linear.predict(X_test)
print(f"\n測試集混淆矩陣:")
print(confusion_table(y_te_pred, y_test))

test_err = (y_te_pred != y_test).sum()
print(f"\n測試錯誤數: {test_err}/{len(y_test)}")
print(f"測試準確率: {(len(y_test)-test_err)/len(y_test)*100:.1f}%")

訓練集零錯誤在 \(p \gg n\) 的情境下並不意外——2,308 個特徵為 63 個點找到完全分離的超平面是輕而易舉的。真正有價值的是測試集準確率 90%，證明了即使在高維度下，SVM 仍然能夠有效泛化。

📊 C 參數的影響總覽

🔧 核心函數比較

🧠 關鍵重點回顧

1. SVC() 的 C 參數控制對邊界違規的容忍度：C 大 → 窄邊界、少支援向量、低偏誤；C 小 → 寬邊界、多支援向量、更平滑。
2. RBF 核心透過 \(\gamma\) 控制局部影響範圍，配合 C 形成雙參數調優空間。Small \(\gamma\) + moderate C = 平滑、可泛化的邊界。
3. 多類別策略：ovo（一對一投票）是 SVM 的預設策略，對於 \(K\) 類別建立 \(K(K-1)/2\) 個二元分類器。
4. 高維度 (\(p > n\)) 優勢：線性 SVM 在基因表達等微陣列資料上表現出色，不需要非線性核心。
5. ROC 曲線不依賴固定閾值，而是展示所有可能閾值下的 TPR-FPR 權衡，是比較分類器的黃金標準。