模型评估方法

3. 模型评估方法

数据集切分

import numpy as np
import os
%matplotlib inline
import matplotlib
import matplotlib.pyplot as plt
plt.rcParams['axes.labelsize'] = 14
plt.rcParams['xtick.labelsize'] = 12
plt.rcParams['ytick.labelsize'] = 12
import warnings
warnings.filterwarnings('ignore')
np.random.seed(42)

# Mnist数据是图像数据：(28,28,1)的灰度图
from sklearn.datasets import fetch_mldata
mnist = fetch_mldata('MNIST original')
mnist


X.shape
X, y = mnist["data"], mnist["target"]
X.shape
(70000, 784)
y.shape
(70000,)

X_train, X_test, y_train, y_test = X[:60000], X[60000:], y[:60000], y[60000:]

import numpy as np
# 洗牌操作
shuffle_index = np.random.permutation(60000)
X_train, y_train = X_train[shuffle_index], y_train[shuffle_index]

交叉验证

y_train_5 = (y_train==5)
y_test_5 = (y_test==5)

from sklearn.linear_model import SGDClassifier
sgd_clf = SGDClassifier(max_iter=5,random_state=42)
sgd_clf.fit(X_train,y_train_5)


from sklearn.model_selection import cross_val_score
cross_val_score(sgd_clf,X_train,y_train_5,cv=3,scoring='accuracy')
array([0.9502 , 0.96565, 0.96495])

X_train.shape
(60000, 784)
y_train_5.shape
(60000,)

from sklearn.model_selection import StratifiedKFold
from sklearn.base import clone
​
skflods = StratifiedKFold(n_splits=3,random_state=42)
for train_index,test_index in skflods.split(X_train,y_train_5):
    clone_clf = clone(sgd_clf)
    X_train_folds = X_train[train_index]
    y_train_folds = y_train_5[train_index]
    X_test_folds = X_train[test_index]
    y_test_folds = y_train_5[test_index]
    
    clone_clf.fit(X_train_folds,y_train_folds)
    y_pred = clone_clf.predict(X_test_folds)
    n_correct = sum(y_pred == y_test_folds)
    print(n_correct/len(y_pred))
0.9502
0.96565
0.96495

混淆矩阵

from sklearn.model_selection import cross_val_predict
y_train_pred = cross_val_predict(sgd_clf,X_train,y_train_5,cv=3)


from sklearn.metrics import confusion_matrix
confusion_matrix(y_train_5,y_train_pred)
array([[53272,  1307],
       [ 1077,  4344]], dtype=int64)

negative class [[ true negatives , false positives ],
positive class [ false negatives , true positives ]]

true negatives: 53,272个数据被正确的分为非5类别
false positives：1307张被错误的分为5类别
false negatives：1077张错误的分为非5类别
true positives： 4344张被正确的分为5类别

# 准确率： TP/(TP+FP)
# 召回率： TP/(TP+FN)

from sklearn.metrics import precision_score,recall_score
precision_score(y_train_5,y_train_pred)
0.7687135020350381

recall_score(y_train_5,y_train_pred)
0.801328168234643


from sklearn.metrics import f1_score
f1_score(y_train_5,y_train_pred)
0.7846820809248555

评估指标对比分析

阈值对结果的影响

Scikit-Learn不允许直接设置阈值，但它可以得到决策分数，调用其decision_function（）方法，而不是调用分类器的predict（）方法，该方法返回每个实例的分数，然后使用想要的阈值根据这些分数进行预测

y_scores = cross_val_predict(sgd_clf, X_train, y_train_5, cv=3,
                             method="decision_function")
from sklearn.metrics import precision_recall_curve
precisions, recalls, thresholds = precision_recall_curve(y_train_5, y_scores)

def plot_precision_recall_vs_threshold(precisions,recalls,thresholds):
    plt.plot(thresholds,
             precisions[:-1],
            "b--",
            label="Precision")
    
    plt.plot(thresholds,
             recalls[:-1],
            "g-",
            label="Recall")
    plt.xlabel("Threshold",fontsize=16)
    plt.legend(loc="upper left",fontsize=16)
    plt.ylim([0,1])
    
plt.figure(figsize=(8, 4))
plot_precision_recall_vs_threshold(precisions,recalls,thresholds)
plt.xlim([-700000, 700000])
plt.show()

def plot_precision_vs_recall(precisions, recalls):
    plt.plot(recalls, 
             precisions, 
             "b-", 
             linewidth=2)
    
    plt.xlabel("Recall", fontsize=16)
    plt.ylabel("Precision", fontsize=16)
    plt.axis([0, 1, 0, 1])

plt.figure(figsize=(8, 6))
plot_precision_vs_recall(precisions, recalls)
plt.show()

ROC曲线

receiver operating characteristic (ROC) 曲线是二元分类中的常用评估方法

它与精确度/召回曲线非常相似，但ROC曲线不是绘制精确度与召回率，而是绘制true positive rate(TPR) 与false positive rate(FPR)

要绘制ROC曲线，首先需要使用roc_curve（）函数计算各种阈值的TPR和FPR：

TPR = TP / (TP + FN) (Recall)
FPR = FP / (FP + TN)

from sklearn.metrics import roc_curve
fpr, tpr, thresholds = roc_curve(y_train_5, y_scores)

def plot_roc_curve(fpr, tpr, label=None):
    plt.plot(fpr, tpr, linewidth=2, label=label)
    plt.plot([0, 1], [0, 1], 'k--')
    plt.axis([0, 1, 0, 1])
    plt.xlabel('False Positive Rate', fontsize=16)
    plt.ylabel('True Positive Rate', fontsize=16)

plt.figure(figsize=(8, 6))
plot_roc_curve(fpr, tpr)
plt.show()

虚线表示纯随机分类器的ROC曲线; 一个好的分类器尽可能远离该线（朝左上角）。

比较分类器的一种方法是测量曲线下面积（AUC）。完美分类器的ROC AUC等于1，而纯随机分类器的ROC AUC等于0.5。 Scikit-Learn提供了计算ROC AUC的函数：

from sklearn.metrics import roc_auc_score
roc_auc_score(y_train_5, y_scores)
0.9624496555967156