Python calculates the evaluation index of the classification problem (accuracy, precision, recall and F1 value, Kappa index)

Commonly used review indicators for classification problems in machine learning include: accuracy, precision, recall, and F1 value, as well as kappa indicators.

It is very troublesome to find their calculation code every time you change the package, so here is a function defined at once to directly calculate all the evaluation indicators.

It is very convenient to get the above index value every time you input the predicted value and real value.

The formulas for calculating these metrics are as follows:

Kappa indicator:

  

---

Python calculation code

The following is the definition function: (mainly with the help of the sklearn library)

#导入数据分析常用包
import numpy as np 
import pandas as pd 

from sklearn.metrics import confusion_matrix
from sklearn.metrics import classification_report
from sklearn.metrics import cohen_kappa_score

def evaluation(y_test, y_predict):
    accuracy=classification_report(y_test, y_predict,output_dict=True)['accuracy']
    s=classification_report(y_test, y_predict,output_dict=True)['weighted avg']
    precision=s['precision']
    recall=s['recall']
    f1_score=s['f1-score']
    #kappa=cohen_kappa_score(y_test, y_predict)
    return accuracy,precision,recall,f1_score #, kappa

This function has only two parameters, the real value and the predicted value, and all the indicators above can be calculated by putting them in. The return value of the function is accuracy, precision, recall, f1_score #, kappa.

I commented out kappa, just take it out if you want to use it.

---

Drawing display:

For example, we have prepared the feature variable X and the response variable y .

The test set and training set are divided as follows:

#划分训练集和测试集
from sklearn.model_selection import train_test_split
X_train,X_test,y_train,y_test=train_test_split(X,y,stratify=y,test_size=0.2,random_state=0)

Normalize it: 

#数据标准化
from sklearn.preprocessing import StandardScaler
scaler = StandardScaler()
scaler.fit(X_train)
X_train_s = scaler.transform(X_train)
X_test_s = scaler.transform(X_test)
print('训练数据形状：')
print(X_train_s.shape,y_train.shape)
print('验证集数据形状：')
print(X_test_s.shape,y_test.shape)

Compare ten machine learning models:

from sklearn.linear_model import LogisticRegression
from sklearn.discriminant_analysis import LinearDiscriminantAnalysis
from sklearn.neighbors import KNeighborsClassifier
from sklearn.tree import DecisionTreeClassifier
from sklearn.ensemble import RandomForestClassifier
from sklearn.ensemble import GradientBoostingClassifier
from xgboost.sklearn import XGBClassifier
from lightgbm import LGBMClassifier
from sklearn.svm import SVC
from sklearn.neural_network import MLPClassifier

 Instantiate the classifier:

#逻辑回归
model1 =  LogisticRegression(C=1e10,max_iter=10000)

#线性判别分析
model2 = LinearDiscriminantAnalysis()

#K近邻
model3 = KNeighborsClassifier(n_neighbors=10)

#决策树
model4 = DecisionTreeClassifier(random_state=77)

#随机森林
model5= RandomForestClassifier(n_estimators=1000,  max_features='sqrt',random_state=10)

#梯度提升
model6 = GradientBoostingClassifier(random_state=123)

#极端梯度提升
model7 =  XGBClassifier(use_label_encoder=False,eval_metric=['logloss','auc','error'],
                        objective='multi:softmax',random_state=0)
#轻量梯度提升
model8 = LGBMClassifier(objective='multiclass',num_class=3,random_state=1)

#支持向量机
model9 = SVC(kernel="rbf", random_state=77)

#神经网络
model10 = MLPClassifier(hidden_layer_sizes=(16,8), random_state=77, max_iter=10000)

model_list=[model1,model2,model3,model4,model5,model6,model7,model8,model9,model10]
model_name=['逻辑回归','线性判别','K近邻','决策树','随机森林','梯度提升','极端梯度提升','轻量梯度提升','支持向量机','神经网络']

 Calculate the evaluation index: use the df_eval data frame to calculate the evaluation index value

df_eval=pd.DataFrame(columns=['Accuracy','Precision','Recall','F1_score'])
for i in range(10):
    model_C=model_list[i]
    name=model_name[i]
    model_C.fit(X_train_s, y_train)
    pred=model_C.predict(X_test_s)
    #s=classification_report(y_test, pred)
    s=evaluation(y_test,pred)
    df_eval.loc[name,:]=list(s)

 Check

df_eval

 

 

 Draw the corresponding histogram:

import matplotlib.pyplot as plt 
plt.rcParams['font.sans-serif'] = ['KaiTi']  #中文
plt.rcParams['axes.unicode_minus'] = False   #负号

bar_width = 0.4
colors=['c', 'b', 'g', 'tomato', 'm', 'y', 'lime', 'k','orange','pink','grey','tan']
fig, ax = plt.subplots(2,2,figsize=(10,8),dpi=128)
for i,col in enumerate(df_eval.columns):
    n=int(str('22')+str(i+1))
    plt.subplot(n)
    df_col=df_eval[col]
    m =np.arange(len(df_col))
    plt.bar(x=m,height=df_col.to_numpy(),width=bar_width,color=colors)
    
    #plt.xlabel('Methods',fontsize=12)
    names=df_col.index
    plt.xticks(range(len(df_col)),names,fontsize=10)
    plt.xticks(rotation=40)
    plt.ylabel(col,fontsize=14)
    
plt.tight_layout()
#plt.savefig('柱状图.jpg',dpi=512)
plt.show()

 The four evaluation indicators are compared with the corresponding ten models.

This custom calculation classification evaluation index function is still very convenient, and can also be used in cross-validation to comprehensively evaluate the prediction quality of the model.