近年来，国内的电信诈骗案件呈愈演愈烈之势，本文以某省电信公司简化版本的防诈骗模型为案例，利用python机器学习工具，使用随机森林算法，从数据处理、特征工程、到反诈骗模型的模型的构建及评估等完整流程进行一个简单的记录和介绍。
流程图
环境设置、模块加载
# coding: utf-8
import os
import numpy as np
import pandas as pd
from sklearn.ensemble import IsolationForest
from sklearn.model_selection import train_test_split
from sklearn.ensemble import RandomForestClassifier
from sklearn.metrics import confusion_matrix
from sklearn.externals import joblib
from sklearn import metrics
from scipy import stats
import time
from datetime import datetime
import warnings
warnings.filterwarnings("ignore")
os.chdir('home//zj//python//python3.6.9//bin//python3')
123456789101112131415161718
数据加载
自定义工作目录，并加载样本数据
def read_file(filepath):
os.chdir(os.path.dirname(filepath))
return pd.read_csv(os.path.basename(filepath),encoding='utf-8')
file_pos = "E:\\工作文件\\***\\防诈骗识别\\data_train.csv"
data_pos = read_file(file_pos)
123456
特征重命名
data_pos.columns = ['BIL_ACCS_NBR','ASSET_ROW_ID','CCUST_ROW_ID','LATN_ID','TOTAL_CNT', 'TOTAL_DURATION','ZJ_CNT','ZJ_TOTAL_DURATION','TOTAL_ROAM_CNT','ZJ_ROAM_CNT','ZJ_LOCAL_CNT','ZJ_ROAM_DURATION','ZJ_LOCAL_DURATION','ZJ_LONG_CNT','BJ_LOCAL_CNT','WORK_TIME_TH_TT_CNT','FREE_TIME_TH_TT_CNT','NIGHT_TIME_TH_TT_CNT','DURATION_TP_1','DURATION_TP_2','DURATION_TP_3','DURATION_TP_4','DURATION_TP_5','DURATION_TP_6','DURATION_TP_7','DURATION_TP_8', 'DURATION_TP_9','TOTAL_DIS_BJ_NUM','DIS_BJ_NUM','DIS_OPP_HOME_NUM','OPP_HOME_NUM','MSC_NUM','DIS_MSC_NUM','ZJ_AVG_DURATION','TOTAL_ROAM_CNT_RATE','ZJ_DURATION_RATE','ZJ_CNT_RATE','ZJ_ROAM_DURATION_RATE','ZJ_ROAM_CNT_RATE','DURATION_RATIO_0_15','DURATION_RATIO_15_30', 'DURATION_RATIO_30_45','DURATION_RATIO_45_60','DURATION_RATIO_60_300','DUR_30_CNT_RATE', 'DUR_60_CNT_RATE','DUR_90_CNT_RATE','DUR_120_CNT_RATE','DUR_180_CNT_RATE','DUR_BIGGER_180_CNT_RATE','DIS_BJ_NUM_RATE','TOTAL_DIS_BJ_NUM_RATE','CALLING_REGION_DISTRI_LEVEL','ACT_DAY','ACT_DAY_RATE','WEEK_DIS_BJ_NUM','YY_WORK_DAY_OIDD_23_NUM','IS_GJMY','ZJ_DURATION_0_15_CNT','ZJ_DURATION_15_30_CNT','ZJ_DURATION_30_60_CNT','ZJ_DURATION_RATIO_0_15','ZJ_DURATION_RATIO_15_30','ZJ_DURATION_RATIO_30_60','H_MAX_CNT','H_MAX_CIRCLE','INNER_MONTH','MIX_CDSC_FLG','CPRD_NAME','AMT','CUST_ASSET_CNT','CUST_TELE_CNT','CUST_C_CNT','ALL_LL_USE', 'MY_LL_USE','MY_LL_ZB','ALL_LL_DUR','MY_LL_DUR','MY_DUR_ZB','AGE','GENDER','CUST_TYPE_GRADE_NAME','ISP','TERM_PRICE','SALES_CHANNEL_LVL2_NAME','CORP_USER_NAME','TOTOL_7_CNT', 'TOTOL_7_DUR','TOTOL_7_ZJ_DUR','TOTOL_7_ZJ_CNT','TOTOL_7_ZJ_D_CNT','TOTOL_7_BJ_D_DUR', 'TOTOL_7_JZGS_CNT','WEEK_CNT','WEEK_DUR','ZB_WS','COUPLE_NUMBER','TIME_COUPLE_NUMBER','ZJ_0912','HB_0912','ZJ_1417','HB_1417','CHG_CELLS','ZHANBI','ETL_DT','IS_HARASS'] 12345678
数据查看
数据表行/列
data_pos.shape 1
可以看出，正样本数据只有3436，负样本较多，属于极度不平衡样本数据
数据预处理
无意义字段删除
data_pos_1 = data_pos.drop([
'BIL_ACCS_NBR',
'ASSET_ROW_ID',
'CCUST_ROW_ID',
'LATN_ID',
'CPRD_NAME',
'ISP',
'AGE',
'CUST_TYPE_GRADE_NAME',
'ETL_DT',
'WEEK_DIS_BJ_NUM',
'TOTOL_7_ZJ_D_CNT',
'TOTOL_7_JZGS_CNT',
'INNER_MONTH'
],axis = 1)
123456789101112131415
正负样本规模
data_pos.IS_HARASS.value_counts() 1
TERM_PRICE 进行分箱处理
data_pos_1['TERM_PRICE'] = data_pos_1['TERM_PRICE'].apply(lambda x: np.where(x > 5000, '>5000',
np.where(x>3000, '(3000,5000]',
np.where(x>2000, '(2000,3000]',
np.where(x>1000, '(1000,2000]',
np.where(x>0, '(0,1000]', '未识别'))))))
123456
字段填充及转化
将类别型变量空值及极小规模类别做替换
data_pos_1.TERM_PRICE.value_counts() data_pos_1.MIX_CDSC_FLG.value_counts() data_pos_1.CORP_USER_NAME.value_counts() data_pos_1.SALES_CHANNEL_LVL2_NAME.value_counts() 1234
#依次处理TERM_PRICE、MIX_CDSC_FLG、CORP_USER_NAME、SALES_CHANNEL_LVL2_NAME
def CHANGE_SALES_CHANNEL_LVL2_NAME(data):
if data in ['社会渠道','实体渠道','电子渠道','直销渠道']:
return data
else:
return '未识别'
data_pos_1['SALES_CHANNEL_LVL2_NAME'] = data_pos_1.SALES_CHANNEL_LVL2_NAME.apply(CHANGE_SALES_CHANNEL_LVL2_NAME)
12345678
缺失值处理
##缺失值统计
def na_count(data):
data_count = data.count()
na_count = len(data) - data_count
na_rate = na_count/len(data)
na_result = pd.concat([data_count,na_count,na_rate],axis = 1)
return na_result
na_count = na_count(data_pos_1)
na_count
12345678910
拆分字段
字段按照连续、类别拆分
def category_continuous_resolution(data,variable_category):
for key in list(data.columns):
if key not in variable_category:
variable_continuous.append(key)
else:
continue
return variable_continuous
#字段按照类型拆分
variable_category = ['MIX_CDSC_FLG','GENDER','TERM_PRICE','SALES_CHANNEL_LVL2_NAME','CORP_USER_NAME']
variable_continuous = []
variable_continuous = category_continuous_resolution(data_pos_1,variable_category)
1234567891011121314
字段类型转化
def feture_type_change(data,variable_category):
'''
字段类型转化
'''
for col_key in list(data.columns):
if col_key in variable_category:
data[col_key] = data[col_key].astype(eval('object'), copy=False)
else:
data[col_key] = data[col_key].astype(eval('float'), copy=False)
return data
data_pos_2 = feture_type_change(data_pos_1,variable_category)
123456789101112
缺失值填充
def na_fill(data,col_name_1,col_name_2):
'''
缺失值填充
'''
for col_key in list(data.columns):
if col_key in col_name_1:
data[col_key] = data[col_key].fillna(value = '未识别')
elif col_key in col_name_2:
data[col_key] = data[col_key].fillna(data[col_key].mean())
else:
data[col_key] = data[col_key].fillna(value = 0)
return data
#缺失值填充
col_name_1 = variable_category
col_name_2 = []
data_pos_3 = na_fill(data_pos_2,col_name_1,col_name_2)
1234567891011121314151617
类别变量one_hot处理
##one_hot
def data_deliver(data,variable_category):
'''
ont_hot衍生
'''
for col_key in list(data.columns):
if col_key in variable_category:
temp_one_hot_code = pd.get_dummies(data[col_key],prefix = col_key)
data = pd.concat([data,temp_one_hot_code],axis = 1)
del data[col_key]
else:
continue
return data
data_pos_4 = data_deliver(data_pos_3,variable_category)
123456789101112131415
特征工程
相关性分析
def max_corr_feture_droped(train_data,variable_continuous,k):
'''
相关性分析
'''
table_col = train_data.columns
table_col_list = table_col.values.tolist()
all_lines = len(train_data)
train_data_number = train_data[variable_continuous]
###连续型变量的处理过程：数据的标准化
from numpy import array
from sklearn import preprocessing
def normalization(data,method,feature_range=(0,1)):
if method=='MaxMin':
train_data_scale=data.apply(lambda x: (x - np.min(x)) / (np.max(x) - np.min(x)))
return train_data_scale
if method=='z_score':
train_data_scale=data.apply(lambda x: (x - np.mean(x)) / (np.std(x)))
return train_data_scale
train_data_scale = normalization(train_data_number,method=scale_method)
# 输出各个变量之间的相关性报告
def data_corr_analysis(raw_data, sigmod = k):
corr_data = raw_data.corr()
for i in range(len(corr_data)):
for j in range(len(corr_data)):
if j == i:
corr_data.iloc[i, j] = 0
x, y, corr_xishu = [], [], []
for i in list(corr_data.index):
for j in list(corr_data.columns):
if abs(corr_data.loc[i, j]) > sigmod: # 保留相关性系数绝对值大于阈值的属性
x.append(i)
y.append(j)
corr_xishu.append(corr_data.loc[i, j])
z = [[x[i], y[i], corr_xishu[i]] for i in range(len(x))]
high_corr = pd.DataFrame(z, columns=['VAR1','VAR2','CORR_XISHU'])
return high_corr
high_corr_data = data_corr_analysis(train_data_number, sigmod=k)
def data_corr_choice(data,train_data_scale,high_corr_data):
high_corr_data_1=[]
target_var=pd.DataFrame(data.loc[:,target_col])
for i in range(high_corr_data.shape[0]):
for j in range(high_corr_data.shape[1]-1):
d1=pd.DataFrame(train_data_scale.loc[:,high_corr_data.iloc[i,j]])
data1=pd.concat([d1, data.loc[:,target_col]], axis=1, join='inner')
corr_data = data1.corr()
high_corr_data_1.append(corr_data.iloc[0,-1]) #输出的为各个变量与目标变量之间的相关关系
high_corr_data_2=np.array(high_corr_data_1).reshape(high_corr_data.shape[0],high_corr_data.shape[1]-1)
high_corr_data_2=pd.DataFrame(high_corr_data_2,columns=high_corr_data.columns[:-1])
del_var_cor=[]
for i in range(high_corr_data_2.shape[0]):
if abs(high_corr_data_2.iloc[i,0])>=abs(high_corr_data_2.iloc[i,1]):
del_var_cor.append(high_corr_data.iloc[i,1])
else:
del_var_cor.append(high_corr_data.iloc[i,0])
train_data_number_2.drop(del_var_cor,axis=1,inplace = True) #将强相关的变量直接剔除
return set(high_corr_data_1),set(del_var_cor),train_data_number_2
train_data_number_2 = pd.concat([train_data[variable_continuous],train_data[target_col]],axis=1)
high_corr_data_1,del_var_cor,train_data_scale = data_corr_choice(train_data_number_2,train_data_scale,high_corr_data)
train_data2 = train_data[:]
train_data2.drop(set(del_var_cor),axis=1,inplace = True)
return train_data2,del_var_cor
#相关性分析，去除高相关变量
scale_method = 'MaxMin'
target_col = 'IS_HARASS'
data_pos_5,del_var_cor = max_corr_feture_droped(data_pos_4,variable_continuous,k=0.8)
del_var_cor #删除的variable查看
1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768
特征重要性分析
def data_sample(data, target_col, smp):
'''
数据平衡
'''
data_1 = data[data[target_col] == 1].sample(frac=1)
data_0 = data[data[target_col] == 0].sample(n=len(data_1)*smp)
# data_1 = data_1.sample(len(data_2)*smp)
data = pd.concat([data_1, data_0]).reset_index()
return data
123456789
def train_test_spl(data):
'''
训练数据、测试数据切分
'''
X_train, X_test, y_train, y_test = train_test_split(
data[ipt_col], data[target_col], test_size=0.2, random_state=42)
return X_train, X_test, y_train, y_test
1234567
定义特征重要性分析函数，并循环遍历获取最佳抽样比例
def feture_extracted(train_data, alpha):
'''
维度重要性判断
'''
global ipt_col
ipt_col= list(train_data.columns)
ipt_col.remove(target_col)
sample_present = [1,5] # 定义抽样比例
f1_score_list = []
model_dict = {}
for i in sample_present:
try:
train_data = data_sample(train_data, target_col, smp=i)
except ValueError:
break
X_train, X_test, y_train, y_test = train_test_spl(train_data)
# 开始RF选取特征
model = RandomForestClassifier()
model = model.fit(X_train, y_train)
model_pred = model.predict(X_test)
f1_score = metrics.f1_score(y_test, model_pred)
f1_score_list.append(f1_score)
model_dict[i] = model
max_f1_index = f1_score_list.index(max(f1_score_list))
print('最优的抽样比例是：1:',sample_present[max_f1_index])
d = dict(zip(ipt_col, [float('%.3f' %i) for i in model_dict[sample_present[max_f1_index]].feature_importances_]))
f = zip(d.values(), d.keys())
importance_df = pd.DataFrame(sorted(f, reverse=True), columns=['importance', 'feture_name'])
list_imp = np.cumsum(importance_df['importance']).tolist()
for i, j in enumerate(list_imp):
if j >= alpha:
break
print('大于alpha的特征及重要性如下：\n',importance_df.iloc[0:i+1, :])
print('其特征如下：')
feture_selected = importance_df.iloc[0:i+1, 1].tolist()
print(feture_selected)
return feture_selected
#重要性检验，选择重要变量
data_pos_5_feture = feture_extracted(data_pos_5, alpha = 0.9)
12345678910111213141516171819202122232425262728293031323334353637383940
模型训练
数据平衡
data_pos_6 = data_sample(data_pos_5, target_col, smp = 3) 1
正负样本拆分
def model_select(data, rf_feture, target_col ,test_size):
'''
正负样本拆分
'''
X_train, X_test, y_train, y_test = train_test_split(
data[rf_feture], data[target_col], test_size=test_size, random_state=42)
return X_train, X_test, y_train, y_test
#拆分比例7:3
X_train, X_test, y_train, y_test = model_select(data_pos_6,data_pos_5_feture,target_col,test_size=0.3)
12345678910
定义模型函数
RF两个主要参数说明:
- min_samples_split：当对一个内部结点划分时，要求该结点上的最小样本数，默认为2；
- min_samples_leaf：设置叶子结点上的最小样本数，默认为1。当尝试划分一个结点时，只有划分后其左右分支上的样本个数不小于该参数指定的值时，才考虑将该结点划分，换句话说，当叶子结点上的样本数小于该参数指定的值时，则该叶子节点及其兄弟节点将被剪枝。在样本数据量较大时，可以考虑增大该值，提前结束树的生长。
def model_train(x_train, y_train, model):
'''
算法模型，默认为RF
'''
if model == 'RF':
res_model = RandomForestClassifier(min_samples_split = 50,min_samples_leaf = 50)
res_model = res_model.fit(x_train, y_train)
feature_importances = res_model.feature_importances_[1]
if model == 'LR':
res_model = LogisticRegression()
res_model = res_model.fit(x_train, y_train)
list_feature_importances = [x for x in res_model.coef_[0]]
list_index = list(x_train.columns)
feature_importances = pd.DataFrame(list_feature_importances, list_index)
else:
pass
return res_model, feature_importances
#训练模型
rf_model, feature_importances = model_train(X_train, y_train, model='RF') #也可以选择使用LR
123456789101112131415161718192021
模型验证
def model_predict(res_model, input_data, alpha ):
# 模型预测
# input_data: 输入新的无目标变量的数据
data_proba = pd.DataFrame(res_model.predict_proba(input_data).round(4))
data_proba.columns = ['neg', 'pos']
data_proba['res'] = data_proba['pos'].apply(lambda x: np.where(x >= alpha, 1, 0)) #将>0.5输出为正调整为1
return data_proba
def model_evaluate(y_true, y_pred):
y_true = np.array(y_true)
y_true.shape = (len(y_true),)
y_pred = np.array(y_pred)
y_pred.shape = (len(y_pred),)
print(metrics.classification_report(y_true, y_pred))
1234567891011121314
data_pos_6 = data_sample(data_pos_5, target_col, smp = 50)
X_train, X_test, y_train, y_test = model_select(data_pos_6,data_pos_5_feture,target_col,test_size=0.5)
Precision = []
Recall = []
for alpha in np.arange(0, 1, 0.1):
y_pred_rf = model_predict(rf_model, X_test, alpha = alpha)
cnf_matrix = confusion_matrix(y_test, y_pred_rf['res'])
Precision.append((cnf_matrix[1,1]/(cnf_matrix[0,1] + cnf_matrix[1,1])).round(4))
Recall.append((cnf_matrix[1,1]/(cnf_matrix[1,0] + cnf_matrix[1,1])).round(4))
score = pd.DataFrame(np.arange(0, 1, 0.1),columns = ['score'])
Precision = pd.DataFrame(Precision,columns = ['Precision'])
Recall = pd.DataFrame(Recall,columns = ['Recall'])
Precision_Recall_F1 = pd.concat([score, Precision, Recall],axis = 1)
Precision_Recall_F1['F1'] = (2 * Precision_Recall_F1['Precision'] * Precision_Recall_F1['Recall'] / (Precision_Recall_F1['Precision'] + Precision_Recall_F1['Recall'])).round(2)
Precision_Recall_F1
12345678910111213141516171819
模型封装保存
start = datetime.now()
joblib.dump(rf_model, 'model.dmp', compress=3)
print("模型保存所用时间： %s 秒" %(datetime.now() - start).seconds)
123
上述案例比较简单，没有过多涉及数据清洗及预处理，包括RF算法也只定义了两个参数，且没有参数的优化过程，感兴趣的可以在此基础上深入一下。
近期有很多朋友通过私信咨询有关Python学习问题。为便于交流，点击蓝色自己加入讨论解答资源基地