学习宝书使用K近邻算法检测异常操作、检测Rookit和WebShell

版权声明：请多多关注博主哟～ https://blog.csdn.net/qq_37865996/article/details/87711980

学习《Web安全之机器学习》，颇有收获。

1.检测异常操作

因为我的python版本是3.7，和作者有所不同。修改了很多部分后，暂时的KNN.py代码为：

# -*- coding:utf-8 -*-

import sys
import urllib
from urllib.parse import urlparse
import re
from hmmlearn import hmm
import numpy as np
from sklearn.externals import joblib
import HTMLParser
import nltk
import csv
import matplotlib.pyplot as plt
from nltk.probability import FreqDist
from sklearn.feature_extraction.text import CountVectorizer
from sklearn.neighbors import KNeighborsClassifier

from sklearn.metrics import classification_report
from sklearn import metrics

#测试样本数
N=100

"""
    def load_alexa(filename):
    domain_list=[]
    csv_reader = csv.reader(open(filename))
    for row in csv_reader:
    domain=row[1]
    if domain >= MIN_LEN:
    domain_list.append(domain)
    return domain_list
    def domain2ver(domain):
    ver=[]
    for i in range(0,len(domain)):
    ver.append([ord(domain[i])])
    return ver
    #domain_list=load_alexa("../data/top-1m.csv")
    domain_list = load_alexa("../data/top-1000.csv")
    #remodel=train_hmm(domain_list)
    remodel=joblib.load(FILE_MODEL)
    x_3,y_3=test_dga(remodel, "../data/dga-post-tovar-goz-1000.txt")
    x_2,y_2=test_dga(remodel,"../data/dga-cryptolocke-1000.txt")
    x_1,y_1=test_alexa(remodel, "../data/test-top-1000.csv")
    #test_alexa(remodel, "../data/top-1000.csv")
    #%matplotlib inline
    fig,ax=plt.subplots()
    ax.set_xlabel('Domain Length')
    ax.set_ylabel('HMM Score')
    ax.scatter(x_3,y_3,color='b',label="dga_post-tovar-goz")
    ax.scatter(x_2, y_2, color='g', label="dga_cryptolock")
    #ax.scatter(x_1, y_1, color='r', label="alexa")
    ax.legend(loc='right')
    plt.show()
    """

def load_user_cmd(filename):
    cmd_list=[]
    dist_max=[]
    dist_min=[]
    dist=[]
    with open(filename) as f:
        i=0
        x=[]
        for line in f:
            line=line.strip('\n')
            x.append(line)
            dist.append(line)
            i+=1
            if i == 100:
                cmd_list.append(x)
                x=[]
                i=0

    fdist = list(FreqDist(dist).keys())
    dist_max=set(fdist[0:50])
    dist_min = set(fdist[-50:])
    return cmd_list,dist_max,dist_min

def get_user_cmd_feature(user_cmd_list,dist_max,dist_min):
    user_cmd_feature=[]
    for cmd_block in user_cmd_list:
        f1=len(set(cmd_block))
        fdist = list(FreqDist(cmd_block).keys())
        f2=fdist[0:10]
        f3=fdist[-10:]
        f2 = len(set(f2) & set(dist_max))
        f3=len(set(f3)&set(dist_min))
        x=[f1,f2,f3]
        user_cmd_feature.append(x)
    return user_cmd_feature

def get_label(filename,index=0):
    x=[]
    with open(filename) as f:
        for line in f:
            line=line.strip('\n')
            x.append( int(line.split()[index]))
    return x

if __name__ == '__main__':
    user_cmd_list,user_cmd_dist_max,user_cmd_dist_min=load_user_cmd("/Users/zhanglipeng/Library/Python/3.7/lib/python/data/masquerade-data/User3")
    user_cmd_feature=get_user_cmd_feature(user_cmd_list,user_cmd_dist_max,user_cmd_dist_min)
    labels=get_label("/Users/zhanglipeng/Library/Python/3.7/lib/python/data/masquerade-data/label.txt",2)
    y=[0]*50+labels
    
    x_train=user_cmd_feature[0:N]
    y_train=y[0:N]
    
    x_test=user_cmd_feature[N:150]
    y_test=y[N:150]
    
    neigh = KNeighborsClassifier(n_neighbors=3)
    neigh.fit(x_train, y_train)
    y_predict=neigh.predict(x_test)
    
    score=np.mean(y_test==y_predict)*100
    
    #print y
    #print y_train
    print (y_test)
    print (y_predict)
    print (score)
    
    print (classification_report(y_test, y_predict))
    
    print (metrics.confusion_matrix(y_test, y_predict))

报错：================= RESTART: /Users/zhanglipeng/Desktop/KNN.py =================
Traceback (most recent call last):
  File "/Users/zhanglipeng/Desktop/KNN.py", line 114, in <module>
    neigh.fit(x_train, y_train)
  File "/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/sklearn/neighbors/base.py", line 891, in fit
    X, y = check_X_y(X, y, "csr", multi_output=True)
  File "/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/sklearn/utils/validation.py", line 766, in check_X_y
    check_consistent_length(X, y)
  File "/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/sklearn/utils/validation.py", line 235, in check_consistent_length
    " samples: %r" % [int(l) for l in lengths])
ValueError: Found input variables with inconsistent numbers of samples: [100, 50]

这里出错的原因是输入参数变量与样本数不一致。应该是出现在label.txt这个文件上，暂时未知道这个文件的来源。

第二个代码，更换了思路进行异常行为监测，当然最后出的问题是一样的：

# -*- coding:utf-8 -*-

import sys
import urllib
from urllib.parse import urlparse
import re
from hmmlearn import hmm
import numpy as np
from sklearn.externals import joblib
import HTMLParser
import nltk
import csv
import matplotlib.pyplot as plt
from nltk.probability import FreqDist
from sklearn.feature_extraction.text import CountVectorizer
from sklearn.neighbors import KNeighborsClassifier

from sklearn.metrics import classification_report
from sklearn import metrics

#测试样本数
N=90

def load_user_cmd_new(filename):
    cmd_list=[]
    dist=[]
    with open(filename) as f:
        i=0
        x=[]
        for line in f:
            line=line.strip('\n')
            x.append(line)
            dist.append(line)
            i+=1
            if i == 100:
                cmd_list.append(x)
                x=[]
                i=0

    fdist = list(FreqDist(dist).keys())
    return cmd_list,fdist

def load_user_cmd(filename):
    cmd_list=[]
    dist_max=[]
    dist_min=[]
    dist=[]
    with open(filename) as f:
        i=0
        x=[]
        for line in f:
            line=line.strip('\n')
            x.append(line)
            dist.append(line)
            i+=1
            if i == 100:
                cmd_list.append(x)
                x=[]
                i=0

    fdist = list(FreqDist(dist).keys())
    dist_max=set(fdist[0:50])
    dist_min = set(fdist[-50:])
    return cmd_list,dist_max,dist_min

def get_user_cmd_feature(user_cmd_list,dist_max,dist_min):
    user_cmd_feature=[]
    for cmd_block in user_cmd_list:
        f1=len(set(cmd_block))
        fdist = list(FreqDist(cmd_block).keys())
        f2=fdist[0:10]
        f3=fdist[-10:]
        f2 = len(set(f2) & set(dist_max))
        f3=len(set(f3)&set(dist_min))
        x=[f1,f2,f3]
        user_cmd_feature.append(x)
    return user_cmd_feature

def get_user_cmd_feature_new(user_cmd_list,dist):
    user_cmd_feature=[]

    for cmd_list in user_cmd_list:
        v=[0]*len(dist)
        for i in range(0,len(dist)):
            if dist[i] in cmd_list:
                v[i]+=1
        user_cmd_feature.append(v)

    return user_cmd_feature

def get_label(filename,index=0):
    x=[]
    with open(filename) as f:
        for line in f:
            line=line.strip('\n')
            x.append( int(line.split()[index]))
    return x

if __name__ == '__main__':
    user_cmd_list,dist=load_user_cmd_new("/Users/zhanglipeng/Library/Python/3.7/lib/python/data/masquerade-data/User3")
    print ("Dist:(%s)" % dist)
    user_cmd_feature=get_user_cmd_feature_new(user_cmd_list,dist)
    #print  user_cmd_feature
    labels=get_label("/Users/zhanglipeng/Library/Python/3.7/lib/python/data/masquerade-data/label.txt",2)
    y=[0]*50+labels

    x_train=user_cmd_feature[0:N]
    y_train=y[0:N]

    x_test=user_cmd_feature[N:150]
    y_test=y[N:150]

    neigh = KNeighborsClassifier(n_neighbors=3)
    neigh.fit(x_train, y_train)
    y_predict=neigh.predict(x_test)

    #score=np.mean(y_test==y_predict)*100

    #print score

    print  (cross_validation.cross_val_score(neigh, user_cmd_feature, y, n_jobs=-1,cv=10))

2.使用K近邻算法检测Rookit

Rootkit是一种特殊的恶意软件，数据集使用KDD 99数据集，数据展示如下：

代码：

# -*- coding:utf-8 -*-

import re
import matplotlib.pyplot as plt
import os
from sklearn.feature_extraction.text import CountVectorizer
from sklearn import cross_validation
import os
from sklearn.naive_bayes import GaussianNB
from sklearn.neighbors import KNeighborsClassifier


def load_kdd99(filename):
    x=[]
    with open(filename) as f:
        for line in f:
            line=line.strip('\n')
            line=line.split(',')
            x.append(line)
    return x

def get_rootkit2andNormal(x):
    v=[]
    w=[]
    y=[]
    for x1 in x:
        if ( x1[41] in ['rootkit.','normal.'] ) and ( x1[2] == 'telnet' ):
            if x1[41] == 'rootkit.':
                y.append(1)
            else:
                y.append(0)

            x1 = x1[9:21]
            v.append(x1)

    for x1 in v :
        v1=[]
        for x2 in x1:
            v1.append(float(x2))
        w.append(v1)
    return w,y

if __name__ == '__main__':
    v=load_kdd99("/用户/zhanglipeng/资源库/Python/3.7/lib/python/Data/kdd99_corrected")
    x,y=get_rootkit2andNormal(v)
    clf = KNeighborsClassifier(n_neighbors=3)
    print  (cross_validation.cross_val_score(clf, x, y, n_jobs=-1, cv=10))

报错：======== RESTART: /Users/zhanglipeng/Desktop/Web安全与机器学习/KNNRookit.py ========
Traceback (most recent call last):
  File "/Users/zhanglipeng/Desktop/Web安全与机器学习/KNNRookit.py", line 7, in <module>
    from sklearn import cross_validation
  File "/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/sklearn/cross_validation.py", line 35, in <module>
    from .utils.fixes import bincount
ImportError: cannot import name 'bincount' from 'sklearn.utils.fixes' (/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/sklearn/utils/fixes.py)

待处理。

3.使用K近邻算法检测Webshell

使用ADFA-LD数据集，大神的代码是这样的

扫描二维码关注公众号，回复： 5237409 查看本文章

# -*- coding:utf-8 -*-

import re
import matplotlib.pyplot as plt
import os
from sklearn.feature_extraction.text import CountVectorizer
from sklearn import cross_validation
import os
from sklearn.datasets import load_iris
from sklearn import tree
import pydotplus
import numpy as np
from sklearn.neighbors import KNeighborsClassifier


def load_one_flle(filename):
    x=[]
    with open(filename) as f:
        line=f.readline()
        line=line.strip('\n')
    return line

def load_adfa_training_files(rootdir):
    x=[]
    y=[]
    list = os.listdir(rootdir)
    for i in range(0, len(list)):
        path = os.path.join(rootdir, list[i])
        if os.path.isfile(path):
            x.append(load_one_flle(path))
            y.append(0)
    return x,y

def dirlist(path, allfile):
    filelist = os.listdir(path)

    for filename in filelist:
        filepath = os.path.join(path, filename)
        if os.path.isdir(filepath):
            dirlist(filepath, allfile)
        else:
            allfile.append(filepath)
    return allfile

def load_adfa_webshell_files(rootdir):
    x=[]
    y=[]
    allfile=dirlist(rootdir,[])
    for file in allfile:
        if re.match(r"../data/ADFA-LD/Attack_Data_Master/Web_Shell_\d+/UAD-W*",file):
            x.append(load_one_flle(file))
            y.append(1)
    return x,y



if __name__ == '__main__':

    x1,y1=load_adfa_training_files("../data/ADFA-LD/Training_Data_Master/")
    x2,y2=load_adfa_webshell_files("../data/ADFA-LD/Attack_Data_Master/")

    x=x1+x2
    y=y1+y2
    #print x
    vectorizer = CountVectorizer(min_df=1)
    x=vectorizer.fit_transform(x)
    x=x.toarray()
    #print y
    clf = KNeighborsClassifier(n_neighbors=3)
    scores=cross_validation.cross_val_score(clf, x, y, n_jobs=-1, cv=10)
    print (scores)
    print (np.mean(scores))

仍旧出现问题：

======= RESTART: /Users/zhanglipeng/Desktop/Web安全与机器学习/KNNWebshell.py =======
Traceback (most recent call last):
  File "/Users/zhanglipeng/Desktop/Web安全与机器学习/KNNWebshell.py", line 7, in <module>
    from sklearn import cross_validation
  File "/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/sklearn/cross_validation.py", line 35, in <module>
    from .utils.fixes import bincount
ImportError: cannot import name 'bincount' from 'sklearn.utils.fixes' (/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/sklearn/utils/fixes.py)

个人的sklearn类库出现了问题，重新安装一下之前提到的类库试试,也没发现问题，请教一下我的专业老师再说吧。

也希望看到博客的大神能告诉我一下怎么解决。