python 分析nginx日志（2）

闲着也是闲着，练练手，分析nginx日志http code码短信告知

#!/usr/bin/python
# -*- coding: UTF-8 -*-
'''
auth yufei
2016-4-3
分析nginx日志，计算200以及500等个数，并且短信通知
'''

import os
import fileinput
import re
import json
import urllib,urllib2

dir_log = r"/data/logs/access.log"

ipP = r"?P<ip>[\d.]*"
uidP = r"?P<uid>[\d.-]*"
timeP = r"?P<time>\[[^\[\]]*\]"
servernaemeP = r'?P<servernaeme>[\w.]*'
requestP = r'?P<request>\"[^\"]*\"'
statusP = r"?P<status>\d+"
bodyBytesSentP = r"?P<bodyBytesSent>\d+"
referP = r'?P<refer>\"[^\"]*\"'
userAgentP = r'?P<userAgent>\"[^\"]*\"'
phpP = r"?P<php>[\d.:]*"
qP = r"?P<q>[\d.]*"
hP = r"?P<h>[\d.]*"

nginxLogPattern = re.compile(r"(%s)\ -\ (%s)\ (%s)\ (%s)\ (%s)\ (%s)\ (%s)\ (%s)\ (%s)\ (%s)\ (%s)\ (%s)" %(ipP, uidP, timeP, servernaemeP, requestP, phpP, statusP, bodyBytesSentP, qP,hP,referP,userAgentP), re.VERBOSE)

StatusDict = {}
UidDict = {}
sname = {}
ipdict = {}
Apidict = {}
datas = {}

def processLog(dir_log):
    for line in fileinput.input(dir_log):
        matchs = nginxLogPattern.match(line)
        if matchs !=None:
            allGroups = matchs.groups()
            ip = allGroups[0]
            uid = allGroups[1]
            time = allGroups[2]
            servernaeme = allGroups[3]
            request = allGroups[4]
            status = allGroups[6]
            bodyBytesSent = allGroups[7]
            refer = allGroups[10]
            userAgent = allGroups[11]
            #userAgent = matchs.group("userAgent")

            api = re.match(r'.*\/[api2/]*\/(.*)\?.*', request)
            if api:
                GetResponseStatusCount(Apidict,api.group(1))

            GetResponseStatusCount(StatusDict,status)
            GetResponseStatusCount(sname,servernaeme)
            GetResponseStatusCount(ipdict,ip)
            if len(uid) < 10:
               GetResponseStatusCount(UidDict,uid)

            #pattern = r'.*\d{4}:([\d:\/]*):.*'
            #a = re.match(pattern, str)     

            if int(status) >= 499 :
                f = open('/data/html/phpinfo/py_nginx_500.log','a')
                f.write(line)
                f.close()

        else:
            #raise Exception
            pass
    fileinput.close()

def TallMe(status,shuzi,phone="17788888888"):
    service = status+"as" + str(shuzi)
    url='http://test.com/api/sendMessage'
    textmod ={'tplid':'111111',
              'ip':"web1",
              's': service,
              'time':"Past_hour",
              'phone':phone,
              'check':0}
    data1 = urllib.urlencode(textmod)
    data = urllib.urlencode(textmod)
    print(data)
    header_dict = {'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; Trident/7.0; rv:11.0) like Gecko',"Content-Type": "application/json"}

    req = urllib2.Request(url)
    opener = urllib2.build_opener(urllib2.HTTPCookieProcessor())
    response = opener.open(req, data)
    res = response.read()

    req = urllib2.Request(url,data,header_dict)
    res = urllib2.urlopen(req)
    res = res.read()
    print(res)
    return

def GetResponseStatusCount(dit,status):
    if dit.has_key(status):
        dit[status] += 1
    else:
        dit[status] = 1;

if __name__ == "__main__":
    processLog(dir_log)

    datas["ip"] = ipdict
    datas["stus"] = StatusDict
    datas["sname"] = sname
    datas["uid"] = UidDict
    datas["api"] = Apidict
    datas = json.dumps(datas)

    for key in StatusDict:
        #print(key)
        if key == "499":
            f499 = StatusDict["499"]
            #TallMe("499",f499)
        elif key == "500":
            f500 = StatusDict["500"]
            if int(f500) >= 5:
                TallMe("500",f500)
                TallMe("500",f500,"18956070000")
        elif key == "502":
            f502 = StatusDict["502"]
            #TallMe("502",f502)
        elif key == "504":
            f504 = StatusDict["504"]
            #TallMe("504",f504)
        else:
            pass

    f = open('/data/logs/py_nginx.html','w')
    f.write(datas)
    f.close()