# -*- coding: utf-8-*- # import sys import os import io import json reload(sys) sys.setdefaultencoding('utf-8') def unicode_convert(input): if isinstance(input, dict): return {unicode_convert(key): unicode_convert(value) for key, value in input.iteritems()} elif isinstance(input, list): return [unicode_convert(element) for element in input] elif isinstance(input, unicode): return input.encode('utf-8') else: return input filter=["Dm.log.wf"] #设置过滤后的文件类型 当然可以设置多个类型 def all_path(dirname): result = []#所有的文件 for maindir, subdir, file_name_list in os.walk(dirname): # print("1:",maindir) #当前主目录 # print("2:",subdir) #当前主目录下的所有目录 # print("3:",file_name_list) #当前主目录下的所有文件 for filename in file_name_list: apath = os.path.join(maindir, filename)#合并成一个完整路径 ext = os.path.splitext(apath)[0] # 获取文件后缀 [0]获取的是除了文件名以外的内容 ext = ext.split('/') ext = ext[5] if ext in filter: result.append(apath) return result if __name__ == "__main__": print sys.getdefaultencoding() result = all_path("/home/work/yangq/dm_analysis") for file_log in result: print "*****************************************" print file_log f = open(file_log, "r") f_new = open("/home/work/yangq/dm_analysis/result", "a+") f_new_error = open("/home/work/yangq/dm_analysis/error", "a+") haproxy_list = f.readlines() dm_statistics="dm_statistics" f_new.write(file_log + "\n") f_new.write("###################################################################" + "\n") f_new_error.write(file_log + "\n") f_new_error.write("###################################################################" + "\n") for line in haproxy_list: if dm_statistics in line: #print sys._getframe().f_lineno #print line strlist = line.split('dm_statistics: ') sss = strlist[1] #test2 = unicode_convert(json.loads(sss)) try: test2 = json.loads(sss.decode('utf-8')) except UnicodeDecodeError as w: f_new_error.write(line) f_new_error.write("有特殊自符" + "\n") continue except ValueError: f_new_error.write(line) f_new_error.write("ValueError" + "\n") continue f_new.write("query:" + str(test2['req']['query']) + " ") if 'recommend_word' in test2['latency']: f_new.write("recommend_word:" + str(test2['latency']['recommend_word']) + " ") if 'um' in test2['latency']: f_new.write("um:" + str(test2['latency']['um']) + " ") if 'se' in test2['latency']: f_new.write("se:" + str(test2['latency']['se']) + " ") if 'ia' in test2['latency']: f_new.write("ia:" + str(test2['latency']['ia']) + " ") if 'ir' in test2['latency']: f_new.write("ir:" + str(test2['latency']['ir']) + " ") if 'dm' in test2['latency']: f_new.write("dm:" + str(test2['latency']['dm']) + " ") if 'chat' in test2['latency']: f_new.write("chat:" + str(test2['latency']['chat']) + " ") if 'ACTION_CALCULATE.calc' in test2['detail_latency']: f_new.write("ACTION_CALCULATE.calc: " + str(test2['detail_latency']['nli']['ACTION_CALCULATE.calc']) + "\n") f_new.write("\n") #try: # f_new.write("query:" + str(test2['req']['query']) + " " + \ # "recommend_word:" + str(test2['latency']['recommend_word']) + \ # " " "um:" + str(test2['latency']['um']) + " " + \ # "se:" + str(test2['latency']['se']) + " " + \ # "ia:" + str(test2['latency']['ia']) + " " + \ # "ir:" + str(test2['latency']['ir']) + " " + \ # "dm:" + str(test2['latency']['dm']) + " " + \ # "chat:" + str(test2['latency']['chat']) + " " + \ # "ACTION_CALCULATE.calc: " + str(test2['detail_latency']['nli']['ACTION_CALCULATE.calc']) + "\n") #except KeyError as e: # f_new_error.write(line) # f_new_error.write("losing key:" + str(e) + "\n") # continue f.close() f_new.close() f_new_error.close() pass
dm_analysis
猜你喜欢
转载自www.cnblogs.com/nickkkk/p/10615661.html
今日推荐
周排行