python xlog代码赏析

import os
import sys
import time
from datetime import datetime
import traceback
import threading
import json
import shutil
import types

reload(sys)
sys.setdefaultencoding('utf8')

CRITICAL = 50
FATAL = CRITICAL
ERROR = 40
WARNING = 30
WARN = WARNING
INFO = 20
DEBUG = 10
NOTSET = 0


class Logger():
    def __init__(self, name, buffer_size=0, file_name=None, roll_num=1):
        self.name = name
        self.file_max_size = 1024 * 1024
        self.buffer_lock = threading.Lock()
        self.buffer = {} # id => line
        self.buffer_size = buffer_size
        self.last_no = 0
        self.min_level = NOTSET
        self.log_fd = None
        self.set_color()
        self.roll_num = roll_num
        if file_name:
            self.set_file(file_name)

    def set_buffer(self, buffer_size):
        with self.buffer_lock:
            self.buffer_size = buffer_size
            buffer_len = len(self.buffer)
            if buffer_len > self.buffer_size:
                for i in range(self.last_no - buffer_len, self.last_no - self.buffer_size):
                    try:
                        del self.buffer[i]
                    except:
                        pass

    def setLevel(self, level):
        if level == "DEBUG":
            self.min_level = DEBUG
        elif level == "INFO":
            self.min_level = INFO
        elif level == "WARN":
            self.min_level = WARN
        elif level == "ERROR":
            self.min_level = ERROR
        elif level == "FATAL":
            self.min_level = FATAL
        else:
            print("log level not support:%s", level)
    
    def set_color(self):
        self.err_color = None
        self.warn_color = None
        self.debug_color = None
        self.reset_color = None
        self.set_console_color = lambda x: None
        if hasattr(sys.stderr, 'isatty') and sys.stderr.isatty():
            if os.name == 'nt':
                self.err_color = 0x04
                self.warn_color = 0x06
                self.debug_color = 0x002
                self.reset_color = 0x07
        
                import ctypes
                SetConsoleTextAttribute = ctypes.windll.kernel32.SetConsoleTextAttribute
                GetStdHandle = ctypes.windll.kernel32.GetStdHandle
                self.set_console_color = lambda color: SetConsoleTextAttribute(GetStdHandle(-11), color)
        
            elif os.name == 'posix':
                self.err_color = '\033[31m'
                self.warn_color = '\033[33m'
                self.debug_color = '\033[32m'
                self.reset_color = '\033[0m'
        
                self.set_console_color = lambda color: sys.stderr.write(color)

    def set_file(self, file_name):
        self.log_filename = file_name
        if os.path.isfile(file_name):
            self.file_size = os.path.getsize(file_name)
            if self.file_size > self.file_max_size:
                self.roll_log()
                self.file_size = 0
        else:
            self.file_size = 0

        self.log_fd = open(file_name, "a+")

    def roll_log(self):
        for i in range(self.roll_num, 1, -1):
            new_name = "%s.%d" % (self.log_filename, i)
            old_name = "%s.%d" % (self.log_filename, i - 1)
            if not os.path.isfile(old_name):
                continue

            #self.info("roll_log %s -> %s", old_name, new_name)
            shutil.move(old_name, new_name)

        shutil.move(self.log_filename, self.log_filename + ".1")

    def log(self, level, console_color, html_color, fmt, *args, **kwargs):
        now = datetime.now()
        time_str = now.strftime("%b %d %H:%M:%S.%f")[:19]
        string = '%s - [%s] %s\n' % (time_str, level, fmt % args)
        self.buffer_lock.acquire()
        try:
            self.set_console_color(console_color)
            try:
                console_string = '%s [%s][%s] %s\n' % (time_str, self.name, level, fmt % args)
                sys.stderr.write(console_string)
            except:
                pass
            self.set_console_color(self.reset_color)
    
            if self.log_fd:
                self.log_fd.write(string)
                try:
                    self.log_fd.flush()
                except:
                    pass

                self.file_size += len(string)
                if self.file_size > self.file_max_size:
                    self.log_fd.close()
                    self.log_fd = None
                    self.roll_log()
                    self.log_fd = open(self.log_filename, "w")
                    self.file_size = 0

            if self.buffer_size:
                self.last_no += 1
                self.buffer[self.last_no] = string
                buffer_len = len(self.buffer)
                if buffer_len > self.buffer_size:
                    del self.buffer[self.last_no - self.buffer_size]
        except Exception as e:
            string = '%s - [%s]LOG_EXCEPT: %s, Except:%s<br> %s' % (time.ctime()[4:-5], level, fmt % args, e, traceback.format_exc())
            self.last_no += 1
            self.buffer[self.last_no] = string
            buffer_len = len(self.buffer)
            if buffer_len > self.buffer_size:
                del self.buffer[self.last_no - self.buffer_size]
        finally:
            self.buffer_lock.release()

    def debug(self, fmt, *args, **kwargs):
        if self.min_level > DEBUG:
            return
        self.log('DEBUG', self.debug_color, '21610b', fmt, *args, **kwargs)
    
    def info(self, fmt, *args, **kwargs):
        if self.min_level > INFO:
            return
        self.log('INFO', self.reset_color, '000000', fmt, *args)
    
    def warning(self, fmt, *args, **kwargs):
        if self.min_level > WARN:
            return
        self.log('WARNING', self.warn_color, 'FF8000', fmt, *args, **kwargs)
    
    def warn(self, fmt, *args, **kwargs):
        self.warning(fmt, *args, **kwargs)
    
    def error(self, fmt, *args, **kwargs):
        if self.min_level > ERROR:
            return
        self.log('ERROR', self.err_color, 'FE2E2E', fmt, *args, **kwargs)
    
    def exception(self, fmt, *args, **kwargs):
        self.error(fmt, *args, **kwargs)
        self.error("Except stack:%s", traceback.format_exc(), **kwargs)
    
    def critical(self, fmt, *args, **kwargs):
        if self.min_level > CRITICAL:
            return
        self.log('CRITICAL', self.err_color, 'D7DF01', fmt, *args, **kwargs)
    
    #=================================================================
    def get_last_lines(self, max_lines):
        self.buffer_lock.acquire()
        buffer_len = len(self.buffer)
        if buffer_len > max_lines:
            first_no = self.last_no - max_lines
        else:
            first_no = self.last_no - buffer_len + 1

        jd = {}
        if buffer_len > 0:
            for i in range(first_no, self.last_no + 1):
                jd[i] = self.unicode_line(self.buffer[i])
        self.buffer_lock.release()
        return json.dumps(jd)
    
    def get_new_lines(self, from_no):
        self.buffer_lock.acquire()
        jd = {}
        first_no = self.last_no - len(self.buffer) + 1
        if from_no < first_no:
            from_no = first_no

        if self.last_no >= from_no:
            for i in range(from_no, self.last_no + 1):
                jd[i] = self.unicode_line(self.buffer[i])
        self.buffer_lock.release()
        return json.dumps(jd)

    def unicode_line(self, line):
        try:
            if type(line) is types.UnicodeType:
                return line
            else:
                return unicode(line, errors='ignore')
        except Exception as e:
            print("unicode err:%r" % e)
            print("line can't decode:%s" % line)
            print("Except stack:%s" % traceback.format_exc())
            return ""


class null():
    @staticmethod
    def debug(fmt, *args, **kwargs):
        pass
    @staticmethod
    def info(fmt, *args, **kwargs):
        pass
    @staticmethod
    def warn(fmt, *args, **kwargs):
        pass
    @staticmethod
    def exception(fmt, *args, **kwargs):
        pass


loggerDict = {}


def getLogger(name=None, buffer_size=0, file_name=None, roll_num=1):
    global loggerDict, default_log
    if name is None:
        for n in loggerDict:
            name = n
            break
    if name is None:
        name = "default"

    if not isinstance(name, basestring):
        raise TypeError('A logger name must be string or Unicode')
    if isinstance(name, unicode):
        name = name.encode('utf-8')

    if name in loggerDict:
        return loggerDict[name]
    else:
        logger_instance = Logger(name, buffer_size, file_name, roll_num)
        loggerDict[name] = logger_instance
        default_log = logger_instance
        return logger_instance


default_log = getLogger()


def debug(fmt, *args, **kwargs):
    default_log.debug(fmt, *args, **kwargs)


def info(fmt, *args, **kwargs):
    default_log.info(fmt, *args, **kwargs)


def warning(fmt, *args, **kwargs):
    default_log.warnin(fmt, *args, **kwargs)


def warn(fmt, *args, **kwargs):
    default_log.warn(fmt, *args, **kwargs)


def error(fmt, *args, **kwargs):
    default_log.error(fmt, *args, **kwargs)


def exception(self, fmt, *args, **kwargs):
    self.error(fmt, *args, **kwargs)
    self.error("Except stack:%s", traceback.format_exc(), **kwargs)


def critical(fmt, *args, **kwargs):
    default_log.critical(fmt, *args, **kwargs)

安装Python 2.7x

自带了xlog

xlog的特点

使用流式压缩方式对单行日志进行压缩，压缩加密后写进作为 log 中间 buffer的 mmap 中，当 mmap 中的数据到达一定大小后再写进磁盘文件中。 

xlog还存在一些其他策略：

每次启动的时候会清理日志，防止占用太多用户磁盘空间
为了防止 sdcard 被拔掉导致写不了日志，支持设置缓存目录，当 sdcard 插上时会把缓存目录里的日志写入到 sdcard 上
从下面的log2file流程图中可以看出xlog是如何利用cahce目录解决插拔sdcard的问题的。