The logging module of python is used, and the log files are saved according to the date path and scrolled and divided by time

At the beginning of this article, I tried three log file modules of logging, namely

1. The most basic FileHandler

2. TimedRotatingFileHandler for time scrolling and segmentation

3. RotatingFileHandler for file size scrolling slices

Finally, due to the needs of actual use, the file rollover block code of the doRollover method of TimedRotatingFileHandler is rewritten to save the log files to different folder paths according to the date, such as the path structure of /2022/08/11/, and reload the log files Name it for a specific time, such as 235959.info.log, and of course save two kinds of logs, info and error.

The idea of ​​rewriting comes from the following articles:

Python logs are automatically segmented by time - based on logging_Programmer? Migrant Workers! Blog-CSDN Blog

Effect:

 

source code:

import logging
import os
import datetime
import time
from logging.handlers import TimedRotatingFileHandler
from random import uniform
# fileName = __file__.split('.')[0]
# 如果日志文件夹不存在，则创建
log_dir = "log"  # 日志存放文件夹名称
log_path = os.getcwd() + os.sep + log_dir

# # logging 全局设置
logging.basicConfig(level=logging.INFO,
                    # level=__debug__,
                    # format=log_format,
                    datefmt='%Y/%m/%d %H:%M:%S',
                    # datefmt='%a, %d %b %Y %H:%M:%S',
                    # filename='{0}.log'.format(fileName),
                    filemode='a')
LOG_FORMAT = '%(asctime)s %(filename)s[line:%(lineno)d] %(levelname)s %(message)s'
formatter = logging.Formatter(LOG_FORMAT)


def get_myLogger(fileName='xxlog.log'):
    fileName = log_path + os.sep + fileName
    # 获取对象logger
    # level = logging.DEBUG
    level = logging.INFO
    logger = logging.getLogger(__name__)
    logger.setLevel(level=level)  # 设置日志基础级别

    if not os.path.isdir(log_path):
        os.makedirs(log_path)

    # # FileHandler
    # file_handler = logging.FileHandler(fileName)
    # file_handler.setLevel(level=level)
    # formatter = logging.Formatter(LOG_FORMAT)
    # file_handler.setFormatter(formatter)
    # logger.addHandler(file_handler)

    ## TimedRotatingFileHandler
    timefilehandler = TimedRotatingFileHandler(fileName,
                                               when='midnight',
                                               interval=1,
                                               backupCount=30)
    # timefilehandler.suffix = "%Y-%m-%d_%H-%M-%S.log"
    timefilehandler.setLevel(level=level)
    timefilehandler.setFormatter(formatter)
    logger.addHandler(timefilehandler)

    # ## RotatingFileHandler
    # rotatingfilehandler = RotatingFileHandler(fileName,
    #                                           maxBytes=1024**2,  # 1:kb, 2:mb,
    #                                           backupCount=30)
    # rotatingfilehandler.setLevel(level=level)
    # formatter = logging.Formatter(LOG_FORMAT)
    # rotatingfilehandler.setFormatter(formatter)
    # logger.addHandler(rotatingfilehandler)


    # LOG
    logger.info("file:{0}".format(fileName))
    return logger


class LogFilter:
    @staticmethod
    def info_filter(record):
        if record.levelname == 'INFO':
            return True
        return False

    @staticmethod
    def error_filter(record):
        if record.levelname == 'ERROR':
            return True
        return False


class TimeLoggerRolloverHandler(TimedRotatingFileHandler):
    def __init__(self, filename, when='h', interval=1, backupCount=0, encoding=None, delay=False, utc=False,
                 atTime=None):
        super(TimeLoggerRolloverHandler, self).__init__(filename, when, interval, backupCount, encoding, delay, utc)

    def doRollover(self):
        """
        TimedRotatingFileHandler对日志的切分是在满足设定的时间间隔后，执行doRollover方法，
        将my.log重命名为带有当前时间后缀(my.log.****)的文件，并新建一个my.log，继续记录后续日志。
        (1) 重写TimedRotatingFileHandler的doRollover方法的文件翻转块代码
        做了以下两点改动：
            重定义了新文件名，将日期放在了中间而不是最后
            直接将将baseFilename 指向新文件
        """
        if self.stream:
            self.stream.close()
            self.stream = None
        currentTime = int(time.time())
        dstNow = time.localtime(currentTime)[-1]

        log_type = 'info' if self.level == 20 else 'error'
        # 重新定义了新文件名
        base_dir = os.path.dirname(self.baseFilename)[:-11]
        datetime_now = datetime.datetime.now().strftime('%Y-%m-%d_%H%M%S').split('_')
        date_now = datetime_now[0]
        time_now = datetime_now[1]
        file_date = '/'.join(date_now.split('-'))  # '2022/07/11/15/13/35'
        log_dir = f'{base_dir}/{file_date}'
        if not os.path.exists(log_dir):
            os.makedirs(log_dir, exist_ok=True)
        dfn = f"{log_dir}/{time_now}.{log_type}.log"
        if os.path.exists(dfn):
            os.remove(dfn)
        # dfn = f"my.{datetime.datetime.now().strftime('%Y%m%d')}.{log_type}.log"
        print(self.baseFilename, '...before')
        self.baseFilename = dfn  # 直接将将baseFilename 指向新文件
        print(self.baseFilename, '...after')

        # if os.path.exists(dfn):
        #     os.remove(dfn)
        # self.rotate(self.baseFilename, dfn)
        # if self.backupCount > 0:
        #     for s in self.getFilesToDelete():
        #         os.remove(s)
        if not self.delay:
            self.stream = self._open()
        newRolloverAt = self.computeRollover(currentTime)
        while newRolloverAt <= currentTime:
            newRolloverAt = newRolloverAt + self.interval
        #If DST changes and midnight or weekly rollover, adjust for this.
        if (self.when == 'MIDNIGHT' or self.when.startswith('W')) and not self.utc:
            dstAtRollover = time.localtime(newRolloverAt)[-1]
            if dstNow != dstAtRollover:
                if not dstNow:  # DST kicks in before next rollover, so we need to deduct an hour
                    addend = -3600
                else:           # DST bows out before next rollover, so we need to add an hour
                    addend = 3600
                newRolloverAt += addend
        self.rolloverAt = newRolloverAt


def get_myProjectLogger(project_name, log_file_name, when='H', interval=1):
    base_dir = f"./log_{project_name}"
    datetime_now = datetime.datetime.now().strftime('%Y-%m-%d_%H%M%S').split('_')
    date_now = datetime_now[0]
    time_now = datetime_now[1]
    file_date = '/'.join(date_now.split('-'))  # '2022/07/11/15/13/35'
    log_dir = f'{base_dir}/{file_date}'
    if not os.path.exists(log_dir):
        os.makedirs(log_dir, exist_ok=True)
    log_error_file = f"{log_dir}/{time_now}.error.log"
    log_info_file = f"{log_dir}/{time_now}.info.log"

    error_handler = TimeLoggerRolloverHandler(log_error_file, when=when, interval=interval)
    error_handler.addFilter(LogFilter.error_filter)
    error_handler.setFormatter(formatter)
    error_handler.setLevel(logging.ERROR)

    info_handel = TimeLoggerRolloverHandler(log_info_file, when=when, interval=interval)
    info_handel.addFilter(LogFilter.info_filter)
    info_handel.setFormatter(formatter)
    info_handel.setLevel(logging.INFO)

    level = logging.INFO
    logger = logging.getLogger(__name__)
    logger.setLevel(level=level)  # 设置日志基础级别

    logger.addHandler(info_handel)
    logger.addHandler(error_handler)
    # LOG
    logger.info("file:{0}".format(log_file_name))
    return logger



if __name__ == "__main__":
    # my_logger = get_myLogger(fileName='xxlog.log')
    
    # WHEN = 'H'  # 日志切分单位，时间单位
    # INTERVAL = 1  # 日志切分按多少个WHEN来间隔
    my_logger = get_myProjectLogger("project_name", "log_filename", when='H', interval=1)