Timer events for Nginx event management

1. Cache time

1.1 Management

Each process in Nginx manages the current time individually. The ngx_time_t structure is the type of cached time variable:

typedef struct {
    /* 格林威治时间1970年1月1日凌晨0点0分0秒到当前时间的秒数 */
    time_t      sec;
    /* sec成员只能精确到秒，msec则是当前时间相对于sec的毫秒偏移量 */
    ngx_uint_t  msec;
    /* 时区 */
    ngx_int_t   gmtoff;
}ngx_time_t;

Nginx defines the following global variables for caching time:

/* 格林威治时间1970年1月1日凌晨0点0分0秒到当前时间的毫秒数 */
volatile ngx_msec_t      ngx_current_msec;
/* ngx_time_t结构体形式的当前时间 */
volatile ngx_time_t     *ngx_cached_time;
/* 用于记录error_log的当前时间字符串，它的格式类似于："1970/09/28 12:00:00" */
volatile ngx_str_t       ngx_cached_err_log_time;
/* 用于记录HTTP相关的当前时间字符串，它的格式类似于："Mon, 28 Sep 1970 06:00:00 GMT" */
volatile ngx_str_t       ngx_cached_http_time;
/* 用于记录HTTP日志的当前时间字符串，它的格式类似于："28/Sep1970:12:00:00 +0600" */
volatile ngx_str_t       ngx_cached_http_log_time;
/* 以ISO 8601标准格式记录下的字符串形式的当前时间 */
volatile ngx_str_t       ngx_cached_http_log_iso8601;
volatile ngx_str_t       ngx_cached_syslog_time;

For the worker process, except for updating the time once when Nginx starts, any operation of updating the time can only be performed by the ngx_epoll_process_events
method. In this method, when the NGX_UPDATE_TIME flag is detected in the flags parameter, or the ngx_event_timer_alarm flag bit is 1
, the ngx_time_update method is called to update the cache time.

/*
 * 执行意义：
 * 使用gettimeofday调用以系统时间更新缓存的时间，上述的ngx_current_msec、ngx_cached_time、
 * ngx_cached_err_log_time、ngx_cached_http_time、ngx_cached_http_log_time、
 * ngx_cached_http_log_iso8601、ngx_cached_syslog_time这几个全局变量都会得到刷新 */
void ngx_time_update(void)
{
    u_char          *p0, *p1, *p2, *p3, *p4;
    ngx_tm_t         tm, gmt;
    time_t           sec;
    ngx_uint_t       msec;
    ngx_time_t      *tp;
    struct timeval   tv;

    if (!ngx_trylock(&ngx_time_lock))
    {
        return;
    }

    ngx_gettimeofday(&tv);

    sec = tv.tv_sec;
    msec = tv.tv_usec / 1000;

    ngx_current_msec = (ngx_msec_t)sec * 1000 + msec;

    tp = &cached_time[slot];

    if (tp->sec == sec)
    {
        tp->msec = msec;
        ngx_unlock(&ngx_time_lock);
        return;
    }

    if (slot == NGX_TIME_SLOTS - 1)
    {
        slot = 0;
    }
    else
    {
        slot++;
    }

    tp = &cached_time[slot];

    tp->sec = sec;
    tp->msec = msec;

    ngx_gmtime(sec, &gmt);

    p0 = &cached_http_time[slot][0];

    (void)ngx_sprintf(p0, "%s, %02d %s %4d %02d:%02d:%02d GMT", 
                      week[gmt.ngx_tm_wday], gmt.ngx_tm_mday,
                      months[gmt.ngx_tm_mon - 1], gmt.ngx_tm_year,
                      gmt.ngx_tm_hour, gmt.ngx_tm_min, gmt.ngx_tm_sec);

#if (NGX_HAVE_GETTIMEZONE)

    tp->gmtoff = ngx_gettimezone();
    ngx_gmtime(sec + tp->gmtoff * 60, &tm);

#elif (NGX_HAVE_GMTOFF)

    ngx_localtime(sec, &tm);
    cached_gmtoff = (ngx_int_t)(tm.ngx_tm_gmtoff / 60);
    tp->gmtoff = cached_gmtoff;

#else

    ngx_localtime(sec, &tm);
    cached_gmtoff = ngx_timezone(tm.ngx_tm.isdst);
    tp->gmtoff = cached_gmtoff;

#endif

    p1 = &cached_err_log_time[slot][0];

    (void)ngx_sprintf(p1, "%4d/%02d/%02d %02d:%02d:%02d", 
                      tm.ngx_tm_year, tm.ngx_tm_mon,
                       tm.ngx_tm_mday, tm.ngx_tm_hour,
                       tm.ngx_tm_min, tm.ngx_tm_sec);

    p2 = &cached_http_log_time[slot][0];

    (void) ngx_sprintf(p2, "%02d/%s/%d:%02d:%02d:%02d %c%02i%02i",
                       tm.ngx_tm_mday, months[tm.ngx_tm_mon - 1],
                       tm.ngx_tm_year, tm.ngx_tm_hour,
                       tm.ngx_tm_min, tm.ngx_tm_sec,
                       tp->gmtoff < 0 ? '-' : '+',
                       ngx_abs(tp->gmtoff / 60), ngx_abs(tp->gmtoff % 60));

    p3 = &cached_http_log_iso8601[slot][0];

    (void) ngx_sprintf(p3, "%4d-%02d-%02dT%02d:%02d:%02d%c%02i:%02i",
                       tm.ngx_tm_year, tm.ngx_tm_mon,
                       tm.ngx_tm_mday, tm.ngx_tm_hour,
                       tm.ngx_tm_min, tm.ngx_tm_sec,
                       tp->gmtoff < 0 ? '-' : '+',
                       ngx_abs(tp->gmtoff / 60), ngx_abs(tp->gmtoff % 60));

    p4 = &cached_syslog_time[slot][0];

    (void) ngx_sprintf(p4, "%s %2d %02d:%02d:%02d",
                       months[tm.ngx_tm_mon - 1], tm.ngx_tm_mday,
                       tm.ngx_tm_hour, tm.ngx_tm_min, tm.ngx_tm_sec);

    ngx_memory_barrier();

    ngx_cached_time = tp;
    ngx_cached_http_time.data = p0;
    ngx_cached_err_log_time.data = p1;
    ngx_cached_http_log_time.data = p2;
    ngx_cached_http_log_iso8601.data = p3;
    ngx_cached_syslog_time.data = p4;

    ngx_unlock(&ngx_time_lock);
}

1.2 Accuracy

Nginx provides the function of setting the frequency of updating the cache time (that is, the cache time must be updated at least every timer_resolution milliseconds).
The minimum frequency of updating can be set through the timer_resolution configuration item in the nginx.conf file, thus ensuring the cache time precision.

The ngx_event_core_module module uses the setitimer system call to tell the kernel to call the ngx_timer_signal_handler method every
timer_resolution milliseconds when calling the ngx_event_process_init method to initialize. The ngx_timer_signal_handler method will set the
ngx_event_timer_alarm flag to 1, so that once the ngx_epoll_process_events method is called, if the interval exceeds
timer_resolution milliseconds, the ngx_time_update method will be called to update the cache time.

But if the ngx_epoll_process_events method is not called within milliseconds of timer_resolution, how can the time precision be guaranteed?
In this case, Nginx can only guarantee the time precision from the event module's implementation of the process_events interface in ngx_event_actions.
The second parameter timer of the process_events method represents the maximum wait time when collecting events. For example, under the epoll module, this timer is
the timeout parameter passed in when epoll_wait is called. If timer_resolution is not set, in general, the timer
parameter of the process_events method is a value greater than 0 and less than 500 milliseconds, and if the timer_resolution is set, the timer parameter is -1, which means that if
epoll_wait and other calls cannot detect that the When an event occurs, it will not wait but return immediately, thus controlling the event precision. However, if the callback method of an event consumption module
takes too long to execute, the time accuracy is still difficult to guarantee.

2. Timer

2.1 Overview

The timer is implemented through a red-black tree.

/* Nginx设置了两个全局变量以便在程序的任何地方都可以快速地访问到这颗红黑树 */

/* ngx_event_timer_rbtree封装了整颗红黑树结构 */
ngx_rbtree_t              ngx_event_timer_rbtree;
/* ngx_event_timer_sentinel属于红黑树节点类型变量，在红黑树的操作过程中被当做哨兵节点使用 */
static ngx_rbtree_node_t  ngx_event_timer_sentinel;

Each node in this red-black tree is a timer member in the ngx_event_t event, and the key of the ngx_rbtree_node_t node is the event's timeout
time. The size of this timeout time constitutes a binary sorting tree ngx_event_timer_rbtree. In this way, if you need to find the event that is most likely to time out, just
take out the leftmost node in the ngx_event_timer_rbtree tree. As long as the current time is used to compare the timeout time of the leftmost node, you will
know whether the event has triggered a timeout. If the timeout has not been triggered, you will know at least how many milliseconds will pass to satisfy the timeout condition and trigger the timeout.

2.2 Provided interfaces

2.2.1 ngx_event_timer_init: Initialize the timer

/* 红黑树(即定时器)的初始化函数 */
ngx_int_t ngx_event_timer_init(ngx_log_t *log)
{
    /* ngx_event_timer_rbtree 和 ngx_event_timer_sentinel 是两个全局变量，前者指向
     * 整颗红黑树，后者指向了哨兵节点, ngx_rbtree_insert_timer_value 函数指针则为
     * 将元素插入这棵红黑树的方法 */
    ngx_rbtree_init(&ngx_event_timer_rbtree, &ngx_event_timer_sentinel, ngx_rbtree_insert_timer_value);

    return NGX_OK;
}

2.2.2 ngx_event_add_timer: Add a timed event

/*
 * 参数含义：
 * - ev：是需要操作的事件
 * - timer：单位是毫秒，它告诉定时器事件ev希望timer毫秒后超时，同时需要回调ev的handler方法
 *
 * 执行意义：
 * 添加一个定时器事件，超时时间为 timer 毫秒
 */
static ngx_inline void ngx_event_add_timer(ngx_event_t *ev, ngx_msec_t timer)
{
    ngx_msec_t      key;
    ngx_msec_int_t  diff;

    key = ngx_current_msec + timer;

    /* 若该事件已经添加到红黑树中 */
    if (ev->timer_set)
    {
        /* Use a previous timer value if difference between it and a new 
         * value is less than NGX_TIMER_LAZY_DELY milliseconds: this allows
         * to minimize the rbtree operations for fast connections. */

        diff = (ngx_msec_int_t)(key - ev->timer.key);

        if (ngx_abs(diff) < NGX_TIMER_LAZY_DELAY)
        {
            ngx_log_debug3(NGX_LOG_DEBUG_EVENT, ev->log, 0, 
                           "event timer: %d, old: %M, new: %M", 
                           ngx_event_ident(ev->data), ev->timer.key, key);
            return;
        }

        /* 将该事件从红黑树中删除 */
        ngx_del_timer(ev);
    }

    /* 记录该事件的超时时刻，在后续进行超时检测扫描时需要该字段来进行时刻的先后比较 */
    ev->timer.key = key;

    ngx_log_debug3(NGX_LOG_DEBUG_EVENT, ev->log, 0, 
                   "event timer add: %d: %M:%M",
                   ngx_event_ident(ev->data), timer, ev->timer.key);

    /* 将事件添加到红黑树中
     * 这种添加是间接性的，每个事件对象封装结构体中都有一个timer字段，
     * 其为ngx_rbtree_node_t 类型变量，加入到红黑树中就是该字段，
     * 而非事件对象结构体本身。后面要获取该事件结构体时可以通过利用
     * offsetof宏来根据该timer字段方便找到其所在的对应事件对象结构体. */
    ngx_rbtree_insert(&ngx_event_timer_rbtree, &ev->timer);

    /* 置位该变量，表示添加到红黑树中 */
    ev->timer_set = 1;
}

2.2.3 ngx_event_find_timer

/*
 * 执行意义：
 * 找出红黑树中最左边的节点，如果它的超时时间大于当前时间，也就表明目前的定时器中没有一个事件
 * 满足触发条件，这时返回这个超时与当前时间的差值，也就是需要经过多少毫秒会有事件超时触发；如果
 * 它的超时时间小于或等于当前时间，则返回0，表示定时器中已经存在超时需要触发的事件.
 */
ngx_msec_t ngx_event_find_timer(void)
{
    ngx_msec_int_t      timer;
    ngx_rbtree_node_t  *node, *root, *sentinel;

    /* 检测红黑树是否为空 */
    if (ngx_event_timer_rbtree.root == &ngx_event_timer_sentinel)
    {
        return NGX_TIMER_INFINITE;
    }

    root     = ngx_event_timer_rbtree.root;
    sentinel = ngx_event_timer_rbtree.sentinel;

    /* 找出该树 key 最小的那个节点，即超时时间最小的 */
    node = ngx_rbtree_min(root, sentinel);

    /* 该节点的超时时间与当前时间的毫秒比较，若大于，则表明还没有触发超时，返回它们的差值；
     * 若小于或等于，则表示已经满足超时条件，返回0 */
    timer = (ngx_msec_int_t)(node->key - ngx_current_msec);

    return (ngx_msec_t)(timer > 0 ? timer : 0);
}

2.2.4 ngx_event_expire_timers

/*
 * 执行意义：
 * 检查定时器中的所有事件，按照红黑树关键字由小到大的顺序依次调用已经满足
 * 超时条件的需要被触发事件的 handler 回调方法.
 */
void ngx_event_expire_timers(void)
{
    ngx_event_t        *ev;
    ngx_rbtree_node_t  *node, *root, *sentinel;

    sentinel = ngx_event_timer_rbtree.sentinel;

    for ( ;; )
    {
        root = ngx_event_timer_rbtree.root;

        if (root == sentinel)
        {
            return;
        }

        node = ngx_rbtree_min(root, sentinel);

        /* node->key > ngx_current_msec */

        /* 没有超时，则直接返回 */
        if ((ngx_msec_int_t)(node->key - ngx_current_msec) > 0)
        {
            return;
        }

        /* 计算 ev 的首地址位置 */
        ev = (ngx_event_t *)((char *)node - offsetof(ngx_event_t, timer));

        ngx_log_debug2(NGX_LOG_DEBUG_EVENT, ev->log, 0, 
                       "event timer del: %d: %M", 
                       ngx_event_ident(ev->data), ev->timer.key);

        /* 该事件已经满足超时条件，需要从定时器中移除 */
        ngx_rbtree_delete(&ngx_event_timer_rbtree, &ev->timer);

#if (NGX_DEBUG)
        ev->timer.left = NULL;
        ev->timer.right = NULL;
        ev->timer.parent = NULL;
#endif

        /* 置为 0，表示已经不在定时器中了 */
        ev->timer_set = 0;

        /* 置为 1，表示已经超时了 */
        ev->timedout = 1;

        /* 调用该超时事件的方法 */
        ev->handler(ev);
    }
}

ngx_event_expire_timers flowchart