关于libevent-book-examples_01----ROT13 server by using select() 一处bug分析及修正

刚刚学习libevent这个开源网络库时,在研究libevent-book examples_01 目录下有一例子，使用select（）实现rto13 server.这个select（）的基本框架在并发连接非常少时可以用来做一个简易服务器，可以在此基础上进行相应修改即可。
此server的功能基本上就是一个 echo server. 我比较关注的是select（）监听读写事件的数据处理存在一个bug，如果向 rto13 server 连续发送数据，前提是两次发送数据间隔很短，那么两次的数据会被合并，执行一次写事件。先附上源码在分析：libevent-book/examples_01/01_r0t13_server_select.c
获取源码地址 git clone git://github.com/nmathewson/libevent-book.git

/* For sockaddr_in */
#include <netinet/in.h>
/* For socket functions */
#include <sys/socket.h>
/* For fcntl */
#include <fcntl.h>
/* for select */
#include <sys/select.h>

#include <assert.h>
#include <unistd.h>
#include <string.h>
#include <stdlib.h>
#include <stdio.h>
#include <errno.h>

#define MAX_LINE 16384

char
rot13_char(char c)
{
    /* We don't want to use isalpha here; setting the locale would change
     * which characters are considered alphabetical. */
    if ((c >= 'a' && c <= 'm') || (c >= 'A' && c <= 'M'))
        return c + 13;
    else if ((c >= 'n' && c <= 'z') || (c >= 'N' && c <= 'Z'))
        return c - 13;
    else
        return c;
}

struct fd_state {
    char buffer[MAX_LINE];
    size_t buffer_used;

    int writing;
    size_t n_written;
    size_t write_upto;
};

struct fd_state *
alloc_fd_state(void)
{
    struct fd_state *state = malloc(sizeof(struct fd_state));
    if (!state)
        return NULL;
    state->buffer_used = state->n_written = state->writing =
        state->write_upto = 0;
    return state;
}

void
free_fd_state(struct fd_state *state)
{
    free(state);
}

void
make_nonblocking(int fd)
{
    fcntl(fd, F_SETFL, O_NONBLOCK);
}

int
do_read(int fd, struct fd_state *state)
{
    char buf[1024];
    int i;
    ssize_t result;
    while (1) {
        result = recv(fd, buf, sizeof(buf), 0);
        if (result <= 0)
            break;

        for (i=0; i < result; ++i)  {
            if (state->buffer_used < sizeof(state->buffer))
                state->buffer[state->buffer_used++] = rot13_char(buf[i]);
            if (buf[i] == '\n') {
                state->writing = 1;
                state->write_upto = state->buffer_used;
            }
        }
    }
    if (result == 0) {
        return 1;
    } else if (result < 0) {
        if (errno == EAGAIN)
            return 0;
        return -1;
    }

    return 0;
}

int
do_write(int fd, struct fd_state *state)
{
    while (state->n_written < state->write_upto) {
        ssize_t result = send(fd, state->buffer + state->n_written,
                              state->write_upto - state->n_written, 0);
        if (result < 0) {
            if (errno == EAGAIN)
                return 0;
            return -1;
        }
        assert(result != 0);

        state->n_written += result;
    }

    if (state->n_written == state->buffer_used)
        state->n_written = state->write_upto = state->buffer_used = 0;

    state->writing = 0;

    return 0;
}

void
run(void)
{
    int listener;
    struct fd_state *state[FD_SETSIZE];
    struct sockaddr_in sin;
    int i, maxfd;
    fd_set readset, writeset, exset;

    sin.sin_family = AF_INET;
    sin.sin_addr.s_addr = 0;
    sin.sin_port = htons(40713);

    for (i = 0; i < FD_SETSIZE; ++i)
        state[i] = NULL;

    listener = socket(AF_INET, SOCK_STREAM, 0);
    make_nonblocking(listener);

#ifndef WIN32
    {
        int one = 1;
        setsockopt(listener, SOL_SOCKET, SO_REUSEADDR, &one, sizeof(one));
    }
#endif

    if (bind(listener, (struct sockaddr*)&sin, sizeof(sin)) < 0) {
        perror("bind");
        return;
    }

    if (listen(listener, 16)<0) {
        perror("listen");
        return;
    }

    FD_ZERO(&readset);
    FD_ZERO(&writeset);
    FD_ZERO(&exset);

    while (1) {
        maxfd = listener;

        FD_ZERO(&readset);
        FD_ZERO(&writeset);
        FD_ZERO(&exset);

        FD_SET(listener, &readset);

        for (i=0; i < FD_SETSIZE; ++i) {
            if (state[i]) {
                if (i > maxfd)
                    maxfd = i;
                FD_SET(i, &readset);
                if (state[i]->writing) {
                    FD_SET(i, &writeset);
                }
            }
        }
        if (select(maxfd+1, &readset, &writeset, &exset, NULL) < 0) {
            perror("select");
            return;
        }

        if (FD_ISSET(listener, &readset)) {
            struct sockaddr_storage ss;
            socklen_t slen = sizeof(ss);
            int fd = accept(listener, (struct sockaddr*)&ss, &slen);
            if (fd < 0) {
                perror("accept");
            } else if (fd > FD_SETSIZE) {
                close(fd);
            } else {
                make_nonblocking(fd);
                state[fd] = alloc_fd_state();
                assert(state[fd]);/*XXX*/
            }
        }

        for (i=0; i < maxfd+1; ++i) {
            int r = 0;
            if (i == listener)
                continue;
            if (FD_ISSET(i, &readset)) {
                r = do_read(i, state[i]);
            }
            if (r == 0 && FD_ISSET(i, &writeset)) {
                r = do_write(i, state[i]);
            }
            if (r) {
                free_fd_state(state[i]);
                state[i] = NULL;
                close(i);
            }
        }
    }
}

int
main(int c, char **v)
{
    setvbuf(stdout, NULL, _IONBF, 0);

    run();
    return 0;
}

正常情况测试如下：

上面是rto13_server 监听 40713端口
下面是使用 nc 来作为客户端向server 发送数据，数字是不做处理原样返回的

先分析流程：
1.服务器启动，绑定端口，设置监听事件，然后阻塞在select处

        if (select(maxfd+1, &readset, &writeset, &exset, NULL) < 0) {
            perror("select");
            return;
        }

2.客户端连接上 server时，select()会返回已经准备好的描述符（为正值），即连接事件

        if (FD_ISSET(listener, &readset)) {
            struct sockaddr_storage ss;
            socklen_t slen = sizeof(ss);
            int fd = accept(listener, (struct sockaddr*)&ss, &slen);
            if (fd < 0) {
                perror("accept");
            } else if (fd > FD_SETSIZE) {
                close(fd);
            } else {
                make_nonblocking(fd);
                state[fd] = alloc_fd_state();
                assert(state[fd]);/*XXX*/
            }
        }

此时fd 是在套接口listener接受的一个连接，后续的读写数据就通过此fd，此fd是唯一确定的，一般取值是系统空闲文件描述符最小值，我们记此fd 为 FD;
此FD会在下一个while 中被设置监听读事件，然后继续阻塞在select（)处
3.当客户端发送 “123456\n"，server 中 select（）监听到FD读事件触发，会继续往下执行：

   if (FD_ISSET(i, &readset)) {
                //   printf("read--%d\n", i);
                r = do_read(i, state[i]);
            }

通过 do_read 将“123456\n”读取出来并存放在缓冲区，设置 写标志
state->writing = 1
do_read()

        for (i=0; i < result; ++i)  {
            if (state->buffer_used < sizeof(state->buffer))
                state->buffer[state->buffer_used++] = rot13_char(buf[i]);
            if (buf[i] == '\n') {
                state->writing = 1;
                state->write_upto = state->buffer_used;
            }
        }

当再次回到while处就会设置 写事件：

        for (i=0; i < FD_SETSIZE; ++i) {
            if (state[i]) {
                if (i > maxfd)
                    maxfd = i;
                FD_SET(i, &readset);
                if (state[i]->writing) {
                    FD_SET(i, &writeset);
                }
            }
        }

执行到select（）处立即返回，后续执行
do_write() 返回给客户端

接着 发送“987654\n” 做相同动作。

不过在此之间，如果客户端两次数据间隔比较短，那么server的执行就不是如此，会一次性把两次收到的数据返回给客户端。
下面模拟此种情况：
先在 select（）之前 加上一段延时， 如下：

        sleep(3);
        if (select(maxfd+1, &readset, &writeset, &exset, NULL) < 0) {
            perror("select");
            return;
        }

do_read 添加打印信息

int
do_read(int fd, struct fd_state *state)
{
    char buf[1024];
    int i;
    ssize_t result;
    while (1) {
        result = recv(fd, buf, sizeof(buf), 0);
        if (result <= 0)
            break;

        for (i=0; i < result; ++i)  {
            if (state->buffer_used < sizeof(state->buffer))
                state->buffer[state->buffer_used++] = rot13_char(buf[i]);
            if (buf[i] == '\n') {
                state->writing = 1;
                state->write_upto = state->buffer_used;
            }
        }
    }
    printf("do_read:\n");
    printf("state->buffer_used=%d\nstate->write_upto=%d\nstate->writing=%d\n", state->buffer_used, state->write_upto, state->writing);
    if (result == 0) {
        return 1;
    } else if (result < 0) {
        if (errno == EAGAIN)
            return 0;
        return -1;
    }

    return 0;
}

do_write添加打印信息：

int
do_write(int fd, struct fd_state *state)
{
    while (state->n_written < state->write_upto) {
        ssize_t result = send(fd, state->buffer + state->n_written,
                              state->write_upto - state->n_written, 0);
        if (result < 0) {
            if (errno == EAGAIN)
                return 0;
            return -1;
        }
        assert(result != 0);

        state->n_written += result;
    }
    printf("do_write:\n");
    printf("state->buffer_used=%d\nstate->write_upto=%d\nstate->n_written=%d\n", state->buffer_used, state->write_upto, state->n_written);
    if (state->n_written == state->buffer_used)
        state->n_written = state->write_upto = state->buffer_used = 0;

    state->writing = 0;

    return 0;
}

测试：

可以看到，我们在3秒内连续输入两行 “1234\n” “12315\n”
server 接收两条信息，但是却用一次返回给客户端，造成这个的原因是select（）接收第二条信息 "12315\n"的时候同时也有写事件 ，但是是先处理读事件，再处理写事件，并且第二次的收到的"12315\n直接拼接在第一条信息 “1234\n”的后面，一起作为第一次的返回响应，这个在逻辑上存在一点问题，第二条信息的响应怎么和第一条合在一起了，这个对于输入缓冲带“\n”没有问题，但是如果将此程序移植作为他用，是存在一定问题的，收发响应需一一对应

改善，先处理写事件，再处理读事件，如下：

        for (i=0; i < maxfd+1; ++i) {
            int r = 0;
            if (i == listener)
                continue;

            if (r == 0 && FD_ISSET(i, &writeset)) {
                r = do_write(i, state[i]);
            }
            if (FD_ISSET(i, &readset)) {
                r = do_read(i, state[i]);
            }


            if (r) {
                free_fd_state(state[i]);
                state[i] = NULL;
                close(i);
            }
        }

继续上次测试:

此时符合收发一致。

除了select，还有 poll epoll处理事件的原理基本上差不多，不过在监听效率上要比select好的多（此处是在高并发的前提下）