Why use epoll
This is nothing new, words, multiplexing, or a single thread can only be stopped and other queue. Also select and poll epoll not as powerful easy to use.
Talk Program Structure
Code is very simple, basically a toy. But still just talk.
Epoll model used in a single thread, only one instance epoll while listening socket descriptor and the connection descriptor. When the socket descriptor is in place, it calls accept processing three-way handshake to establish a connection, while calling epoll_ctl this connfd join epoll event listener table. If connfd in place, call the recv function to read data from the buffer and then send back intact.
epoll two modes and Precautions
epoll have LT (Level-Trigger) and ET (Edge-Trigger) mode, for simplicity herein, the LT model.
When using LT models, sockfd and connfd do not need to set non-blocking (non-blocking) mode, but using ET mode when you need to connfd set to non-blocking mode , let's focused on this issue.
If ET mode "must" use connfd set to non-blocking mode of statement is not rigorous, it should be said, "not recommended" or "not suitable."
In edge-triggered mode, EPOLLIN must be to trigger at the end of the new data is written after the event is triggered if a EPOLLIN, user code did not take out all of the data in the buffer, it can only wait for the time to write the next end in, in order to put together a missed taken out.
Imagine such a case, the service and the client during a question and answer two-way communication, the server needs to resolve the client's complete message, then return information at the request of the client. After this process is complete, the client can be the next request. If the situation described above occurs (edge-triggered and there is no complete reading data), then the server will not give the data back to the client, the client can not receive data, naturally there will be no next time for the end of write to form a bi-waiting, quite similar deadlock.
Performance Testing
Because it is a single-threaded echo server, for convenience using a multi-threaded client data read and write cycles. Note that the use of single-threaded client makes no sense , because the single-threaded client is difficult to put pressure on the single-threaded server, to test the greatest qps, you must use a multithreaded client.
The recording of the measured amount QPS different threads.
You can see a small number of threads when unable to test out the more real QPS, because they can not cause the maximum load on the server.
Server Code
// SimpleEpollEcho.h
#include <sys/epoll.h>
#include <sys/socket.h>
#include <netinet/in.h>
#include <arpa/inet.h>
#include <vector>
class SimpleEpollEcho {
public:
SimpleEpollEcho(bool enable_et);
~SimpleEpollEcho();
int init();
void start_lt();
private:
int SetNonblocking(int fd);
bool enable_et;
int sockfd;
int epfd;
const char *ip = "localhost";
const static int port = 8888;
const static int MAX_EPOLL_EVENT_NUM = 1024;
const static int BUFFER_SIZE = 16;
epoll_event events[MAX_EPOLL_EVENT_NUM];
char buf[BUFFER_SIZE];
};// SimpleEpollEcho.cpp
// use Level-Trigger mode, echo message must be 16bytes long. the connfd is in blocking mode
//
#include <strings.h>
#include <cassert>
#include <iostream>
#include <unistd.h>
#include <fcntl.h>
#include "SimpleEpollEcho.h"
SimpleEpollEcho::SimpleEpollEcho(bool enable_et) : enable_et(enable_et)
{
}
SimpleEpollEcho::~SimpleEpollEcho()
{
}
int SimpleEpollEcho::init()
{
int ret;
do
{
struct sockaddr_in address;
bzero(&address, sizeof(address));
address.sin_family = AF_INET;
inet_pton( AF_INET, ip, &address.sin_addr );
address.sin_port = htons(port);
sockfd = socket(PF_INET, SOCK_STREAM, 0);
assert(sockfd >= 0);
int on = 1;
setsockopt(sockfd, SOL_SOCKET, SO_REUSEADDR, &on, sizeof(on));
//SetNonblocking(sockfd);
ret = bind(sockfd, (struct sockaddr *)&address, sizeof(address));
assert(ret != -1);
ret = listen(sockfd, 5);
assert(ret != -1);
epfd = epoll_create(5);
assert(epfd != -1);
// add socket fd to epoll fd
{
epoll_event e;
e.events = EPOLLIN;
if(enable_et)
e.events |= EPOLLET;
e.data.fd = sockfd;
epoll_ctl(epfd, EPOLL_CTL_ADD, sockfd, &e);
}
}
while(0);
}
void SimpleEpollEcho::start_lt()
{
while(1)
{
int ready = epoll_wait(epfd, events, MAX_EPOLL_EVENT_NUM, -1);
if(ready < 0)
{
std::cout << "epoll_wait error" << std::endl;
break;
}
for(int i = 0; i < ready; i++)
{
// the sockfd in epoll table will be never removed
// so if there comes a new connection, we can accept it and get the connfd
if((events[i].data.fd == sockfd) && (events[i].events & EPOLLIN))
{
struct sockaddr_in client_address;
socklen_t client_addrlength = sizeof( client_address );
int connfd = accept(sockfd, (struct sockaddr *)&client_address, &client_addrlength);
{
epoll_event e;
e.events = EPOLLIN;
if(enable_et)
e.events |= EPOLLET;
e.data.fd = connfd;
//SetNonblocking(connfd);
epoll_ctl(epfd, EPOLL_CTL_ADD, connfd, &e);
}
}
// if there comes new data of some connection
// now we read the connection data echo it
else if((events[i].data.fd != sockfd) && (events[i].events & EPOLLIN))
{
int connfd = events[i].data.fd;
// if flags is 0, recv behave just like read on a socket fd
// with Level-Trigger Mode, it's ok to not read all data in buffer
// because if there's data remained, kernel will remind us again
// however, this requires every connection a index and buffer
// to store their data. In single-thread program, I suppose it's not good
// use multi-thread stack to store that may be more reasonable
// so in single thread, we read it all
char *pbuf = buf;
int recvn = 0, sendn = 0;
int recv_len = BUFFER_SIZE;
while(recvn < recv_len)
{
recvn += recv(connfd, pbuf, recv_len - recvn, 0);
pbuf += recvn;
}
pbuf = buf;
while(sendn < recvn)
{
sendn += send(connfd, pbuf, recvn - sendn, 0);
pbuf += sendn;
}
if(buf[0] == '0')
{
close(connfd);
epoll_ctl(epfd, EPOLL_CTL_DEL, connfd, NULL);
}
}
}
}
}
int SimpleEpollEcho::SetNonblocking(int fd) {
int old_option = fcntl( fd, F_GETFL );
int new_option = old_option | O_NONBLOCK;
fcntl( fd, F_SETFL, new_option );
return old_option;
}
Client Code
#include <iostream>
#include <cassert>
#include <pthread.h>
#include <sys/socket.h>
#include <unistd.h>
#include <sys/types.h>
#include <netinet/in.h>
#include <sys/time.h>
#include <arpa/inet.h>
#include <errno.h>
#include <strings.h>
const int MAX_BUF_LEN = 16;
char normal_buf[MAX_BUF_LEN] = "12345678901234\n";
char exit_buf[MAX_BUF_LEN] = "01234567980123\n";
const int MAX_THREAD_NUM = 64;
void* thread_test(void* ptr)
{
printf("thread start\r\n");
struct timeval tv_before;
struct timeval tv_after;
struct timeval tv_total_gap;
struct timezone tz;
struct sockaddr_in servaddr;
bzero(&servaddr, sizeof(servaddr));
servaddr.sin_family = AF_INET;
servaddr.sin_port = htons(8888);
inet_pton( AF_INET, "localhost", &servaddr.sin_addr );
socklen_t socklen = sizeof (servaddr);
int sockfd = socket(AF_INET,SOCK_STREAM, 0 );
assert(sockfd > 0);
connect(sockfd, (const struct sockaddr *)&servaddr, sizeof(servaddr));
int x = errno;
for(int j = 0; j < 10; j++)
{
gettimeofday(&tv_before, &tz);
for(int i = 0; i < 10000; i++)
{
send(sockfd, normal_buf, MAX_BUF_LEN, MSG_WAITALL);
recv(sockfd, normal_buf, MAX_BUF_LEN, MSG_WAITALL);
}
gettimeofday(&tv_after, &tz);
tv_total_gap.tv_usec += tv_after.tv_usec - tv_before.tv_usec;
tv_total_gap.tv_sec += tv_after.tv_sec - tv_before.tv_sec;
}
send(sockfd, exit_buf, MAX_BUF_LEN, MSG_WAITALL);
recv(sockfd, exit_buf, MAX_BUF_LEN, MSG_WAITALL);
printf("total second gap, %ld, total usecond gap: %ld\r\n", tv_total_gap.tv_sec, tv_total_gap.tv_usec);
close(sockfd);
}
int main()
{
pthread_t threads[MAX_THREAD_NUM];
for(int i = 0; i< MAX_THREAD_NUM; i++)
{
pthread_create(&threads[i], NULL, thread_test, NULL);
}
for(int i = 0; i< MAX_THREAD_NUM; i++)
{
pthread_join(threads[i], NULL);
}
return 0;
}