Phxpaxos网络部分(4) —— TCP收发消息

TCP收消息

TCP收消息相关的类有TcpIOThread.TcpAcceptor.TcpRead 三个类。相互间的关系是：TcpIOThread初始化TcpRead和TcpAcceptor，并将后两者关联起来。 这样Acceptor接收到的连接直接放到TcpRead 中读取数据。

class TcpIOThread 
{
public:
    TcpIOThread(NetWork * poNetWork);
    ~TcpIOThread();

 //初始化建立相关的关联关系
    int Init(const std::string & sListenIp, const int iListenPort, const int iIOThreadCount);

    void Start();

    void Stop();

    int AddMessage(const int iGroupIdx, const std::string & sIP, const int iPort, const std::string & sMessage);

private:
    NetWork * m_poNetWork;
    TcpAcceptor m_oTcpAcceptor; //连接建立类
    std::vector<TcpRead *> m_vecTcpRead; //消息读取类
    std::vector<TcpWrite *> m_vecTcpWrite; //消息发送
    bool m_bIsStarted;
};

下面看一下他的初始化部分

int TcpIOThread :: Init(const std::string & sListenIp, const int iListenPort, const int iIOThreadCount)
{
//启动iIOThreadCount线程个读写线程
    for (int i = 0; i < iIOThreadCount; i++)
    {
        TcpRead * poTcpRead = new TcpRead(m_poNetWork);
        assert(poTcpRead != nullptr);
        m_vecTcpRead.push_back(poTcpRead);
        //将Read内部的EventLoop添加到Acceptor中，这样Acceptor接收消息之后可以直接将fd投放Read的EventLoop中直接使用 
        m_oTcpAcceptor.AddEventLoop(poTcpRead->GetEventLoop());

        TcpWrite * poTcpWrite = new TcpWrite(m_poNetWork);
        assert(poTcpWrite != nullptr);
        m_vecTcpWrite.push_back(poTcpWrite);
    }

//启动监听
    m_oTcpAcceptor.Listen(sListenIp, iListenPort);
    int ret = -1;

    for (auto & poTcpRead : m_vecTcpRead)
    {
    //初始化Read
        ret = poTcpRead->Init();
        if (ret != 0)
        {
            return ret;
        }
    }
.........
    return 0;
}

Acceptor

class TcpAcceptor : public Thread
{
public:
    TcpAcceptor();
    ~TcpAcceptor();
    //封装socket的listen函数
    void Listen(const std::string & sListenIP, const int iListenPort);
    //accept并将相应的fd，socketaddr创建Event添加到Event中
    void run();

    void Stop();

    //将Read中的EventLoop添加到私有m_vecEventLoop中
    void AddEventLoop(EventLoop * poEventLoop);
    //添加到最空闲的读线程
    void AddEvent(int iFD, SocketAddress oAddr);

private:
    ServerSocket m_oSocket;
    std::vector<EventLoop *> m_vecEventLoop;

private:
    bool m_bIsEnd;
    bool m_bIsStarted;
};

void TcpAcceptor :: Listen(const std::string & sListenIP, const int iListenPort)
{
    m_oSocket.listen(SocketAddress(sListenIP, (unsigned short)iListenPort));
}
void TcpAcceptor :: run()
{
    m_bIsStarted = true;

    PLHead("start accept...");

    m_oSocket.setAcceptTimeout(500);
    m_oSocket.setNonBlocking(true);

    while (true)
    {
        struct pollfd pfd;
        int ret;

        pfd.fd =  m_oSocket.getSocketHandle();
        pfd.events = POLLIN;
        ret = poll(&pfd, 1, 500);

        if (ret != 0 && ret != -1)
        {
            SocketAddress oAddr;
            int fd = -1;
            try
            {
            //accept函数封装
                fd = m_oSocket.acceptfd(&oAddr);
            }
            catch(...)
            {
                fd = -1;
            }

            if (fd >= 0)
            {
                BP->GetNetworkBP()->TcpAcceptFd();

                PLImp("accepted!, fd %d ip %s port %d",
                        fd, oAddr.getHost().c_str(), oAddr.getPort());
        //将新建立的连接添加到Read的EventLoop中
                AddEvent(fd, oAddr);
            }
        }

        if (m_bIsEnd)
        {
            PLHead("TCP.Acceptor [END]");
            return;
        }
    }
}

void TcpAcceptor :: AddEventLoop(EventLoop * poEventLoop)
{
    m_vecEventLoop.push_back(poEventLoop);
}

void TcpAcceptor :: AddEvent(int iFD, SocketAddress oAddr)
{
    EventLoop * poMinActiveEventLoop = nullptr;
    int iMinActiveEventCount = 1 << 30;
//挑选最空闲的EventLoop也就是最空闲的Read进行添加。空闲指的是监听的事件数量最少
    for (auto & poEventLoop : m_vecEventLoop)
    {
        int iActiveCount = poEventLoop->GetActiveEventCount();
        if (iActiveCount < iMinActiveEventCount)
        {
            iMinActiveEventCount = iActiveCount;
            poMinActiveEventLoop = poEventLoop;
        }
    }

    //printf("this %p addevent %p fd %d ip %s port %d\n", 
            //this, poMinActiveEventLoop, iFD, oAddr.getHost().c_str(), oAddr.getPort());
    poMinActiveEventLoop->AddEvent(iFD, oAddr);
}

OK,分析到这TCP接收的逻辑就全部完成了。Acceptor建立连接后将连接扔到监听事件最少Read的事件循环EventLoop中进行读事件的处理。若触发EPOLL则会接收信息并将信息通过NetWork的OnReceiveMessage将信息传递到Paxos的接收队列中。

TCP发消息

与Tcp发消息主要的类是TcpWrite和TcpClient。TcpClient是TcpWrite的私有成员，两者通过NetWork和EventLoop发生联系。其实TcpClient作用就是将发送消息和事件Event的添加封装起来。

class TcpClient
{
public:
    TcpClient(
            EventLoop * poEventLoop,
            NetWork * poNetWork);
    ~TcpClient();

    int AddMessage(const std::string & sIP, const int iPort, const std::string & sMessage);

    void DealWithWrite();

private:
    //从已经建立的Event中取出，不存在用下面的函数创建并返回
    MessageEvent * GetEvent(const std::string & sIP, const int iPort);

    MessageEvent * CreateEvent(const uint64_t llNodeID, const std::string & sIP, const int iPort);

private:
    EventLoop * m_poEventLoop;
    NetWork * m_poNetWork;

private:
    std::map<uint64_t, MessageEvent *> m_mapEvent;
    std::vector<MessageEvent *> m_vecEvent;
    std::mutex m_oMutex;

};

TcpClient :: TcpClient(EventLoop * poEventLoop, NetWork * poNetWork)
    : m_poEventLoop(poEventLoop), m_poNetWork(poNetWork)
{
   m_vecEvent.reserve(1000); 
}

TcpClient :: ~TcpClient()
{
    //只有在每一次析构时才释放相关的资源，而且貌似没有对添加EPOLLOUT事件的移除逻辑(待查)
    for (auto & it : m_mapEvent)
    {
        delete it.second;
    }
}

int TcpClient :: AddMessage(const std::string & sIP, const int iPort, const std::string & sMessage)
{
    //PLImp("ok");
    MessageEvent * poEvent = GetEvent(sIP, iPort);
    if (poEvent == nullptr)
    {
        PLErr("no event created for this ip %s port %d", sIP.c_str(), iPort);
        return -1;
    }

    return poEvent->AddMessage(sMessage);
}

MessageEvent * TcpClient :: GetEvent(const std::string & sIP, const int iPort)
{
    //根据IP和PORT生成NodeID,使用NodeID进行索引Event
    uint32_t iIP = (uint32_t)inet_addr(sIP.c_str());
    uint64_t llNodeID = (((uint64_t)iIP) << 32) | iPort;

    std::lock_guard<std::mutex> oLockGuard(m_oMutex);

    auto it = m_mapEvent.find(llNodeID);
    if (it != end(m_mapEvent))
    {
        return it->second;
    }

    return CreateEvent(llNodeID, sIP, iPort);
}

MessageEvent * TcpClient :: CreateEvent(const uint64_t llNodeID, const std::string & sIP, const int iPort)
{
    PLImp("start, ip %s port %d", sIP.c_str(), iPort);
   //不存在新建连接并将将其添加到EventLoop中
    Socket oSocket;
    oSocket.setNonBlocking(true);
    oSocket.setNoDelay(true);
    SocketAddress oAddr(sIP, iPort);
    oSocket.connect(oAddr);

    MessageEvent * poEvent = new MessageEvent(MessageEventType_SEND, oSocket.detachSocketHandle(), 
            oAddr, m_poEventLoop, m_poNetWork);
    assert(poEvent != nullptr);

    m_mapEvent[llNodeID] = poEvent;
    m_vecEvent.push_back(poEvent);

    PLImp("ok, ip %s port %d", sIP.c_str(), iPort);

    return poEvent;
}

void TcpClient :: DealWithWrite()
{
    //在每一次epoll_wait结束之后都会打开相应的EPOLLOUT事件，将要发送的数据发送出去。
    size_t iSize = m_vecEvent.size();

    for (size_t i = 0; i < iSize; i++)
    {   
        m_vecEvent[i]->OpenWrite();
    }
    //PLImp("end, live event count %zu", vecEventList.size());
}

OK TCP 发送逻辑结束
Phxpaxos网络部分相关的代码逻辑完成。
实现网络部分的基础帮助类没有进行相应的讲解主要在src/utils目录下面。如Timer,socket等封装。你可以自己看一下，也会有很大的收获。utils相关的测试代码在src/ut目录下面。