QualNet收发包过程分析（一）

因项目需要，对QualNet仿真中节点间收发包行为进行了仔细研究，并不断添加一些输出以验证思路。应该说，大致是正确的，可能有些细节研究还不够透彻。对一般的应用协议开发已经够用了，如果想制作一个完整的组件添加进去，还需进一步深入。

以下以发送端节点向接收端节点发送应用层消息为例，介绍QualNet收发包过程。

1. 发送端

1.1 应用层

创建传输层UDP协议消息MSG_TRANSPORT_FromAppSend，添加了信息字段AppToUdpSend，发送（即调度传输层处理该消息）。函数见~/main/app_util.cpp

void APP_UdpSendNewHeaderData(
    Node *node,
    AppType appType,
    NodeAddress sourceAddr,
    short sourcePort,
    NodeAddress destAddr,
    char *header,
    int headerSize,
    char *payload,
    int payloadSize,
    clocktype delay,
    TraceProtocolType traceProtocol)
{
    Message *msg;
    AppToUdpSend *info;
    msg = MESSAGE_Alloc(
              node,
              TRANSPORT_LAYER,
              TransportProtocol_UDP,
              MSG_TRANSPORT_FromAppSend);
    MESSAGE_PacketAlloc(node, msg, payloadSize, traceProtocol);
    memcpy(MESSAGE_ReturnPacket(msg), payload, payloadSize);
    MESSAGE_AddHeader(node, msg, headerSize, traceProtocol);
    memcpy(MESSAGE_ReturnPacket(msg), header, headerSize);
    MESSAGE_InfoAlloc(node, msg, sizeof(AppToUdpSend));
    info = (AppToUdpSend *) MESSAGE_ReturnInfo(msg);
    SetIPv4AddressInfo(&info->sourceAddr, sourceAddr);
    info->sourcePort = sourcePort;
    SetIPv4AddressInfo(&info->destAddr, destAddr);
    info->destPort = (short) appType;
    info->priority = APP_DEFAULT_TOS;
    info->outgoingInterface = ANY_INTERFACE;
    info->ttl = IPDEFTTL;
    MESSAGE_Send(node, msg, delay);
}

1.2 传输层

应用层调度了传输层消息，由~/main/transport.cpp中消息处理函数TRANSPORT_ProcessEvent处理。如下所示，根据消息的协议类型选择具体协议处理，条理清晰。

void TRANSPORT_ProcessEvent(Node * node, Message * msg)
{
    switch (MESSAGE_GetProtocol(msg))
    {
        case TransportProtocol_UDP: 
        {
            TransportUdpLayer(node, msg);
            break;
        }
        case TransportProtocol_TCP: 
        {
            TransportTcpLayer(node, msg);
            break;
        }
        case TransportProtocol_RSVP:
        ......        
    }
}

UDP消息处理函数TransportUdpLayer如下所示，再根据消息类型选择具体处理方式。

void TransportUdpLayer(Node *node,  Message *msg)
{
    switch (msg->eventType)
    {
        case MSG_TRANSPORT_FromNetwork:
        {
            TransportUdpSendToApp(node, msg);
            break;
        }
        case MSG_TRANSPORT_FromAppSend:
        {
            TransportUdpSendToNetwork(node, msg);
            break;
        }
        default:
            assert(FALSE);
            abort();
    }
}

传输层向网络层发送数据，消息类型MSG_TRANSPORT_FromAppSend，调用函数TransportUdpSendToNetwork。在函数中添加了UDP首部TransportUdpHeader，调用网络层函数接收数据包。如下所示。

void TransportUdpSendToNetwork(Node *node, Message *msg)
{
    TransportDataUdp *udp = (TransportDataUdp *) node->transportData.udp;
    TransportUdpHeader *udpHdr;
    AppToUdpSend *info;
    unsigned char protocol = IPPROTO_UDP;
    info = (AppToUdpSend *) MESSAGE_ReturnInfo(msg);
    MESSAGE_AddHeader(node, msg, sizeof(TransportUdpHeader), TRACE_UDP);
    udpHdr = (TransportUdpHeader *) msg->packet;
    udpHdr->sourcePort = info->sourcePort;
    udpHdr->destPort = info->destPort;
    udpHdr->length = (unsigned short) MESSAGE_ReturnPacketSize(msg);
    udpHdr->checksum = 0;     
    //调用网络层函数接收数据
    NetworkIpReceivePacketFromTransportLayer(
        node,
        msg,
        info->sourceAddr,
        info->destAddr,
        info->outgoingInterface,
        info->priority,
        protocol,
        FALSE,
        info->ttl);
}

1.3 网络层

1.3.1 基本操作

~/libraries/developer/src/network_ip.cpp中，函数NetworkIpReceivePacketFromTransportLayer有两个，区别在于源和目的地址参数的类型，一个是NodeAddress，另一个是Address。消息的信息字段类型AppToUdpSend中，info->sourceAddress是Address型地址。调用函数NetworkIpReceivePacketFromTransportLayer，判断网络层协议类型，采取相应动作。如下所示：

void NetworkIpReceivePacketFromTransportLayer(
    Node *node,
    Message *msg,
    Address sourceAddress,
    Address destinationAddress,
    int outgoingInterface,
    TosType priority,
    unsigned char protocol,
    BOOL isEcnCapable,
    UInt8 ttl)
{
    NetworkDataIp *ip = (NetworkDataIp *) node->networkData.networkVar;
    ip->isPacketEcnCapable = isEcnCapable;
    //判断网络协议类型
    if (((node->networkData.networkProtocol == IPV4_ONLY ) ||
            (node->networkData.networkProtocol == DUAL_IP))
        && (sourceAddress.networkType == NETWORK_IPV4)
        && (destinationAddress.networkType == NETWORK_IPV4))
    {
        NetworkIpSendRawMessage(
            node,
            msg,
            GetIPv4Address(sourceAddress),
            GetIPv4Address(destinationAddress),
            outgoingInterface,
            priority,
            protocol,
            ttl,
	    FALSE);
    }
    else if (((node->networkData.networkProtocol == IPV6_ONLY) ||
                (node->networkData.networkProtocol == DUAL_IP))
             && (sourceAddress.networkType == NETWORK_IPV6)
             && (destinationAddress.networkType == NETWORK_IPV6))
    {
        ......           
    }
    ......
}

~/libraries/developer/src/network_ip.cpp中函数NetworkIpSendRawMessage，如下所示。先按指定参数确定出接口，如果参数是任意接口，则判断是否广播。非广播，按指向源地址的接口为出接口；广播，按目的地址找出接口。若不是任意接口，则按指定的接口来发送。第二步，添加IP首部（AddIpHeader函数调用MESSAGE_AddHeader添加首部IpHeaderType，不再赘述），第三步，调用路由协议发送数据包。在1.1中应用层添加的信息字段AppToUdpSend，info->outgoingInterface = ANY_INTERFACE，

void NetworkIpSendRawMessage(
    Node *node,
    Message *msg,
    NodeAddress sourceAddress,
    NodeAddress destinationAddress,
    int outgoingInterface,
    TosType priority,
    unsigned char protocol,
    unsigned ttl)
{
    NodeAddress newSourceAddress;
    int interfaceIndex;
    NetworkDataIp* ip = (NetworkDataIp *) node->networkData.networkVar;
    //由参数指定的出接口，查找接口索引
    if (outgoingInterface == ANY_INTERFACE)
    {        
        if (sourceAddress != ANY_IP)
        {
            interfaceIndex = NetworkIpGetInterfaceIndexFromAddress(node, sourceAddress);
            newSourceAddress = sourceAddress;
            if ((interfaceIndex == -1) || (newSourceAddress == (unsigned)-1))
            {
                MESSAGE_Free(node, msg);
                return;
            }
        }
        else
        {
            interfaceIndex = NetworkGetInterfaceIndexForDestAddress(node,destinationAddress);
            newSourceAddress = NetworkIpGetInterfaceAddress(node,interfaceIndex);
            if ((interfaceIndex == -1) || (newSourceAddress == (unsigned)-1))
            {
                MESSAGE_Free(node, msg);
                return;
            }
        }
    }
    else
    {
        interfaceIndex = outgoingInterface;
        newSourceAddress = sourceAddress;
    }
    AddIpHeader(
        node,
        msg,
        newSourceAddress,
        destinationAddress,
        priority,
        protocol,
        ttl);
    RoutePacketAndSendToMac(node, msg, CPU_INTERFACE, interfaceIndex, ANY_IP);
    }
}

1.3.2 区分发送类型

RoutePacketAndSendToMac函数很关键，在网络层和MAC层之间有承前启后的作用。先由入口接口判断是首发还是转发，再由目的地址判断是广播、组播还是单播，分别进行处理。参数传递：incomingInterface = CPU_INTERFACE, previousHopAddress = ANY_IP。

void RoutePacketAndSendToMac(Node *node,
                        Message *msg,
                        int incomingInterface,
                        int outgoingInterface,
                        NodeAddress previousHopAddress)
{
    NetworkDataIp *ip = (NetworkDataIp *) node->networkData.networkVar;
    NetworkForwardingTable* rt = &(ip->forwardTable);
    IpHeaderType *ipHeader = (IpHeaderType *) msg->packet;
    int outgoingInterfaceToUse;
    int interfaceIndex;
    NodeAddress outgoingBroadcastAddress;    
    //由入口接口判断是本地消息还是转发消息
    if (incomingInterface == CPU_INTERFACE)
    {
        //本节点发出的消息
        interfaceIndex = outgoingInterface;
    }
    else
    {
        //转发其他节点的消息.
        interfaceIndex = incomingInterface;
    }
    //由入口接口和目的地址判断网络行为
    //节点自环
    if (ip->isLoopbackEnabled && (incomingInterface == CPU_INTERFACE) &&
        NetworkIpLoopbackLoopbackUnicastsToSender(node, msg))
    {        
    }
    //以任意目的地址广播
    else if (ipHeader->ip_dst == ANY_DEST)
    {
        NetworkIpSendPacketOnInterface(
                          node,
                          msg,
                          incomingInterface,
                          outgoingInterface,
                          ipHeader->ip_dst);
    }
    //以出口接口的广播地址广播
    else if (IsOutgoingBroadcast(node,
             ipHeader->ip_dst,
             &outgoingInterfaceToUse,
             &outgoingBroadcastAddress))
    {       
        NetworkIpSendPacketOnInterface(node,
                                      msg,
                                      incomingInterface,
                                      outgoingInterfaceToUse,
                                      outgoingBroadcastAddress);

    }
    //利用组播路由进行组播
    else if (NetworkIpIsMulticastAddress(node, ipHeader->ip_dst))
    {
        ......  
    }
    //利用单播路由进行单播
    else
    {        
        BOOL packetWasRouted = FALSE;
        if (!packetWasRouted)
        {
            RouterFunctionType routerFunction = NULL;
            //查找接口路由协议
            routerFunction = NetworkIpGetRouterFunction(node,interfaceIndex);
            if (routerFunction)
            {
                //路由发送
                (routerFunction)(node,
                                 msg,
                                 ipHeader->ip_dst,
                                 previousHopAddress,
                                 &packetWasRouted);
            }
            if (!packetWasRouted)
            {
                //按转发表发送数据包
                RouteThePacketUsingLookupTable(node,
                                               msg,
                                               incomingInterface);
            }
        }
    }
}

以单播，接口路由协议为AODV4为例，NetworkIpGetRouterFunction将返回函数指针Aodv4RouterFunction（过程略），Aodv4RouterFunction函数调用AodvRouterFunction函数进行路由发送。参数传递：destAddr = ipHeader->ip_dst, previousHopAddress = ANY_IP, packetWasRouted = FALSE。

1.判断本节点地址是否是目的地址，是则不需路由，packetWasRouted = FALSE，不是则需要路由。

2.判断本节点地址是否源地址，是则查找路由发送，不是则调用AodvHandleData函数接收或转发。

3.查找路由，如果有就直接发送，没有则看是否已经发送路由请求信息。

4.没有路由情况，看是否已发送路由请求信息，未发送则立即发送，已发送说明确实没有此路由，数据包将无法发出。

void AodvRouterFunction(
    Node* node,
    Message* msg,
    Address destAddr,
    Address previousHopAddress,
    BOOL* packetWasRouted)
{
    AodvData* aodv=NULL;
    IpHeaderType* ipHeader = NULL;
    ip6_hdr* ip6Header = NULL;
    Address sourceAddress;
    BOOL IPV6 = FALSE;

    if (destAddr.networkType == NETWORK_IPV6)
    { }
    else
    {
        aodv = (AodvData *) NetworkIpGetRoutingProtocol(
                                node,
                                ROUTING_PROTOCOL_AODV,
                                NETWORK_IPV4);
        ipHeader = (IpHeaderType *) MESSAGE_ReturnPacket(msg);
        SetIPv4AddressInfo(&sourceAddress,ipHeader->ip_src);
    }
    //判断本节点是否目的地址，确定是否需要路由
    if (AodvIpIsMyIP(node, destAddr))
    {
        *packetWasRouted = FALSE;
    }
    else
    {
        *packetWasRouted = TRUE;
    }
    //判断本节点是路由的中间节点还是目的节点
    //本节点不是源节点
    if (!AodvIpIsMyIP(node, sourceAddress))
    {        
        //如果是目的节点就接收，如果是中间节点就转发
        AodvHandleData(node, msg, destAddr, previousHopAddress);
    }
    //本节点是源节点
    else
    {
        //不需要路由        
        if (!(*packetWasRouted)) //AodvIpIsMyIP(node, destAddr))
        {
            return;
        }
        //查看是否有至目的地址的路由
        rtToDest = AodvCheckRouteExist(
                       destAddr,
                       &aodv->routeTable,
                       &isValidRt);
        //有合法路由，则发送数据
        if (isValidRt)
        {            
            AodvTransmitData(node, msg, rtToDest, previousHopAddress);
        }
        //没有合法路由，且没有发送RREQ消息
        else if (!AodvCheckSent(destAddr, &aodv->sent))
        {            
            AodvInsertBuffer(
                node,
                msg,
                destAddr,
                previousHopAddress,
                &aodv->msgBuffer);            
            AodvInitiateRREQ(node, destAddr);
        }
        //没有合法路由，且RREQ已经发送
        else
        {               
            AodvInsertBuffer(
                node,
                msg,
                destAddr,
                previousHopAddress,
                &aodv->msgBuffer);
        }
    }
}

考虑节点首发情况，调用函数AodvTransmitData发送数据。更改消息层为MAC层，事件为MAG_MAC_FromNetwork，检查和更新路由有效时间，调用函数NetworkIpSendPacketToMacLayer发送数据。

static
void AodvTransmitData(
         Node* node,
         Message* msg,
         AodvRouteEntry* rtEntryToDest,
         Address previousHopAddress)
{
    AodvData* aodv = NULL;
    IpHeaderType* ipHeader = NULL;
    ip6_hdr* ip6Header = NULL;
    Address src;
    Address dst;
    BOOL IPV6 = FALSE;

    if (previousHopAddress.networkType == NETWORK_IPV6)
    { }
    else
    {
        aodv = (AodvData*)NetworkIpGetRoutingProtocol
                           (node, ROUTING_PROTOCOL_AODV,
                           NETWORK_IPV4);
        ipHeader = (IpHeaderType*) MESSAGE_ReturnPacket(msg);
        SetIPv4AddressInfo(&src,ipHeader->ip_src);
        SetIPv4AddressInfo(&dst,rtEntryToDest->destination.interfaceAddr.ipv4);
    }

    MESSAGE_SetLayer(msg, MAC_LAYER, 0);
    MESSAGE_SetEvent(msg, MSG_MAC_FromNetwork);

    //路由有效时间
    if (rtEntryToDest->lifetime < getSimTime(node) + AODV_ACTIVE_ROUTE_TIMEOUT)
    {
        rtEntryToDest->lifetime = getSimTime(node) + AODV_ACTIVE_ROUTE_TIMEOUT;
        AodvMoveRouteEntry(&aodv->routeTable, rtEntryToDest);
    }
    //更新路由有效时间
    if (((previousHopAddress.interfaceAddr.ipv4 != ANY_IP)
          && (previousHopAddress.networkType == NETWORK_IPV4))
          || ((!IS_MULTIADDR6(previousHopAddress.interfaceAddr.ipv6))
               && (previousHopAddress.networkType == NETWORK_IPV6)))
    {
        AodvUpdateLifetime(
            node,
            aodv,
            previousHopAddress,
            &aodv->routeTable,
            1);
    }
    if (!Address_IsSameAddress(&previousHopAddress,&src))
    {
        AodvUpdateLifetime(
            node,
            aodv,
            src,
            &aodv->routeTable,
            -1);
    }
    AodvUpdateLifetime(
        node,
        aodv,
        rtEntryToDest->nextHop,
        &aodv->routeTable,
        1);

    if (IPV6)
    { }
    else
    {
        NetworkIpSendPacketToMacLayer(
            node,
            msg,
            rtEntryToDest->outInterface,
            rtEntryToDest->nextHop.interfaceAddr.ipv4);
    }
}

1.3.3 分片处理 

NetworkIpSendPacketToMacLayer函数直接调用NetworkIpSendPacketOnInterface函数。在NetworkIpSendPacketOnInterface函数中，主要作用是判断是否需要分片传输，分片则逐片发送后删除原消息，不分片则将原消息调用NetworkIpSendOnBackplane函数直接发送。

void NetworkIpSendPacketOnInterface(
    Node *node,
    Message *msg,
    int incomingInterface,
    int outgoingInterface,
    NodeAddress nextHop)
{
    NetworkDataIp *ip = (NetworkDataIp *) node->networkData.networkVar;
    NetworkDataIcmp *icmp = (NetworkDataIcmp*) ip->icmpStruct;
    IpHeaderType *ipHeader = (IpHeaderType *) msg->packet;   
    int interfaceIndex;
    if (incomingInterface == CPU_INTERFACE)
        interfaceIndex = outgoingInterface;
    else
        interfaceIndex = incomingInterface;
    ipHeader = (IpHeaderType*) MESSAGE_ReturnPacket(msg);
    //获取接口所连通信信道最大传输字节数
    int fragInterface = NetworkIpGetSmallestFragUnitInterface(node);
    if (MESSAGE_ReturnPacketSize(msg) > GetNetworkIPFragUnit(node, fragInterface))
    {
        //看ip首部中是否允许分片，不允许，则发送ICMP消息
        if (IpHeaderGetIpDontFrag(ipHeader ->ipFragment))
        {
            if (ip->isIcmpEnable && icmp->fragmentationNeededEnable)
            {
               //发送ICMP消息
                BOOL ICMPErrorMsgCreated = NetworkIcmpCreateErrorMessage(node,
                                              msg,
                                              ipHeader->ip_src,
                                              incomingInterface,
                                              ICMP_DESTINATION_UNREACHABLE,
                                              ICMP_DGRAM_TOO_BIG,
                                              0,
                                              0);              
               MESSAGE_Free(node,msg);
               return;
            }
        }
    }
    //不分片情况
    if (!fragmentedByIP)
    {
        NetworkIpSendOnBackplane(node,
                                 msg,
                                 incomingInterface,
                                 outgoingInterface,
                                 nextHop);
    }
    //分片情况
    else
    {
        if (fragHead)
        {
            int fragId = 0 ;
            //逐片发送
            while (fragHead)
            {
                ipHeader = (IpHeaderType *) fragHead->msg->packet;                
                NetworkIpSendOnBackplane(node,
                                         fragHead->msg,
                                         incomingInterface,
                                         outgoingInterface,
                                         nextHop);

                tempFH = fragHead;
                fragHead = fragHead->next;
                MEM_free(tempFH);
            } 
            //分片发送后，原消息释放
            MESSAGE_Free(node, msg);
            return;
        }
    }
}

文章太长了，从NetworkIpSendOnBackplane函数开始，另文叙述。