SRS流媒体服务器：服务器读取RTMP推流数据

目录

1. 处理RTMP推流video message

2. 处理RTMP推流audio message

3. 处理RTMP推流onMetaData message

本文是第三篇，第4篇将讲解、服务器给RTMP拉流端转发数据。请按照此顺序阅读：

SRS流媒体服务器：RTMP端口监听逻辑分析

SRS流媒体服务器：RTMP推流、拉流创建连接

1. 服务器读取RTMP推流数据

RTMP推流、拉流创建连接说到，SrsRtmpConn::publishing会创建SrsPublishRecvThread协程来接收数据，然后在SrsRtmpConn::do_publishing开启接收协程。

而最终会执行到SrsRecvThread::do_cycle，负责读取RTMP推流数据。

srs_error_t SrsRecvThread::do_cycle() {
    srs_error_t err = srs_success;

    while (true) {
        if ((err = trd->pull()) != srs_success) {
            return srs_error_wrap(err, "recv thread");
        }

        // When the pumper is interrupted, wait then retry.
        if (pumper->interrupted()) {
            srs_usleep(timeout);
            continue;
        }

        SrsCommonMessage *msg = NULL;

        // Process the received message. 读取msg并进行消费
        if ((err = rtmp->recv_message(&msg)) == srs_success) {
            err = pumper->consume(msg);
        }

        if (err != srs_success) {
            // Interrupt the receive thread for any error.
            trd->interrupt();

            // Notify the pumper to quit for error.
            pumper->interrupt(err);

            return srs_error_wrap(err, "recv thread");
        }
    }

    return err;
}

3.SrsRtmpServer::recv_message会获取message，如果获取的是协议控制消息或者用户控制消息会进行解码成packet然后进行响应处理，比如设置ackowledgement_window_size或者chunk_size。

如果是Aduio message，Video message或者Data message，不进行解码成packet，仅获取message。

TODO：后续写一篇关于如何接收message的分析，详细代码见SrsProtocol::recv_interlaced_message

srs_error_t SrsProtocol::recv_message(SrsCommonMessage** pmsg)
{
    *pmsg = NULL;
    
    srs_error_t err = srs_success;
    
    while (true) {
        SrsCommonMessage* msg = NULL;
        
        if ((err = recv_interlaced_message(&msg)) != srs_success) { //获取message
            srs_freep(msg);
            return srs_error_wrap(err, "recv interlaced message");
        }
        
        if (!msg) {
            continue;
        }
        
        if (msg->size <= 0 || msg->header.payload_length <= 0) {
            srs_trace("ignore empty message(type=%d, size=%d, time=%" PRId64 ", sid=%d).",
                      msg->header.message_type, msg->header.payload_length,
                      msg->header.timestamp, msg->header.stream_id);
            srs_freep(msg);
            continue;
        }
        
        if ((err = on_recv_message(msg)) != srs_success) { //对获取的message进行判断类型并进行相应处理
            srs_freep(msg);
            return srs_error_wrap(err, "on recv message");
        }
        
        *pmsg = msg;
        break;
    }
    
    return err;
}

4.SrsRtmpServer::recv_message会获取message后，在SrsPublishRecvThread::consume中进行处理，最后交由SrsRtmpConn::handle_publish_message进行处理。

srs_error_t SrsPublishRecvThread::consume(SrsCommonMessage *msg) {
    srs_error_t err = srs_success;

    // when cid changed, change it.
    if (ncid.compare(cid)) {
        _srs_context->set_id(ncid);
        cid = ncid;
    }

    _nb_msgs++;

    if (msg->header.is_video()) {
        video_frames++;
    }
    if (msg->header.is_audio()) {
        audio_frames++;
    }

    // log to show the time of recv thread.
    srs_verbose("recv thread now=%" PRId64 "us, got msg time=%" PRId64 "ms, size=%d",
                srs_update_system_time(), msg->header.timestamp, msg->size);

    // the rtmp connection will handle this message 由RTMP连接处理message
    err = _conn->handle_publish_message(_source, msg);

    // must always free it,
    // the source will copy it if need to use.
    srs_freep(msg);

    if (err != srs_success) {
        return srs_error_wrap(err, "handle publish message");
    }

    // Yield to another coroutines.
    // @see https://github.com/ossrs/srs/issues/2194#issuecomment-777463768
    if (++nn_msgs_for_yield_ >= 15) {
        nn_msgs_for_yield_ = 0;
        srs_thread_yield();
    }

    return err;
}

5.SrsRtmpConn::handle_publish_message会先判断message type是否是命令消息，如果是进行解码并进行处理，推流不需要这些事件数据。

6.audio、video以及data数据由SrsRtmpConn::process_publish_message进行处理。

srs_error_t SrsRtmpConn::handle_publish_message(SrsLiveSource* source, SrsCommonMessage* msg)
{
    srs_error_t err = srs_success;
    
    // process publish event. 处理推流事件，因为推流不需要这些事件数据，audio、video以及data数据不会进入此逻辑
    if (msg->header.is_amf0_command() || msg->header.is_amf3_command()) {
        SrsPacket* pkt = NULL;
        if ((err = rtmp->decode_message(msg, &pkt)) != srs_success) {
            return srs_error_wrap(err, "rtmp: decode message");
        }
        SrsAutoFree(SrsPacket, pkt);
        
        // for flash, any packet is republish.
        if (info->type == SrsRtmpConnFlashPublish) {
            // flash unpublish.
            // TODO: maybe need to support republish.
            srs_trace("flash flash publish finished.");
            return srs_error_new(ERROR_CONTROL_REPUBLISH, "rtmp: republish");
        }
        
        // for fmle, drop others except the fmle start packet.
        if (dynamic_cast<SrsFMLEStartPacket*>(pkt)) {
            SrsFMLEStartPacket* unpublish = dynamic_cast<SrsFMLEStartPacket*>(pkt);
            if ((err = rtmp->fmle_unpublish(info->res->stream_id, unpublish->transaction_id)) != srs_success) {
                return srs_error_wrap(err, "rtmp: republish");
            }
            return srs_error_new(ERROR_CONTROL_REPUBLISH, "rtmp: republish");
        }
        
        srs_trace("fmle ignore AMF0/AMF3 command message.");
        return err;
    }
    
    // video, audio, data message
    if ((err = process_publish_message(source, msg)) != srs_success) {
        return srs_error_wrap(err, "rtmp: consume message");
    }
    
    return err;
}

7.SrsRtmpConn::process_publish_message会判断message_type来区别audio、video和onMetaData来进行不同处理。

SrsLiveSource::on_audio负责处理音频message。

SrsLiveSource::on_video负责处理视频message。

SrsLiveSource::on_meta_data负责处理onMetaData message。

srs_error_t SrsRtmpConn::process_publish_message(SrsLiveSource* source, SrsCommonMessage* msg)
{
    srs_error_t err = srs_success;
    
    // for edge, directly proxy message to origin.
    if (info->edge) {
        if ((err = source->on_edge_proxy_publish(msg)) != srs_success) {
            return srs_error_wrap(err, "rtmp: proxy publish");
        }
        return err;
    }
    
    // process audio packet
    if (msg->header.is_audio()) {
        if ((err = source->on_audio(msg)) != srs_success) {
            return srs_error_wrap(err, "rtmp: consume audio");
        }
        return err;
    }
    // process video packet
    if (msg->header.is_video()) {
        if ((err = source->on_video(msg)) != srs_success) {
            return srs_error_wrap(err, "rtmp: consume video");
        }
        return err;
    }
    
    // process aggregate packet
    if (msg->header.is_aggregate()) {
        if ((err = source->on_aggregate(msg)) != srs_success) {
            return srs_error_wrap(err, "rtmp: consume aggregate");
        }
        return err;
    }
    
    // process onMetaData
    if (msg->header.is_amf0_data() || msg->header.is_amf3_data()) {
        SrsPacket* pkt = NULL;
        if ((err = rtmp->decode_message(msg, &pkt)) != srs_success) {
            return srs_error_wrap(err, "rtmp: decode message");
        }
        SrsAutoFree(SrsPacket, pkt);
        
        if (dynamic_cast<SrsOnMetaDataPacket*>(pkt)) {
            SrsOnMetaDataPacket* metadata = dynamic_cast<SrsOnMetaDataPacket*>(pkt);
            if ((err = source->on_meta_data(msg, metadata)) != srs_success) {
                return srs_error_wrap(err, "rtmp: consume metadata");
            }
            return err;
        }
        return err;
    }
    
    return err;
}

本文结尾底部，领取最新最全C++音视频学习提升资料，内容包括（C/C++，Linux 服务器开发，FFmpeg ，webRTC ，rtmp ，hls ，rtsp ，ffplay ，srs）↓↓↓↓↓↓文章底部

 

1. 处理RTMP推流video message

SrsLiveSource::on_video负责处理视频message。主要的功能为：
将SrsCommonMessage转换成SrsSharedPtrMessage，用于更少内存拷贝的音频/视频/数据message，SrsSharedPtrMessage使用共享计数方法。
将SrsSharedPtrMessage交给SrsLiveSource::on_video_imp处理。

srs_error_t SrsLiveSource::on_video(SrsCommonMessage* shared_video)
{
    srs_error_t err = srs_success;
    
    // monotically increase detect.
    if (!mix_correct && is_monotonically_increase) {
        if (last_packet_time > 0 && shared_video->header.timestamp < last_packet_time) {
            is_monotonically_increase = false;
            srs_warn("VIDEO: stream not monotonically increase, please open mix_correct.");
        }
    }
    last_packet_time = shared_video->header.timestamp;
    
    // drop any unknown header video.
    // @see https://github.com/ossrs/srs/issues/421
    if (!SrsFlvVideo::acceptable(shared_video->payload, shared_video->size)) {
        char b0 = 0x00;
        if (shared_video->size > 0) {
            b0 = shared_video->payload[0];
        }
        
        srs_warn("drop unknown header video, size=%d, bytes[0]=%#x", shared_video->size, b0);
        return err;
    }
    
    // convert shared_video to msg, user should not use shared_video again.
    // the payload is transfer to msg, and set to NULL in shared_video.
    SrsSharedPtrMessage msg; //创建共享message ptr，将SrsCommonMessage转换成SrsSharedPtrMessage，用于更少内存拷贝的音频/视频/数据message
    if ((err = msg.create(shared_video)) != srs_success) { //SrsSharedPtrMessage使用引用计数
        return srs_error_wrap(err, "create message");
    }
    
    // directly process the video message. mix_correct为false，直接处理video message
    if (!mix_correct) {
        return on_video_imp(&msg);
    }
    
    // insert msg to the queue. 如果mix_correct为true，先将msg加入按时间戳排序map，然后从中取出第一个msg进行处理
    mix_queue->push(msg.copy());
    
    // fetch someone from mix queue. 取出的msg可能是音频或视频，所以需要进行判断
    SrsSharedPtrMessage* m = mix_queue->pop(); 
    if (!m) {
        return err;
    }
    
    // consume the monotonically increase message. 
    if (m->is_audio()) {
        err = on_audio_imp(m);
    } else {
        err = on_video_imp(m);
    }
    srs_freep(m);
    
    return err;
}

2.SrsLiveSource::on_video_imp处理SrsSharedPtrMessage主要过程包括：

1. 遍历对应source拉流客户端，将msg发送给每个客户端。

2. 如果不是sequence header，加入gop缓存中，gop会缓存最后一个gop的packet。

srs_error_t SrsLiveSource::on_video_imp(SrsSharedPtrMessage* msg)
{
    srs_error_t err = srs_success;
    
    bool is_sequence_header = SrsFlvVideo::sh(msg->payload, msg->size);
    
    // whether consumer should drop for the duplicated sequence header.
    bool drop_for_reduce = false;
    if (is_sequence_header && meta->previous_vsh() && _srs_config->get_reduce_sequence_header(req->vhost)) {
        if (meta->previous_vsh()->size == msg->size) {
            drop_for_reduce = srs_bytes_equals(meta->previous_vsh()->payload, msg->payload, msg->size);
            srs_warn("drop for reduce sh video, size=%d", msg->size);
        }
    }
    
    // cache the sequence header if h264
    // donot cache the sequence header to gop_cache, return here.
    if (is_sequence_header && (err = meta->update_vsh(msg)) != srs_success) {
        return srs_error_wrap(err, "meta update video");
    }
    
    // Copy to hub to all utilities.
    if ((err = hub->on_video(msg, is_sequence_header)) != srs_success) {
        return srs_error_wrap(err, "hub consume video");
    }

    // For bridger to consume the message.
    if (bridger_ && (err = bridger_->on_video(msg)) != srs_success) {
        return srs_error_wrap(err, "bridger consume video");
    }

    // copy to all consumer 遍历对应source拉流客户端，将msg发送给每个客户端
    if (!drop_for_reduce) {
        for (int i = 0; i < (int)consumers.size(); i++) {
            SrsLiveConsumer* consumer = consumers.at(i);
            if ((err = consumer->enqueue(msg, atc, jitter_algorithm)) != srs_success) {
                return srs_error_wrap(err, "consume video");
            }
        }
    }
    
    // when sequence header, donot push to gop cache and adjust the timestamp.
    if (is_sequence_header) {
        return err;
    }
    
    // cache the last gop packets
    if ((err = gop_cache->cache(msg)) != srs_success) {
        return srs_error_wrap(err, "gop cache consume vdieo");
    }
    
    // if atc, update the sequence header to abs time.
    if (atc) {
        if (meta->vsh()) {
            meta->vsh()->timestamp = msg->timestamp;
        }
        if (meta->data()) {
            meta->data()->timestamp = msg->timestamp;
        }
    }
    
    return err;
}

3.每个拉流客户端是一个SrsLiveConsumer，会将msg加入SrsMessageQueue对象中，实际保存的是SrsMessageQueue对象中的SrsFastVector对象中的SrsSharedPtrMessage数组。

srs_error_t SrsLiveConsumer::enqueue(SrsSharedPtrMessage* shared_msg, bool atc, SrsRtmpJitterAlgorithm ag)
{
    srs_error_t err = srs_success;
    
    SrsSharedPtrMessage* msg = shared_msg->copy();

    if (!atc) {
        if ((err = jitter->correct(msg, ag)) != srs_success) {
            return srs_error_wrap(err, "consume message");
        }
    }

    if ((err = queue->enqueue(msg, NULL)) != srs_success) {
        return srs_error_wrap(err, "enqueue message");
    }
    
	...
    
    return err;
}

srs_error_t SrsMessageQueue::enqueue(SrsSharedPtrMessage* msg, bool* is_overflow)
{
    srs_error_t err = srs_success;

    msgs.push_back(msg);

    // If jitter is off, the timestamp of first sequence header is zero, which wll cause SRS to shrink and drop the
    // keyframes even if there is not overflow packets in queue, so we must ignore the zero timestamps, please
    // @see https://github.com/ossrs/srs/pull/2186#issuecomment-953383063
    if (msg->is_av() && msg->timestamp != 0) {
        if (av_start_time == -1) {
            av_start_time = srs_utime_t(msg->timestamp * SRS_UTIME_MILLISECONDS);
        }
        
        av_end_time = srs_utime_t(msg->timestamp * SRS_UTIME_MILLISECONDS);
    }

    if (max_queue_size <= 0) {
        return err;
    }

    while (av_end_time - av_start_time > max_queue_size) {
        // notice the caller queue already overflow and shrinked.
        if (is_overflow) {
            *is_overflow = true;
        }
        
        shrink();
    }
    
    return err;
}

void SrsFastVector::push_back(SrsSharedPtrMessage* msg)
{
    // increase vector.
    if (count >= nb_msgs) {
        int size = srs_max(SRS_PERF_MW_MSGS * 8, nb_msgs * 2);
        SrsSharedPtrMessage** buf = new SrsSharedPtrMessage*[size];
        for (int i = 0; i < nb_msgs; i++) {
            buf[i] = msgs[i];
        }
        srs_info("fast vector incrase %d=>%d", nb_msgs, size);
        
        // use new array.
        srs_freepa(msgs);
        msgs = buf;
        nb_msgs = size;
    }
    
    msgs[count++] = msg;
}

2. 处理RTMP推流audio message

SrsLiveSource::on_audio负责处理音频message。主要的功能跟处理video message一样，为：
将SrsCommonMessage转换成SrsSharedPtrMessage，用于更少内存拷贝的音频/视频/数据message，SrsSharedPtrMessage使用共享计数方法。
将SrsSharedPtrMessage交给SrsLiveSource::on_audio_imp处理。

srs_error_t SrsLiveSource::on_audio(SrsCommonMessage* shared_audio)
{
    srs_error_t err = srs_success;
    
    // monotically increase detect.
    if (!mix_correct && is_monotonically_increase) {
        if (last_packet_time > 0 && shared_audio->header.timestamp < last_packet_time) {
            is_monotonically_increase = false;
            srs_warn("AUDIO: stream not monotonically increase, please open mix_correct.");
        }
    }
    last_packet_time = shared_audio->header.timestamp;
    
    // convert shared_audio to msg, user should not use shared_audio again.
    // the payload is transfer to msg, and set to NULL in shared_audio.
    SrsSharedPtrMessage msg;
    if ((err = msg.create(shared_audio)) != srs_success) {
        return srs_error_wrap(err, "create message");
    }
    
    // directly process the audio message.
    if (!mix_correct) {
        return on_audio_imp(&msg);
    }
    
    // insert msg to the queue.
    mix_queue->push(msg.copy());
    
    // fetch someone from mix queue.
    SrsSharedPtrMessage* m = mix_queue->pop();
    if (!m) {
        return err;
    }
    
    // consume the monotonically increase message.
    if (m->is_audio()) {
        err = on_audio_imp(m);
    } else {
        err = on_video_imp(m);
    }
    srs_freep(m);
    
    return err;
}

2.SrsLiveSource::on_audio_imp处理SrsSharedPtrMessage主要过程包括：

1. 遍历对应source拉流客户端，将msg发送给每个客户端。

2. 如果不是sequence header，加入gop缓存中，gop会缓存最后一个gop的packet。

srs_error_t SrsLiveSource::on_audio_imp(SrsSharedPtrMessage* msg)
{
    srs_error_t err = srs_success;
    
    bool is_aac_sequence_header = SrsFlvAudio::sh(msg->payload, msg->size);
    bool is_sequence_header = is_aac_sequence_header;
    
    // whether consumer should drop for the duplicated sequence header.
    bool drop_for_reduce = false;
    if (is_sequence_header && meta->previous_ash() && _srs_config->get_reduce_sequence_header(req->vhost)) {
        if (meta->previous_ash()->size == msg->size) {
            drop_for_reduce = srs_bytes_equals(meta->previous_ash()->payload, msg->payload, msg->size);
            srs_warn("drop for reduce sh audio, size=%d", msg->size);
        }
    }
    
    // Copy to hub to all utilities.
    if ((err = hub->on_audio(msg)) != srs_success) {
        return srs_error_wrap(err, "consume audio");
    }

    // For bridger to consume the message.
    if (bridger_ && (err = bridger_->on_audio(msg)) != srs_success) {
        return srs_error_wrap(err, "bridger consume audio");
    }

    // copy to all consumer 遍历对应source拉流客户端，将msg发送给每个客户端
    if (!drop_for_reduce) {
        for (int i = 0; i < (int)consumers.size(); i++) {
            SrsLiveConsumer* consumer = consumers.at(i);
            if ((err = consumer->enqueue(msg, atc, jitter_algorithm)) != srs_success) {
                return srs_error_wrap(err, "consume message");
            }
        }
    }
    
    // cache the sequence header of aac, or first packet of mp3.
    // for example, the mp3 is used for hls to write the "right" audio codec.
    // TODO: FIXME: to refine the stream info system.
    if (is_aac_sequence_header || !meta->ash()) {
        if ((err = meta->update_ash(msg)) != srs_success) {
            return srs_error_wrap(err, "meta consume audio");
        }
    }
    
    // when sequence header, donot push to gop cache and adjust the timestamp.
    if (is_sequence_header) {
        return err;
    }
    
    // cache the last gop packets
    if ((err = gop_cache->cache(msg)) != srs_success) {
        return srs_error_wrap(err, "gop cache consume audio");
    }

    // if atc, update the sequence header to abs time.
    if (atc) {
        if (meta->ash()) {
            meta->ash()->timestamp = msg->timestamp;
        }
        if (meta->data()) {
            meta->data()->timestamp = msg->timestamp;
        }
    }
    
    return err;
}

3.每个拉流客户端是一个SrsLiveConsumer，会将msg加入SrsMessageQueue对象中，实际保存的是SrsMessageQueue对象中的SrsFastVector对象中的SrsSharedPtrMessage数组。跟video存储逻辑一样。

3. 处理RTMP推流onMetaData message

对于onMetaData message，SrsRtmpConn::process_publish_message会先解码成SrsOnMetaDataPacket，然后和onMetaData message一起作为参数传入SrsLiveSource::on_meta_data。主要的功能有：

更新onMetaData message。即更新SrsMetaCache的meta（SrsSharedPtrMessage）

遍历对应source拉流客户端，将共享meta发送给每个客户端。

srs_error_t SrsLiveSource::on_meta_data(SrsCommonMessage* msg, SrsOnMetaDataPacket* metadata)
{
    srs_error_t err = srs_success;
    
    // if allow atc_auto and bravo-atc detected, open atc for vhost.
    SrsAmf0Any* prop = NULL;
    atc = _srs_config->get_atc(req->vhost);
    if (_srs_config->get_atc_auto(req->vhost)) {
        if ((prop = metadata->metadata->get_property("bravo_atc")) != NULL) {
            if (prop->is_string() && prop->to_str() == "true") {
                atc = true;
            }
        }
    }
    
    // Update the meta cache. 如果需要更新MetaData，会更新共享SrsSharedPtrMessage* meta
    bool updated = false;
    if ((err = meta->update_data(&msg->header, metadata, updated)) != srs_success) {
        return srs_error_wrap(err, "update metadata");
    }
    if (!updated) {
        return err;
    }
    
    // when already got metadata, drop when reduce sequence header.
    bool drop_for_reduce = false;
    if (meta->data() && _srs_config->get_reduce_sequence_header(req->vhost)) {
        drop_for_reduce = true;
        srs_warn("drop for reduce sh metadata, size=%d", msg->size);
    }
    
    // copy to all consumer 遍历对应source拉流客户端，将共享meta发送给每个客户端。
    if (!drop_for_reduce) {
        std::vector<SrsLiveConsumer*>::iterator it;
        for (it = consumers.begin(); it != consumers.end(); ++it) {
            SrsLiveConsumer* consumer = *it;
            if ((err = consumer->enqueue(meta->data(), atc, jitter_algorithm)) != srs_success) {
                return srs_error_wrap(err, "consume metadata");
            }
        }
    }
    
    // Copy to hub to all utilities.
    return hub->on_meta_data(meta->data(), metadata);
}

2.每个拉流客户端是一个SrsLiveConsumer，会将msg加入SrsMessageQueue对象中，实际保存的是SrsMessageQueue对象中的SrsFastVector对象中的SrsSharedPtrMessage数组。跟video和audio的存储逻辑一样。

原文链接