MediaStreamソースコード分析（3）

Licodeサービスと起動プロセスの分析

MediaStreamソースコード分析（1）

MediaStreamソースコード分析（2）

MediaStreamソースコード分析（3）

WebRtcConnectionソースコード分析（1）

WebRtcConnectionソースコード分析（2）

WebRtcConnectionソースコード分析（3）

MediaStreamを理解するのは少し難しいです：

// pipeline_读取传进来的packet
void MediaStream::onTransportData(std::shared_ptr<DataPacket> incoming_packet, Transport *transport) {
  if ((audio_sink_ == nullptr && video_sink_ == nullptr && fb_sink_ == nullptr)) {
    return;
  }

  std::shared_ptr<DataPacket> packet = std::make_shared<DataPacket>(*incoming_packet);

  if (transport->mediaType == AUDIO_TYPE) {
    packet->type = AUDIO_PACKET;
  } else if (transport->mediaType == VIDEO_TYPE) {
    packet->type = VIDEO_PACKET;
  }
  auto stream_ptr = shared_from_this();

  worker_->task([stream_ptr, packet]{
    if (!stream_ptr->pipeline_initialized_) {
      ELOG_DEBUG("%s message: Pipeline not initialized yet.", stream_ptr->toLog());
      return;
    }

    char* buf = packet->data;
    RtpHeader *head = reinterpret_cast<RtpHeader*> (buf);
    RtcpHeader *chead = reinterpret_cast<RtcpHeader*> (buf);
    if (!chead->isRtcp()) {
      uint32_t recvSSRC = head->getSSRC();
      if (stream_ptr->isVideoSourceSSRC(recvSSRC)) {
        packet->type = VIDEO_PACKET;
      } else if (stream_ptr->isAudioSourceSSRC(recvSSRC)) {
        packet->type = AUDIO_PACKET;
      }
    }

    if (stream_ptr->pipeline_) {
      stream_ptr->pipeline_->read(std::move(packet));
    }
  });
}

// 读取packet,通过audio_sink_、video_sink_和fb_sink_的deliver函数发送出去
void MediaStream::read(std::shared_ptr<DataPacket> packet) {
  char* buf = packet->data;
  int len = packet->length;
  // PROCESS RTCP
  RtpHeader *head = reinterpret_cast<RtpHeader*> (buf);
  RtcpHeader *chead = reinterpret_cast<RtcpHeader*> (buf);
  uint32_t recvSSRC = 0;
  if (!chead->isRtcp()) {
    recvSSRC = head->getSSRC();
  } else if (chead->packettype == RTCP_Sender_PT || chead->packettype == RTCP_SDES_PT) {  // Sender Report
    recvSSRC = chead->getSSRC();
  }
  // DELIVER FEEDBACK (RR, FEEDBACK PACKETS)
  if (chead->isFeedback()) {
    if (fb_sink_ != nullptr && should_send_feedback_) {
      fb_sink_->deliverFeedback(std::move(packet));
    }
  } else {
    // RTP or RTCP Sender Report
    if (bundle_) {
      // Check incoming SSRC
      // Deliver data
      if (isVideoSourceSSRC(recvSSRC) && video_sink_) {
        parseIncomingPayloadType(buf, len, VIDEO_PACKET);
        parseIncomingExtensionId(buf, len, VIDEO_PACKET);
        video_sink_->deliverVideoData(std::move(packet));
      } else if (isAudioSourceSSRC(recvSSRC) && audio_sink_) {
        parseIncomingPayloadType(buf, len, AUDIO_PACKET);
        parseIncomingExtensionId(buf, len, AUDIO_PACKET);
        audio_sink_->deliverAudioData(std::move(packet));
      } else {
        ELOG_DEBUG("%s read video unknownSSRC: %u, localVideoSSRC: %u, localAudioSSRC: %u",
                    toLog(), recvSSRC, this->getVideoSourceSSRC(), this->getAudioSourceSSRC());
      }
    } else {
      if (packet->type == AUDIO_PACKET && audio_sink_) {
        parseIncomingPayloadType(buf, len, AUDIO_PACKET);
        parseIncomingExtensionId(buf, len, AUDIO_PACKET);
        // Firefox does not send SSRC in SDP
        if (getAudioSourceSSRC() == 0) {
          ELOG_DEBUG("%s discoveredAudioSourceSSRC:%u", toLog(), recvSSRC);
          this->setAudioSourceSSRC(recvSSRC);
        }
        audio_sink_->deliverAudioData(std::move(packet));
      } else if (packet->type == VIDEO_PACKET && video_sink_) {
        parseIncomingPayloadType(buf, len, VIDEO_PACKET);
        parseIncomingExtensionId(buf, len, VIDEO_PACKET);
        // Firefox does not send SSRC in SDP
        if (getVideoSourceSSRC() == 0) {
          ELOG_DEBUG("%s discoveredVideoSourceSSRC:%u", toLog(), recvSSRC);
          this->setVideoSourceSSRC(recvSSRC);
        }
        // change ssrc for RTP packets, don't touch here if RTCP
        video_sink_->deliverVideoData(std::move(packet));
      }
    }  // if not bundle
  }  // if not Feedback
}

// 设置media_stream_event_listener_ 
void MediaStream::setMediaStreamEventListener(MediaStreamEventListener* listener) {
  boost::mutex::scoped_lock lock(event_listener_mutex_);
  this->media_stream_event_listener_ = listener;
}

// media_stream_event_listener_回调通知事件
void MediaStream::notifyMediaStreamEvent(const std::string& type, const std::string& message) {
  boost::mutex::scoped_lock lock(event_listener_mutex_);
  if (this->media_stream_event_listener_ != nullptr) {
    media_stream_event_listener_->notifyMediaStreamEvent(type, message);
  }
}

// event_sink_通知事件
void MediaStream::notifyToEventSink(MediaEventPtr event) {
  event_sink_->deliverEvent(std::move(event));
}

// 发送PLI包，重新发送关键帧的请求
int MediaStream::sendPLI() {
  RtcpHeader thePLI;
  thePLI.setPacketType(RTCP_PS_Feedback_PT);
  thePLI.setBlockCount(1);
  thePLI.setSSRC(this->getVideoSinkSSRC());
  thePLI.setSourceSSRC(this->getVideoSourceSSRC());
  thePLI.setLength(2);
  char *buf = reinterpret_cast<char*>(&thePLI);
  int len = (thePLI.getLength() + 1) * 4;
  sendPacketAsync(std::make_shared<DataPacket>(0, buf, len, VIDEO_PACKET));
  return len;
}

// 通过fb_sink_将PLI请求发送出去
void MediaStream::sendPLIToFeedback() {
  if (fb_sink_) {
    fb_sink_->deliverFeedback(RtpUtils::createPLI(this->getVideoSinkSSRC(),
      this->getVideoSourceSSRC()));
  }
}

// 异步发送数据包，改变了每个输出包的载荷类型和扩展字
// changes the outgoing payload type for in the given data packet
void MediaStream::sendPacketAsync(std::shared_ptr<DataPacket> packet) {
  if (!sending_) {
    return;
  }
  auto stream_ptr = shared_from_this();
  if (packet->comp == -1) {
    sending_ = false;
    auto p = std::make_shared<DataPacket>();
    p->comp = -1;
    worker_->task([stream_ptr, p]{
      stream_ptr->sendPacket(p);
    });
    return;
  }

  changeDeliverPayloadType(packet.get(), packet->type);
  changeDeliverExtensionId(packet.get(), packet->type);
  worker_->task([stream_ptr, packet]{
    stream_ptr->sendPacket(packet);
  });
}

// 设置滑动条显示模式
void MediaStream::setSlideShowMode(bool state) {
  ELOG_DEBUG("%s slideShowMode: %u", toLog(), state);
  if (slide_show_mode_ == state) {
    return;
  }
  asyncTask([state] (std::shared_ptr<MediaStream> media_stream) {
    media_stream->stats_->getNode()[media_stream->getVideoSinkSSRC()].insertStat(
      "erizoSlideShow",
       CumulativeStat{state});
  });
  slide_show_mode_ = state;
  notifyUpdateToHandlers();
}

// 禁止音频或视频，并在stat中设置禁止状态
void MediaStream::muteStream(bool mute_video, bool mute_audio) {
  asyncTask([mute_audio, mute_video] (std::shared_ptr<MediaStream> media_stream) {
    ELOG_DEBUG("%s message: muteStream, mute_video: %u, mute_audio: %u", media_stream->toLog(), mute_video, mute_audio);
    media_stream->audio_muted_ = mute_audio;
    media_stream->video_muted_ = mute_video;
    media_stream->stats_->getNode()[media_stream->getAudioSinkSSRC()].insertStat("erizoAudioMute",
                                                                             CumulativeStat{mute_audio});
    media_stream->stats_->getNode()[media_stream->getAudioSinkSSRC()].insertStat("erizoVideoMute",
                                                                             CumulativeStat{mute_video});
    if (media_stream && media_stream->pipeline_) {
      media_stream->pipeline_->notifyUpdate();
    }
  });
}

// 设置视频的宽高和帧率
void MediaStream::setVideoConstraints(int max_video_width, int max_video_height, int max_video_frame_rate) {
  asyncTask([max_video_width, max_video_height, max_video_frame_rate] (std::shared_ptr<MediaStream> media_stream) {
    media_stream->quality_manager_->setVideoConstraints(max_video_width, max_video_height, max_video_frame_rate);
  });
}

// 设置audio和video的clientHostType统计信息
void MediaStream::setTransportInfo(std::string audio_info, std::string video_info) {
  if (video_enabled_) {
    uint32_t video_sink_ssrc = getVideoSinkSSRC();
    uint32_t video_source_ssrc = getVideoSourceSSRC();

    if (video_sink_ssrc != kDefaultVideoSinkSSRC) {
      stats_->getNode()[video_sink_ssrc].insertStat("clientHostType", StringStat{video_info});
    }
    if (video_source_ssrc != 0) {
      stats_->getNode()[video_source_ssrc].insertStat("clientHostType", StringStat{video_info});
    }
  }

  if (audio_enabled_) {
    uint32_t audio_sink_ssrc = getAudioSinkSSRC();
    uint32_t audio_source_ssrc = getAudioSourceSSRC();

    if (audio_sink_ssrc != kDefaultAudioSinkSSRC) {
      stats_->getNode()[audio_sink_ssrc].insertStat("clientHostType", StringStat{audio_info});
    }
    if (audio_source_ssrc != 0) {
      stats_->getNode()[audio_source_ssrc].insertStat("clientHostType", StringStat{audio_info});
    }
  }
}

// 设置将要发送反馈标记和目标码率
void MediaStream::setFeedbackReports(bool will_send_fb, uint32_t target_bitrate) {
  if (slide_show_mode_) {
    target_bitrate = 0;
  }

  this->should_send_feedback_ = will_send_fb;
  if (target_bitrate == 1) {
    this->video_enabled_ = false;
  }
  this->rate_control_ = target_bitrate;
}

// 设置元数据统计信息
void MediaStream::setMetadata(std::map<std::string, std::string> metadata) {
  for (const auto &item : metadata) {
    log_stats_->getNode().insertStat("metadata-" + item.first, StringStat{item.second});
  }
  setLogContext(metadata);
}

// 获取connection的当前状态
WebRTCEvent MediaStream::getCurrentState() {
  return connection_->getCurrentState();
}

// 获取json格式的stats统计信息
void MediaStream::getJSONStats(std::function<void(std::string)> callback) {
  asyncTask([callback] (std::shared_ptr<MediaStream> stream) {
    std::string requested_stats = stream->stats_->getStats();
    //  ELOG_DEBUG("%s message: Stats, stats: %s", stream->toLog(), requested_stats.c_str());
    callback(requested_stats);
  });
}

// 更换数据包的扩展字段
void MediaStream::changeDeliverExtensionId(DataPacket *dp, packetType type) {
  RtpHeader* h = reinterpret_cast<RtpHeader*>(dp->data);
  RtcpHeader *chead = reinterpret_cast<RtcpHeader*>(dp->data);
  if (!chead->isRtcp()) {
    // Extension Id to external
    if (h->getExtension()) {
      std::array<RTPExtensions, 15> extMap;
      RtpExtensionProcessor& ext_processor = getRtpExtensionProcessor();
      switch (type) {
        case VIDEO_PACKET:
          extMap = ext_processor.getVideoExtensionMap();
          break;
        case AUDIO_PACKET:
          extMap = ext_processor.getAudioExtensionMap();
          break;
        default:
          ELOG_WARN("%s Won't process RTP extensions for unknown type packets", toLog());
          return;
          break;
      }
      uint16_t totalExtLength = h->getExtLength();
      if (h->getExtId() == 0xBEDE) {  // One-Byte Header
        char* extBuffer = (char*)&h->extensions;  // NOLINT
        uint8_t extByte = 0;
        uint16_t currentPlace = 1;
        uint8_t extId = 0;
        uint8_t extLength = 0;
        while (currentPlace < (totalExtLength*4)) {
          extByte = (uint8_t)(*extBuffer);
          extId = extByte >> 4;
          extLength = extByte & 0x0F;
          // extId == 0 should never happen, see https://tools.ietf.org/html/rfc5285#section-4.2
          if (extId != 0) {
            for (int i = 1; i < 15; i++) {
              if (extMap.at(i) == extId) {
                extBuffer[0] = (extBuffer[0] | 0xF0) & (i << 4 | 0x0F);
              }
            }
          }
          extBuffer = extBuffer + extLength + 2;
          currentPlace = currentPlace + extLength + 2;
        }
      } else {
        ELOG_WARN("%s Two-Byte Header not handled!", toLog());
      }
    }
  }
}

// 通过remote_sdp_更换数据包的载荷类型
void MediaStream::changeDeliverPayloadType(DataPacket *dp, packetType type) {
  RtpHeader* h = reinterpret_cast<RtpHeader*>(dp->data);
  RtcpHeader *chead = reinterpret_cast<RtcpHeader*>(dp->data);
  if (!chead->isRtcp()) {
      int internalPT = h->getPayloadType();
      int externalPT = internalPT;
      if (type == AUDIO_PACKET) {
          externalPT = remote_sdp_->getAudioExternalPT(internalPT);
      } else if (type == VIDEO_PACKET) {
          externalPT = remote_sdp_->getVideoExternalPT(externalPT);
      }
      if (internalPT != externalPT) {
          h->setPayloadType(externalPT);
      }
  }
}

// 解析出数据包中的扩展字，并更换这个扩展字
void MediaStream::parseIncomingExtensionId(char *buf, int len, packetType type) {
  RtcpHeader* chead = reinterpret_cast<RtcpHeader*>(buf);
  RtpHeader* h = reinterpret_cast<RtpHeader*>(buf);
  if (!chead->isRtcp()) {
    // Extension Id to internal
    if (h->getExtension()) {
      std::array<RTPExtensions, 15> extMap;
      RtpExtensionProcessor& ext_processor = getRtpExtensionProcessor();
      switch (type) {
        case VIDEO_PACKET:
          extMap = ext_processor.getVideoExtensionMap();
          break;
        case AUDIO_PACKET:
          extMap = ext_processor.getAudioExtensionMap();
          break;
        default:
          ELOG_WARN("%s Won't process RTP extensions for unknown type packets", toLog());
          return;
          break;
      }
      uint16_t totalExtLength = h->getExtLength();
      if (h->getExtId() == 0xBEDE) {  // One-Byte Header
        char* extBuffer = (char*)&h->extensions;  // NOLINT
        uint8_t extByte = 0;
        uint16_t currentPlace = 1;
        uint8_t extId = 0;
        uint8_t extLength = 0;
        while (currentPlace < (totalExtLength*4)) {
          extByte = (uint8_t)(*extBuffer);
          extId = extByte >> 4;
          extLength = extByte & 0x0F;
          if (extId != 0 && extMap[extId] != 0) {
            extBuffer[0] = (extBuffer[0] | 0xF0) & (extMap[extId] << 4 | 0x0F);
          }
          extBuffer = extBuffer + extLength + 2;
          currentPlace = currentPlace + extLength + 2;
        }
      } else {
        ELOG_WARN("%s Two-Byte Header not handled!", toLog());
      }
    }
  }
}

// 解析出数据包中的载荷类型，并更换这个载荷类型
// parses incoming payload type, replaces occurence in buf
void MediaStream::parseIncomingPayloadType(char *buf, int len, packetType type) {
  RtcpHeader* chead = reinterpret_cast<RtcpHeader*>(buf);
  RtpHeader* h = reinterpret_cast<RtpHeader*>(buf);
  if (!chead->isRtcp()) {
    int externalPT = h->getPayloadType();
    int internalPT = externalPT;
    if (type == AUDIO_PACKET) {
      internalPT = remote_sdp_->getAudioInternalPT(externalPT);
    } else if (type == VIDEO_PACKET) {
      internalPT = remote_sdp_->getVideoInternalPT(externalPT);
    }
    if (externalPT != internalPT) {
      h->setPayloadType(internalPT);
    } else {
//        ELOG_WARN("onTransportData did not find mapping for %i", externalPT);
    }
  }
}

// 向connection_写入包
void MediaStream::write(std::shared_ptr<DataPacket> packet) {
  if (connection_) {
    connection_->write(packet);
  }
}

// 通过名字的找到pipeline下的这个子handle功能，并是它有效
void MediaStream::enableHandler(const std::string &name) {
  asyncTask([name] (std::shared_ptr<MediaStream> conn) {
      if (conn && conn->pipeline_) {
        conn->pipeline_->enable(name);
      }
  });
}

// 通过名字的找到pipeline下的这个子handle功能，并是它无效
void MediaStream::disableHandler(const std::string &name) {
  asyncTask([name] (std::shared_ptr<MediaStream> conn) {
    if (conn && conn->pipeline_) {
      conn->pipeline_->disable(name);
    }
  });
}

// pipeline下的通知更新
void MediaStream::notifyUpdateToHandlers() {
  asyncTask([] (std::shared_ptr<MediaStream> conn) {
    if (conn && conn->pipeline_) {
      conn->pipeline_->notifyUpdate();
    }
  });
}

// 异步任务函数
boost::future<void> MediaStream::asyncTask(std::function<void(std::shared_ptr<MediaStream>)> f) {
  auto task_promise = std::make_shared<boost::promise<void>>();
  std::weak_ptr<MediaStream> weak_this = shared_from_this();
  worker_->task([weak_this, f, task_promise] {
    if (auto this_ptr = weak_this.lock()) {
      f(this_ptr);
    }
    task_promise->set_value();
  });
  return task_promise->get_future();
}

// 发送数据包，向pipeline中写入数据包
void MediaStream::sendPacket(std::shared_ptr<DataPacket> p) {
  if (!sending_) {
    return;
  }
  uint32_t partial_bitrate = 0;
  uint64_t sentVideoBytes = 0;
  uint64_t lastSecondVideoBytes = 0;

  if (rate_control_ && !slide_show_mode_) {
    if (p->type == VIDEO_PACKET) {
      if (rate_control_ == 1) {
        return;
      }
      now_ = clock::now();
      if ((now_ - mark_) >= kBitrateControlPeriod) {
        mark_ = now_;
        lastSecondVideoBytes = sentVideoBytes;
      }
      partial_bitrate = ((sentVideoBytes - lastSecondVideoBytes) * 8) * 10;
      if (partial_bitrate > this->rate_control_) {
        return;
      }
      sentVideoBytes += p->length;
    }
  }
  if (!pipeline_initialized_) {
    ELOG_DEBUG("%s message: Pipeline not initialized yet.", toLog());
    return;
  }

  if (pipeline_) {
    pipeline_->write(std::move(p));
  }
}

// 设置质量层层级
void MediaStream::setQualityLayer(int spatial_layer, int temporal_layer) {
  asyncTask([spatial_layer, temporal_layer] (std::shared_ptr<MediaStream> media_stream) {
    media_stream->quality_manager_->forceLayers(spatial_layer, temporal_layer);
  });
}
//  设置空间层有效
void MediaStream::enableSlideShowBelowSpatialLayer(bool enabled, int spatial_layer) {
  asyncTask([enabled, spatial_layer] (std::shared_ptr<MediaStream> media_stream) {
    media_stream->quality_manager_->enableSlideShowBelowSpatialLayer(enabled, spatial_layer);
  });
}