Like the RTMP or RTSP players under Unity3D, many developers are struggling with how to collect and encode data and push them to the streaming media server in real time with high efficiency and low latency in the Unity environment, so as to realize the low-latency push-pull streaming solution in the Unity scenario.
Regarding screen capture, there are two options:
1. Directly encapsulate Android's native screen capture project, provide an interface in Unity, obtain screen data and push it after getting screen permission;
2. If you only need to get Unity's form or camera data and push it out, you can get the raw data that needs to be pushed under Unity, then encapsulate the native RTMP streaming interface, and call the native SDK to push the data. The benefits of this approach Yes, you can customize the data content to be collected, as long as you complete the data connection according to the interface provided by the native SDK. For the specific implementation, please refer to this article.
This article takes the Android platform as an example to introduce the Android platform RTMP streaming in the Unity environment. The data collection is completed in Unity, the data is encoded and pushed, and the SDK Android platform is called for RTMP live broadcast to push the interface of the SDK native library for external secondary encapsulation, and realize RTMP efficiently. push. Without further ado, let’s see the effect first.
The picture below shows the screen capture in the Android platform Unity environment, the encoding is pushed to the RTMP server, and then the Windows platform player pulls the RTMP stream for playback. Overall latency in milliseconds:
data collection push
Unity data collection is relatively simple, you can easily get RGB24 data:
texture_ = new Texture2D(video_width_, video_height_, TextureFormat.RGB24, false);
texture_.ReadPixels(new Rect(0, 0, video_width_, video_height_), 0, 0, false);
texture_.Apply();Then get the data by calling texture_.GetRawTextureData();.
After getting the data, call the NT_PB_U3D_OnCaptureVideoRGB24PtrData() interface encapsulated by the native SDK to complete the data delivery.
Simple call process
private void Start()
{
game_object_ = this.gameObject.name;
AndroidJavaClass android_class = new AndroidJavaClass("com.unity3d.player.UnityPlayer");
java_obj_cur_activity_ = android_class.GetStatic<AndroidJavaObject>("currentActivity");
pusher_obj_ = new AndroidJavaObject("com.daniulive.smartpublisher.SmartPublisherUnity3d");
NT_PB_U3D_Init();
//NT_U3D_SetSDKClientKey("", "", 0);
btn_encode_mode_.onClick.AddListener(OnEncodeModeBtnClicked);
btn_pusher_.onClick.AddListener(OnPusherBtnClicked);
btn_mute_.onClick.AddListener(OnMuteBtnClicked);
}
Private message me to receive the latest and most complete learning and improvement materials in 2022 , including ( C/C++ , Linux , FFmpeg , webRTC , rtmp , hls , rtsp , ffplay , srs )
After completing the interface initialization, call the Push() interface
public void Push()
{
if (is_running)
{
Debug.Log("已推送..");
return;
}
if (texture_ != null)
{
UnityEngine.Object.Destroy(texture_);
texture_ = null;
}
video_width_ = Screen.width;
video_height_ = Screen.height;
scale_width_ = (video_width_ + 1) / 2;
scale_height_ = (video_height_ + 1) / 2;
if (scale_width_ % 2 != 0)
{
scale_width_ = scale_width_ + 1;
}
if (scale_height_ % 2 != 0)
{
scale_height_ = scale_height_ + 1;
}
texture_ = new Texture2D(video_width_, video_height_, TextureFormat.RGB24, false);
//获取输入框的url
string url = input_url_.text.Trim();
if (!url.StartsWith("rtmp://"))
{
push_url_ = "rtmp://192.168.0.199:1935/hls/stream1";
}
else
{
push_url_ = url;
}
OpenPusher();
if (pusher_handle_ == 0)
return;
NT_PB_U3D_Set_Game_Object(pusher_handle_, game_object_);
/* ++ 推送前参数配置可加在此处 ++ */
InitAndSetConfig();
NT_PB_U3D_SetPushUrl(pusher_handle_, push_url_);
/* -- 推送前参数配置可加在此处 -- */
int flag = NT_PB_U3D_StartPublisher(pusher_handle_);
if (flag == DANIULIVE_RETURN_OK)
{
Debug.Log("推送成功..");
}
else
{
Debug.LogError("推送失败..");
}
is_running = true;
}
call OpenPusher()
private void OpenPusher()
{
if ( java_obj_cur_activity_ == null )
{
Debug.LogError("getApplicationContext is null");
return;
}
int audio_opt = 1;
int video_opt = 1;
pusher_handle_ = NT_PB_U3D_Open(audio_opt, video_opt, video_width_, video_height_);
if (pusher_handle_ != 0)
Debug.Log("NT_PB_U3D_Open success");
else
Debug.LogError("NT_PB_U3D_Open fail");
}
InitAndSetConfig()
private void InitAndSetConfig()
{
if (is_hw_encode_)
{
int h264HWKbps = setHardwareEncoderKbps(true, video_width_, video_height_);
Debug.Log("h264HWKbps: " + h264HWKbps);
int isSupportH264HWEncoder = NT_PB_U3D_SetVideoHWEncoder(pusher_handle_, h264HWKbps);
if (isSupportH264HWEncoder == 0)
{
Debug.Log("Great, it supports h.264 hardware encoder!");
}
}
else {
if (is_sw_vbr_mode_) //H.264 software encoder
{
int is_enable_vbr = 1;
int video_quality = CalVideoQuality(video_width_, video_height_, true);
int vbr_max_bitrate = CalVbrMaxKBitRate(video_width_, video_height_);
NT_PB_U3D_SetSwVBRMode(pusher_handle_, is_enable_vbr, video_quality, vbr_max_bitrate);
//NT_PB_U3D_SetSWVideoEncoderSpeed(pusher_handle_, 2);
}
}
NT_PB_U3D_SetAudioCodecType(pusher_handle_, 1);
NT_PB_U3D_SetFPS(pusher_handle_, 25);
NT_PB_U3D_SetGopInterval(pusher_handle_, 25*2);
//NT_PB_U3D_SetSWVideoBitRate(pusher_handle_, 600, 1200);
}
ClosePusher()
private void ClosePusher()
{
if (texture_ != null)
{
UnityEngine.Object.Destroy(texture_);
texture_ = null;
}
int flag = NT_PB_U3D_StopPublisher(pusher_handle_);
if (flag == DANIULIVE_RETURN_OK)
{
Debug.Log("停止成功..");
}
else
{
Debug.LogError("停止失败..");
}
flag = NT_PB_U3D_Close(pusher_handle_);
if (flag == DANIULIVE_RETURN_OK)
{
Debug.Log("关闭成功..");
}
else
{
Debug.LogError("关闭失败..");
}
pusher_handle_ = 0;
NT_PB_U3D_UnInit();
is_running = false;
}
To facilitate testing, Update() refreshes the current time:
private void Update()
{
//获取当前时间
hour = DateTime.Now.Hour;
minute = DateTime.Now.Minute;
millisecond = DateTime.Now.Millisecond;
second = DateTime.Now.Second;
year = DateTime.Now.Year;
month = DateTime.Now.Month;
day = DateTime.Now.Day;
GameObject.Find("Canvas/Panel/LableText").GetComponent<Text>().text = string.Format("{0:D2}:{1:D2}:{2:D2}:{3:D2} " + "{4:D4}/{5:D2}/{6:D2}", hour, minute, second, millisecond, year, month, day);
}
Related Event Handling
public void onNTSmartEvent(string param)
{
if (!param.Contains(","))
{
Debug.Log("[onNTSmartEvent] android传递参数错误");
return;
}
string[] strs = param.Split(',');
string player_handle =strs[0];
string code = strs[1];
string param1 = strs[2];
string param2 = strs[3];
string param3 = strs[4];
string param4 = strs[5];
Debug.Log("[onNTSmartEvent] code: 0x" + Convert.ToString(Convert.ToInt32(code), 16));
String publisher_event = "";
switch (Convert.ToInt32(code))
{
case EVENTID.EVENT_DANIULIVE_ERC_PUBLISHER_STARTED:
publisher_event = "开始..";
break;
case EVENTID.EVENT_DANIULIVE_ERC_PUBLISHER_CONNECTING:
publisher_event = "连接中..";
break;
case EVENTID.EVENT_DANIULIVE_ERC_PUBLISHER_CONNECTION_FAILED:
publisher_event = "连接失败..";
break;
case EVENTID.EVENT_DANIULIVE_ERC_PUBLISHER_CONNECTED:
publisher_event = "连接成功..";
break;
case EVENTID.EVENT_DANIULIVE_ERC_PUBLISHER_DISCONNECTED:
publisher_event = "连接断开..";
break;
case EVENTID.EVENT_DANIULIVE_ERC_PUBLISHER_STOP:
publisher_event = "关闭..";
break;
case EVENTID.EVENT_DANIULIVE_ERC_PUBLISHER_RECORDER_START_NEW_FILE:
publisher_event = "开始一个新的录像文件 : " + param3;
break;
case EVENTID.EVENT_DANIULIVE_ERC_PUBLISHER_ONE_RECORDER_FILE_FINISHED:
publisher_event = "已生成一个录像文件 : " + param3;
break;
case EVENTID.EVENT_DANIULIVE_ERC_PUBLISHER_SEND_DELAY:
publisher_event = "发送时延: " + param1 + " 帧数:" + param2;
break;
case EVENTID.EVENT_DANIULIVE_ERC_PUBLISHER_CAPTURE_IMAGE:
publisher_event = "快照: " + param1 + " 路径：" + param3;
if (Convert.ToInt32(param1) == 0)
{
publisher_event = publisher_event + "截取快照成功..";
}
else
{
publisher_event = publisher_event + "截取快照失败..";
}
break;
case EVENTID.EVENT_DANIULIVE_ERC_PUBLISHER_RTSP_URL:
publisher_event = "RTSP服务URL: " + param3;
break;
case EVENTID.EVENT_DANIULIVE_ERC_PUSH_RTSP_SERVER_RESPONSE_STATUS_CODE:
publisher_event = "RTSP status code received, codeID: " + param1 + ", RTSP URL: " + param3;
break;
case EVENTID.EVENT_DANIULIVE_ERC_PUSH_RTSP_SERVER_NOT_SUPPORT:
publisher_event = "服务器不支持RTSP推送, 推送的RTSP URL: " + param3;
break;
}
Debug.Log(publisher_event);
}
Summarize
Through the above process, the screen or camera data in the Unity environment can be pushed and played in milliseconds, and interested developers can refer to it as appropriate.