制作网页网站小说教程,wordpress com login,成都模板网建站,建设银行车主卡网上交罚款网站技术背景
随着Unity3D的应用范围越来越广#xff0c;越来越多的行业开始基于Unity3D开发产品#xff0c;如传统行业中虚拟仿真教育、航空工业、室内设计、城市规划、工业仿真等领域。
基于此#xff0c;好多开发者苦于在Unity环境下#xff0c;没有低延迟的推拉流解决方案…技术背景
随着Unity3D的应用范围越来越广越来越多的行业开始基于Unity3D开发产品如传统行业中虚拟仿真教育、航空工业、室内设计、城市规划、工业仿真等领域。
基于此好多开发者苦于在Unity环境下没有低延迟的推拉流解决方案前几年我们在Unity环境下推出了跨平台低延迟的RTMP|RTSP直播播放器很好的解决了好多对延迟要求苛刻的使用场景。
随着时间的推移越来越多的开发者联系我们希望我们能推出Unity环境下的RTMP推送模块获取到unity的实时数据更低延迟更高效率的实现数据传输推送基于此我们发布了Unity环境下的RTMP推送模块。
本文以Windows平台为例数据源分别为Unity的窗口、摄像头或整个屏幕编码传输模块还是调用大牛直播SDK官方的原生接口简单界面先睹为快 技术实现
1. 基础初始化
private bool InitSDK(){if (!is_pusher_sdk_init_){// 设置日志路径(请确保目录存在)String log_path D:\\pulisherlog;NTSmartLog.NT_SL_SetPath(log_path);UInt32 isInited NTSmartPublisherSDK.NT_PB_Init(0, IntPtr.Zero);if (isInited ! 0){Debug.Log(调用NT_PB_Init失败..);return false;}is_pusher_sdk_init_ true;}return true;}
2. 调用Open()接口获取推送实例 public bool OpenPublisherHandle(uint video_option, uint audio_option){if (publisher_handle_ ! IntPtr.Zero){return true;}publisher_handle_count_ 0;if (NTBaseCodeDefine.NT_ERC_OK ! NTSmartPublisherSDK.NT_PB_Open(out publisher_handle_,video_option, audio_option, 0, IntPtr.Zero)){return false;}if (publisher_handle_ ! IntPtr.Zero){pb_event_call_back_ new NT_PB_SDKEventCallBack(PbEventCallBack);NTSmartPublisherSDK.NT_PB_SetEventCallBack(publisher_handle_, IntPtr.Zero, pb_event_call_back_);return true;}else{return false;}}3. 初始化参数配置
这里需要注意下如果要采集unity窗口需要设置图层模式先填充一层RGBA黑色背景然后再添加一层用于叠加外部数据。 private void SetCommonOptionToPublisherSDK(){if (!IsPublisherHandleAvailable()){Debug.Log(SetCommonOptionToPublisherSDK, publisher handle with null..);return;}NTSmartPublisherSDK.NT_PB_ClearLayersConfig(publisher_handle_, 0,0, IntPtr.Zero);if (video_option NTSmartPublisherDefine.NT_PB_E_VIDEO_OPTION.NT_PB_E_VIDEO_OPTION_LAYER){// 第0层填充RGBA矩形, 目的是保证帧率, 颜色就填充全黑int red 0;int green 0;int blue 0;int alpha 255;NT_PB_RGBARectangleLayerConfig rgba_layer_c0 new NT_PB_RGBARectangleLayerConfig();rgba_layer_c0.base_.type_ (Int32)NTSmartPublisherDefine.NT_PB_E_LAYER_TYPE.NT_PB_E_LAYER_TYPE_RGBA_RECTANGLE;rgba_layer_c0.base_.index_ 0;rgba_layer_c0.base_.enable_ 1;rgba_layer_c0.base_.region_.x_ 0;rgba_layer_c0.base_.region_.y_ 0;rgba_layer_c0.base_.region_.width_ video_width_;rgba_layer_c0.base_.region_.height_ video_height_;rgba_layer_c0.base_.offset_ Marshal.OffsetOf(rgba_layer_c0.GetType(), base_).ToInt32();rgba_layer_c0.base_.cb_size_ (uint)Marshal.SizeOf(rgba_layer_c0);rgba_layer_c0.red_ System.BitConverter.GetBytes(red)[0];rgba_layer_c0.green_ System.BitConverter.GetBytes(green)[0];rgba_layer_c0.blue_ System.BitConverter.GetBytes(blue)[0];rgba_layer_c0.alpha_ System.BitConverter.GetBytes(alpha)[0];IntPtr rgba_conf Marshal.AllocHGlobal(Marshal.SizeOf(rgba_layer_c0));Marshal.StructureToPtr(rgba_layer_c0, rgba_conf, true);UInt32 rgba_r NTSmartPublisherSDK.NT_PB_AddLayerConfig(publisher_handle_, 0,rgba_conf, (int)NTSmartPublisherDefine.NT_PB_E_LAYER_TYPE.NT_PB_E_LAYER_TYPE_RGBA_RECTANGLE,0, IntPtr.Zero);Marshal.FreeHGlobal(rgba_conf);NT_PB_ExternalVideoFrameLayerConfig external_layer_c1 new NT_PB_ExternalVideoFrameLayerConfig();external_layer_c1.base_.type_ (Int32)NTSmartPublisherDefine.NT_PB_E_LAYER_TYPE.NT_PB_E_LAYER_TYPE_EXTERNAL_VIDEO_FRAME;external_layer_c1.base_.index_ 1;external_layer_c1.base_.enable_ 1;external_layer_c1.base_.region_.x_ 0;external_layer_c1.base_.region_.y_ 0;external_layer_c1.base_.region_.width_ video_width_;external_layer_c1.base_.region_.height_ video_height_;external_layer_c1.base_.offset_ Marshal.OffsetOf(external_layer_c1.GetType(), base_).ToInt32();external_layer_c1.base_.cb_size_ (uint)Marshal.SizeOf(external_layer_c1);IntPtr external_layer_conf Marshal.AllocHGlobal(Marshal.SizeOf(external_layer_c1));Marshal.StructureToPtr(external_layer_c1, external_layer_conf, true);UInt32 external_r NTSmartPublisherSDK.NT_PB_AddLayerConfig(publisher_handle_, 0,external_layer_conf, (int)NTSmartPublisherDefine.NT_PB_E_LAYER_TYPE.NT_PB_E_LAYER_TYPE_EXTERNAL_VIDEO_FRAME,0, IntPtr.Zero);Marshal.FreeHGlobal(external_layer_conf);}else if (video_option NTSmartPublisherDefine.NT_PB_E_VIDEO_OPTION.NT_PB_E_VIDEO_OPTION_CAMERA){CameraInfo camera cameras_[cur_sel_camera_index_];NT_PB_VideoCaptureCapability cap camera.capabilities_[cur_sel_camera_resolutions_index_];SetVideoCaptureDeviceBaseParameter(camera.id_.ToString(), (UInt32)cap.width_, (UInt32)cap.height_);}SetFrameRate((UInt32)CalBitRate(edit_key_frame_, video_width_, video_height_));Int32 type 0; //软编码Int32 encoder_id 1;UInt32 codec_id (UInt32)NTCommonMediaDefine.NT_MEDIA_CODEC_ID.NT_MEDIA_CODEC_ID_H264;Int32 param1 0;SetVideoEncoder(type, encoder_id, codec_id, param1);SetVideoQualityV2(CalVideoQuality(video_width_, video_height_, is_h264_encoder));SetVideoMaxBitRate((CalMaxKBitRate(edit_key_frame_, video_width_, video_height_, false)));SetVideoKeyFrameInterval((edit_key_frame_));if (is_h264_encoder){SetVideoEncoderProfile(1);}SetVideoEncoderSpeed(CalVideoEncoderSpeed(video_width_, video_height_, is_h264_encoder));// 音频相关设置SetAuidoInputDeviceId(0);SetPublisherAudioCodecType(1);SetPublisherMute(is_mute);SetEchoCancellation(0, 0);SetNoiseSuppression(0);SetAGC(0);SetVAD(0);SetInputAudioVolume(Convert.ToSingle(edit_audio_input_volume_));}
4. 数据采集
摄像头和屏幕的数据采集还是调用原生的SDK接口本文不再赘述如果需要采集Unity窗体的数据可以用参考以下代码 if ( texture_ null || video_width_ ! Screen.width || video_height_ ! Screen.height){Debug.Log(OnPostRender screen changed scr_width: Screen.width scr_height: Screen.height);if (screen_image_ ! IntPtr.Zero){Marshal.FreeHGlobal(screen_image_);screen_image_ IntPtr.Zero;}if (texture_ ! null){UnityEngine.Object.Destroy(texture_);texture_ null;}video_width_ Screen.width;video_height_ Screen.height;texture_ new Texture2D(video_width_, video_height_, TextureFormat.BGRA32, false);screen_image_ Marshal.AllocHGlobal(video_width_ * 4 * video_height_);Debug.Log(OnPostRender screen changed--);return;}texture_.ReadPixels(new Rect(0, 0, video_width_, video_height_), 0, 0, false);texture_.Apply();
从 texture里面通过调用 GetRawTextureData()获取到原始数据。
5. 数据对接
获取到数据后通过调用 OnPostRGBAData()接口传递给SDK层。
6. 本地数据预览 public bool StartPreview(){if(CheckPublisherHandleAvailable() false)return false;video_preview_image_callback_ new NT_PB_SDKVideoPreviewImageCallBack(SDKVideoPreviewImageCallBack);NTSmartPublisherSDK.NT_PB_SetVideoPreviewImageCallBack(publisher_handle_, (int)NTSmartPublisherDefine.NT_PB_E_IMAGE_FORMAT.NT_PB_E_IMAGE_FORMAT_RGB32, IntPtr.Zero, video_preview_image_callback_);if (NTBaseCodeDefine.NT_ERC_OK ! NTSmartPublisherSDK.NT_PB_StartPreview(publisher_handle_, 0, IntPtr.Zero)){if (0 publisher_handle_count_){NTSmartPublisherSDK.NT_PB_Close(publisher_handle_);publisher_handle_ IntPtr.Zero;}return false;}publisher_handle_count_;is_previewing_ true;return true;}
设置preview后处理preview的数据回调 //预览数据回调public void SDKVideoPreviewImageCallBack(IntPtr handle, IntPtr user_data, IntPtr image){NT_PB_Image pb_image (NT_PB_Image)Marshal.PtrToStructure(image, typeof(NT_PB_Image));NT_VideoFrame pVideoFrame new NT_VideoFrame();pVideoFrame.width_ pb_image.width_;pVideoFrame.height_ pb_image.height_;pVideoFrame.stride_ pb_image.stride_[0];Int32 argb_size pb_image.stride_[0] * pb_image.height_;pVideoFrame.plane_data_ new byte[argb_size];if (argb_size 0){Marshal.Copy(pb_image.plane_[0],pVideoFrame.plane_data_,0, argb_size);}{cur_image_ pVideoFrame;}}
7. 相关event回调处理 private void PbEventCallBack(IntPtr handle, IntPtr user_data, UInt32 event_id,Int64 param1,Int64 param2,UInt64 param3,UInt64 param4,[MarshalAs(UnmanagedType.LPStr)] String param5,[MarshalAs(UnmanagedType.LPStr)] String param6,IntPtr param7){String event_log ;switch (event_id){case (uint)NTSmartPublisherDefine.NT_PB_E_EVENT_ID.NT_PB_E_EVENT_ID_CONNECTING:event_log 连接中;if (!String.IsNullOrEmpty(param5)){event_log event_log url: param5;}break;case (uint)NTSmartPublisherDefine.NT_PB_E_EVENT_ID.NT_PB_E_EVENT_ID_CONNECTION_FAILED:event_log 连接失败;if (!String.IsNullOrEmpty(param5)){event_log event_log url: param5;}break;case (uint)NTSmartPublisherDefine.NT_PB_E_EVENT_ID.NT_PB_E_EVENT_ID_CONNECTED:event_log 已连接;if (!String.IsNullOrEmpty(param5)){event_log event_log url: param5;}break;case (uint)NTSmartPublisherDefine.NT_PB_E_EVENT_ID.NT_PB_E_EVENT_ID_DISCONNECTED:event_log 断开连接;if (!String.IsNullOrEmpty(param5)){event_log event_log url: param5;}break;default:break;}if(OnLogEventMsg ! null) OnLogEventMsg.Invoke(event_id, event_log);}
8. 开始推送、停止推送 public bool StartPublisher(String url){if (CheckPublisherHandleAvailable() false) return false;if (publisher_handle_ IntPtr.Zero){return false;}if (!String.IsNullOrEmpty(url)){NTSmartPublisherSDK.NT_PB_SetURL(publisher_handle_, url, IntPtr.Zero);}if (NTBaseCodeDefine.NT_ERC_OK ! NTSmartPublisherSDK.NT_PB_StartPublisher(publisher_handle_, IntPtr.Zero)){if (0 publisher_handle_count_){NTSmartPublisherSDK.NT_PB_Close(publisher_handle_);publisher_handle_ IntPtr.Zero;}is_publishing_ false;return false;}publisher_handle_count_;is_publishing_ true;return true;}public void StopPublisher(){if (is_publishing_ false) return;publisher_handle_count_--;NTSmartPublisherSDK.NT_PB_StopPublisher(publisher_handle_);if (0 publisher_handle_count_){NTSmartPublisherSDK.NT_PB_Close(publisher_handle_);publisher_handle_ IntPtr.Zero;}is_publishing_ false;}
9. 关闭实例 public void Close(){if (0 publisher_handle_count_){NTSmartPublisherSDK.NT_PB_Close(publisher_handle_);publisher_handle_ IntPtr.Zero;}}
总结
经测试Unity环境下通过高效率的数据采集、编码和推送配合SmartPlayer播放器播放整体延迟可控制在毫秒级可适用于大多数Unity环境下对延迟和稳定性要求苛刻的场景。
