wpf下如何实现超低延迟的RTMP或RTSP播放

技术背景

我们在做Windows平台RTMP和RTSP播放模块对接的时候，有开发者需要在wpf下调用，如果要在wpf下使用，只需要参考C#的对接demo即可，唯一不同的是，视频流数据显示的话，要么通过控件模式，要么可以让RTMP、RTSP播放模块回调rgb数据上来，在wpf直接绘制即可。

技术实现

本文以大牛直播SDK的Windows平台SmartPlayer为例，回调数据的模式，其他不再说明，只要处理好上来的数据就好：

播放之前，设置回调，选择NT_SP_E_VIDEO_FRAME_FORMAT_RGB32：

video_frame_call_back_ = new SP_SDKVideoFrameCallBack(SetVideoFrameCallBack);NTSmartPlayerSDK.NT_SP_SetVideoFrameCallBack(player_handle_, (Int32)NT.NTSmartPlayerDefine.NT_SP_E_VIDEO_FRAME_FORMAT.NT_SP_E_VIDEO_FRAME_FORMAT_RGB32, IntPtr.Zero, video_frame_call_back_);

处理rgb数据回调：

       /** nt_player_wrapper.cs* Author: daniusdk.com*/public void SetVideoFrameCallBack(IntPtr handle, IntPtr userData, UInt32 status, IntPtr frame){if (frame == IntPtr.Zero){return;}//如需直接处理RGB数据，请参考以下流程NT_SP_VideoFrame video_frame = (NT_SP_VideoFrame)Marshal.PtrToStructure(frame, typeof(NT_SP_VideoFrame));if (video_frame.format_ != (Int32)NT.NTSmartPlayerDefine.NT_SP_E_VIDEO_FRAME_FORMAT.NT_SP_E_VIDEO_FRAME_FORMAT_RGB32)return;NT_SP_VideoFrame pVideoFrame = new NT_SP_VideoFrame();pVideoFrame.format_ = video_frame.format_;pVideoFrame.width_ = video_frame.width_;pVideoFrame.height_ = video_frame.height_;pVideoFrame.timestamp_ = video_frame.timestamp_;pVideoFrame.stride0_ = video_frame.stride0_;pVideoFrame.stride1_ = video_frame.stride1_;pVideoFrame.stride2_ = video_frame.stride2_;pVideoFrame.stride3_ = video_frame.stride3_;Int32 argb_size = video_frame.stride0_ * video_frame.height_;pVideoFrame.plane0_ = Marshal.AllocHGlobal(argb_size);CopyMemory(pVideoFrame.plane0_, video_frame.plane0_, (UInt32)argb_size);}

另外一种，可以用picturebox，在MainWindow.xaml 做以下设置：

        <WindowsFormsHost HorizontalAlignment="Left" Height="338" Margin="10,10,0,0" VerticalAlignment="Top" Width="480" Background="Black"><wf:PictureBox x:Name="RealPlayWnd"></wf:PictureBox></WindowsFormsHost>

为了便于多实例集成参考，以播放2路为例（一路2560*1440，一路1920*1080）：

具体实现如下：

       /** MainWindow.xaml.cs* Author: daniusdk.com*/public MainWindow(){InitializeComponent();if (!InitSDK())return;UIDispatcher = Dispatcher.CurrentDispatcher;player1_ = new nt_player_wrapper(RealPlayWnd, UIDispatcher);player1_.EventGetPlayerEventMsg += new DelGetPlayerEventMsg(GetPlayerEventMsgInfo);player1_.EventGetVideoSize += new DelGetVideoSize(GetVideoSize);player2_ = new nt_player_wrapper(RealPlayWnd1, UIDispatcher);player2_.EventGetPlayerEventMsg += new DelGetPlayerEventMsg(GetPlayerEventMsgInfo);player2_.EventGetVideoSize += new DelGetVideoSize(GetVideoSize);}private void GetPlayerEventMsgInfo(IntPtr handle, String msg){this.Dispatcher.Invoke((Action)delegate(){event_label.Content = msg;});}private void GetVideoSize(IntPtr handle, String size){this.Dispatcher.Invoke((Action)delegate(){video_size.Content = size;});}private bool InitSDK(){if (!is_player_sdk_init_){UInt32 isPlayerInited = NT.NTSmartPlayerSDK.NT_SP_Init(0, IntPtr.Zero);if (isPlayerInited != 0){MessageBox.Show("调用NT_SP_Init失败..");return false;}is_player_sdk_init_ = true;}return true;}private void Button_Click_1(object sender, RoutedEventArgs e){if (!player1_.IsPlaying()){player1_.SetBuffer(0);bool is_mute = true;if (!player1_.StartPlay("rtsp://admin:daniulive12345@192.168.0.120:554/h264/ch1/main/av_stream", false, is_mute))return;btn_playback1.Content = "停止播放";}else{player1_.StopPlay();btn_playback1.Content = "开始播放";}}private void Button_Click_2(object sender, RoutedEventArgs e){if (!player2_.IsPlaying()){player2_.SetBuffer(0);bool is_mute = true;if (!player2_.StartPlay("rtsp://admin:admin123456@192.168.0.121:554/cam/realmonitor?channel=1&subtype=0", false, is_mute))return;btn_playback2.Content = "停止播放";}else{player2_.StopPlay();btn_playback2.Content = "开始播放";}}protected override void OnClosing(System.ComponentModel.CancelEventArgs e){if (MessageBox.Show("确定要关闭窗口吗？", "确认", MessageBoxButton.YesNo) != MessageBoxResult.Yes){// 如果用户选择“否”，取消关闭e.Cancel = true;}if (player1_.IsPlaying()){player1_.StopPlay();}player1_.Dispose();if (player2_.IsPlaying()){player2_.StopPlay();}player2_.Dispose();if (is_player_sdk_init_){NTSmartPlayerSDK.NT_SP_UnInit();is_player_sdk_init_ = false;}    base.OnClosing(e);}

延迟依旧毫秒级，CPU占用如下，如果用硬解码，体验会更好：