前言

需求：我这边遇到的需求是需要在RK3568上进行拉流处理，然后使用MPP进行硬解码，但目前解码解的还是比较繁琐一点，不过，没关系把。先记录下，自己后面如果有对这个东西进行重写，再进行详细描述即可。
主要内容：
1、FFmpeg 拉流处理。
2、RK3568 MPP处理。

正文

一、FFmpeg 拉流处理

code

bool CVideoDecodeThd::_AnalysisFile(const QString& _sFilePath)
{
#ifdef ARMLOG_INFO << "play video file : " << _sFilePath.toStdString();// 打开文件流读取文件头解析出视频信息如轨道信息、时长等// m_pFormatCtx初始化为NULL,如果打开成功,它会被设置成非NULL的值,在不需要的时候可以通过avcodec_free_context释放。// 这个方法实际可以打开多种来源的数据,url可以是本地路径、rtmp地址等// 在不需要的时候通过avformat_close_input关闭文件流if (avformat_open_input(&m_pFormatCtx, _sFilePath.toLatin1().data(), nullptr, nullptr) != 0){LOG_ERROR << "can't open the file.";return false;}// 读取媒体文件的数据包以获取流信息if (avformat_find_stream_info(m_pFormatCtx, nullptr) < 0){LOG_ERROR << "can't find stream infomation.";return false;}//查找视频轨道int iVideoStream = av_find_best_stream(m_pFormatCtx, AVMEDIA_TYPE_VIDEO, -1, -1, nullptr, 0);if (iVideoStream < 0){LOG_ERROR << "can't find video stream";return false;}// 查找解码器AVCodecParameters *pCodecParame = m_pFormatCtx->streams[iVideoStream]->codecpar;if (!pCodecParame){LOG_ERROR << "can't find AVCodecParameters";return false;}m_pCodec = (AVCodec *)avcodec_find_decoder(pCodecParame->codec_id);if (!m_pCodec){LOG_ERROR << "Codec cant found, code id:" << pCodecParame->codec_id;return false;}m_pCodecCtx = avcodec_alloc_context3(m_pCodec);if (!m_pCodecCtx){LOG_ERROR << "can't alloc codec context";return false;}if (avcodec_parameters_to_context(m_pCodecCtx, (const AVCodecParameters *)pCodecParame) < 0){LOG_ERROR << "can't set codec params";avcodec_close(m_pCodecCtx);return false;}// 打开解码器if (avcodec_open2(m_pCodecCtx, m_pCodec, nullptr) < 0){LOG_ERROR << "can't open codec.";avcodec_close(m_pCodecCtx);return false;}LOG_INFO << "the file video format is : " << m_pCodec->name;float fFrameRate = 0.0;if (m_pFormatCtx->streams[iVideoStream]->avg_frame_rate.den != 0 && m_pFormatCtx->streams[iVideoStream]->avg_frame_rate.num != 0){fFrameRate = m_pFormatCtx->streams[iVideoStream]->avg_frame_rate.num * 1.0 /m_pFormatCtx->streams[iVideoStream]->avg_frame_rate.den;//每秒多少帧LOG_INFO << "Frame Rate : " << fFrameRate;}m_pFrame = av_frame_alloc();m_pFrameRGB = av_frame_alloc();m_pImageConvertCtx = sws_getContext(m_pCodecCtx->width, m_pCodecCtx->height,m_pCodecCtx->pix_fmt, m_pCodecCtx->width,m_pCodecCtx->height, AV_PIX_FMT_RGB32,SWS_BICUBIC, nullptr, nullptr, nullptr);int iNumBytes = av_image_get_buffer_size(AV_PIX_FMT_RGB32, m_pCodecCtx->width, m_pCodecCtx->height, 1);uint8_t *pOutBuffer = (uint8_t*)av_malloc(iNumBytes * sizeof(uint8_t));av_image_fill_arrays(m_pFrameRGB->data, m_pFrameRGB->linesize, pOutBuffer, AV_PIX_FMT_RGB32, m_pCodecCtx->width, m_pCodecCtx->height, 1);int iYsize = m_pCodecCtx->width * m_pCodecCtx->height;AVPacket * pPacket = av_packet_alloc();  ///< 分配一个packet内存av_new_packet(pPacket, iYsize);     ///< 分配一个packet的数据av_dump_format(m_pFormatCtx, 0, _sFilePath.toLatin1().data(), 0);   ///< 输出视频信息qint64 iStartTime = 0;while (m_bRunning){if (0 == iStartTime){iStartTime = QDateTime::currentDateTime().toMSecsSinceEpoch();}if (av_read_frame(m_pFormatCtx, pPacket) < 0){emit SIGNAL_PlayEnd();break;      // 文件读取完成}while (!m_bPlay){QThread::msleep(20);if (!m_bRunning){break;}}if (pPacket->stream_index == iVideoStream){
#ifdef MPPint iRet = _DecodeAVPacket(pPacket);
#elifint iRet =avcodec_send_packet(m_pCodecCtx, pPacket);if (0 == iRet){while (avcodec_receive_frame(m_pCodecCtx, m_pFrame) >= 0){if (!m_bRunning || !m_bPlay){break;}sws_scale(m_pImageConvertCtx, (uint8_t const* const*)m_pFrame->data,m_pFrame->linesize, 0, m_pCodecCtx->height,m_pFrameRGB->data, m_pFrameRGB->linesize);QImage oImg ((uchar*) pOutBuffer, m_pCodecCtx->width, m_pCodecCtx->height, QImage::Format_RGB32);QImage oTempImg = oImg.copy();emit SIGNAL_NewFrame(oTempImg);int iInternal = QDateTime::currentDateTime().toMSecsSinceEpoch() - iStartTime;iStartTime = 0;int iSleepMs = 1000 / fFrameRate - iInternal;if (iSleepMs > 0){msleep(iSleepMs);}}}else{LOG_ERROR << "send packet fail : " << iRet;}
#endif}av_packet_unref(pPacket);//QThread::msleep(10);}QThread::msleep(500);   ///< 等待外界渲染完毕av_free(pOutBuffer);//回收数据包av_packet_free(&pPacket);//销毁帧av_frame_free(&m_pFrame);av_frame_free(&m_pFrameRGB);//销毁SwsContextsws_freeContext(m_pImageConvertCtx);//释放解码器avcodec_close(m_pCodecCtx);// 销毁AVFormatContextavformat_close_input(&m_pFormatCtx);LOG_INFO << "thread is finished";return true;
#elsereturn true;
#endif
}

这个函数是关于拉流的函数，具体的拉流的比较详细的内容，在另一篇文章进行详细叙述。

二、RK3568 mpp硬解码

1、简介

瑞芯微提供的媒体处理软件平台（Media Process Platform，简称 MPP）是适用于瑞芯微芯片系列的
通用媒体处理软件平台。该平台对应用软件屏蔽了芯片相关的复杂底层处理，其目的是为了屏蔽不
同芯片的差异，为使用者提供统一的视频媒体处理接口（Media Process Interface，缩写 MPI）。MPP
提供的功能包括：
视频解码：
H.265 / H.264 / H.263 / VP9 / VP8 / MPEG-4 / MPEG-2 / MPEG-1 / VC1 / MJPEG
视频编码：
H.264 / VP8 / MJPEG
视频处理：
视频拷贝，缩放，色彩空间转换，场视频解交织（Deinterlace）

2、普通mpp解码流程

3、核心代码

上面mpp使用的地方在这里：
_DecodeAVPacket(pPacket);
基本就是拉流获取到AVPacket 包传入Mpp进行解码。
_InitMpp():

int CVideoDecodeThd::_InitMpp()
{LOG_INFO << "--> CVideoDecodeThd::_InitMpp Start";MPP_RET ret = mpp_create(&m_Ctx, &m_pMpi);if (MPP_OK != ret){LOG_INFO << ("mpp_create error\n");return -1;}/*** 4. 配置解器*      - 解码文件需要 split 模式*      - 设置非阻塞模式，0非阻塞（默认），-1阻塞，+val 超时(ms)*/RK_U32 need_split = -1;ret = m_pMpi->control(m_Ctx, MPP_DEC_SET_PARSER_SPLIT_MODE, (MppParam*)&need_split);if (MPP_OK != ret){LOG_INFO << ("mpi->control error MPP_DEC_SET_PARSER_SPLIT_MODE\n");return -1;}ret = mpp_init(m_Ctx, MPP_CTX_DEC, MPP_VIDEO_CodingAVC);  // 固定为H264if (MPP_OK != ret){LOG_INFO << ("mpp_init error\n");return -1;}//这个是为了后面测试使用的。真实使用不用。m_pOut_fp = fopen("/ics/test.yuv", "wb+");if (!m_pOut_fp){LOG_INFO << ("fopen error\n");return -1;}LOG_INFO << "--> CVideoDecodeThd::_InitMpp End";return 0;
}

_DecodeAVPacket:

int CVideoDecodeThd::_DecodeAVPacket(AVPacket *_pPacket)
{LOG_INFO << "--> CVideoDecodeThd::_DecodeAVPacket Start";RK_U32 pkt_done = 0;MPP_RET ret = MPP_OK;MppPacket packet = NULL;MppFrame  frame  = NULL;LOG_INFO << "--->z CVideoDecodeThd::_DecodeAVPacket _pPacket size:"<<_pPacket->size;if (_pPacket->size <= 0){LOG_INFO << "--> CVideoDecodeThd::_DecodeAVPacket _pPacket->size is:"<<_pPacket->size;return -1;}ret = mpp_packet_init(&packet, _pPacket->data, _pPacket->size);if(ret < 0){LOG_INFO << "mpp_packet_init fail ret:"<<ret;return -1;}mpp_packet_set_pts(packet, _pPacket->pts);do{RK_S32 times = 5;// send the packet first if packet is not doneif (!pkt_done){LOG_INFO << "pkt remain:" << mpp_packet_get_length(packet);ret = m_pMpi->decode_put_packet(m_Ctx, packet);if (MPP_OK == ret){LOG_INFO << "pkt send success remain:" << mpp_packet_get_length(packet);pkt_done = 1;}}// then get all available frame and releasedo {
try_again:ret = m_pMpi->decode_get_frame(m_Ctx, &frame);if (MPP_ERR_TIMEOUT == ret){if (times > 0) {times--;msleep(2);goto try_again;}qDebug() << "decode_get_frame failed too much time:" <<ret;}if (frame == NULL){qDebug() << "get frame:"<<frame;return -1;}qDebug() << "get MPP_OK:" <<MPP_OK;qDebug() << "get ret:"<<ret;LOG_INFO << ("decode_get_frame success\n") <<"||"<<frame;dump_frame_to_file(m_Ctx, m_pMpi, frame, m_pOut_fp);if (mpp_frame_get_eos(frame)){LOG_INFO << ("mpp_frame_get_eos\n");mpp_frame_deinit(&frame);//                over = 1;continue;}mpp_frame_deinit(&frame);break;}while(1);msleep(3);break;}while(1);LOG_INFO << "--> CVideoDecodeThd::_DecodeAVPacket End";mpp_packet_deinit(&packet);return 1;
}

dump_frame_to_file:

void CVideoDecodeThd::dump_frame_to_file(MppCtx ctx, MppApi *mpi, MppFrame frame, FILE *out_fp)
{LOG_INFO << "decode_and_dump_to_file:" << frame;MPP_RET ret;if (mpp_frame_get_info_change(frame)) {LOG_INFO << ("mpp_frame_get_info_change\n");/*** 第一次解码会到这个分支，需要为解码器设置缓冲区.* 解码器缓冲区支持3种模式。参考【图像内存分配以及交互模式】Rockchip_Developer_Guide_MPP_CN.pdf* 这里使用纯内部模式。*/ret = mpi->control(ctx, MPP_DEC_SET_INFO_CHANGE_READY, NULL);if (ret){LOG_INFO << "mpp_frame_get_info_change mpi->control error""MPP_DEC_SET_INFO_CHANGE_READY %d\n" << ret;}return;}RK_U32 err_info = mpp_frame_get_errinfo(frame);RK_U32 discard = mpp_frame_get_discard(frame);LOG_INFO << "err_info: discard:\n"<< err_info<<"||"<<discard;if (err_info){LOG_INFO << "--> CVideoDecodeThd::dump_frame_to_file err_info:"<<err_info;return;}// savedump_frame(frame, out_fp);return;
}

dump_frame:

void CVideoDecodeThd::dump_frame(MppFrame frame, FILE *out_fp)
{LOG_INFO << ("dump_frame_to_file\n");RK_U32 width    = 0;RK_U32 height   = 0;RK_U32 h_stride = 0;RK_U32 v_stride = 0;MppFrameFormat fmt  = MPP_FMT_YUV420SP;MppBuffer buffer    = NULL;RK_U8 *base = NULL;width    = mpp_frame_get_width(frame);height   = mpp_frame_get_height(frame);h_stride = mpp_frame_get_hor_stride(frame);v_stride = mpp_frame_get_ver_stride(frame);fmt      = mpp_frame_get_fmt(frame);buffer   = mpp_frame_get_buffer(frame);RK_U32 buf_size = mpp_frame_get_buf_size(frame);LOG_INFO << "w x h: %dx%d hor_stride:%d ver_stride:%d buf_size:%d\n"<<width<<"||"<< height<<"||"<<h_stride<<"||"<<v_stride<<"||"<< buf_size;if (NULL == buffer){LOG_INFO << ("buffer is null\n");return ;}base = (RK_U8 *)mpp_buffer_get_ptr(buffer);// MPP_FMT_YUV420SPif (fmt != MPP_FMT_YUV420SP){LOG_INFO << ("fmt %d not supported\n", fmt);return;}char *pOutput = NULL;int iLen = static_cast<int>(buf_size);LOG_INFO << "--->z iLen:"<<iLen;pOutput = (char*)base;if (iLen > 0 && pOutput){//用信号把QImage 发送出去emit SIGNAL_Image(QByteArray(pOutput, iLen), width, height);}
}

END、总结的知识与问题

1、一直出现jitter buffer full 这样的问题

原先开始跑，并不会出现这个问题，就是跑久了，直到抓了2000多个帧之后，慢慢就会报这个错，然后就不再拉流了，就很奇葩，这可是浪费了我很长的时间进行排查。
错误：

[rtsp @ 0x292ee690] jitter buffer full
[rtsp @ 0x292ee690] RTP: missed 84 packets
[rtsp @ 0x292ee690] jitter buffer full
[rtsp @ 0x292ee690] RTP: missed 8 packets

2、如何打印帧率？

    LOG_INFO << "the file video format is : " << m_pCodec->name;float fFrameRate = 0.0;if (m_pFormatCtx->streams[iVideoStream]->avg_frame_rate.den != 0 && m_pFormatCtx->streams[iVideoStream]->avg_frame_rate.num != 0){fFrameRate = m_pFormatCtx->streams[iVideoStream]->avg_frame_rate.num * 1.0 /m_pFormatCtx->streams[iVideoStream]->avg_frame_rate.den;//每秒多少帧LOG_INFO << "Frame Rate : " << fFrameRate;}

3、分析av_packet_alloc、av_init_packet、av_packet_unref、av_packet_free、av_frame_move_ref、av_packet_clone

参考：从源码的层面理解ffmpeg这几个API

参考

1、从源码的层面理解ffmpeg这几个API
2、GitHub - MUZLATAN/ffmpeg_rtsp_mpp: ffmpeg 拉取rtsp h264流， 使用mpp解码， 目前在firefly 板子上跑通了
3、RK3568 MPP编码
4、Rockchip MPP(Media Process Platform)解码H264
5、mpi_dec_test 解码失败