公司快速建站,建设软件资源网站,项目可行性报告怎样写,国内管理咨询公司1. 最近实在头疼#xff0c;因为项目换了平台。折腾来折腾去#xff0c;到今天算是把很多坑踩完了。
RK上实现硬解码方案一共有一下几种方式
1#xff09;opencvgstreamer插件#xff0c;采用硬解码#xff0c;只能解码出图像#xff0c;无法解出声音
2#xff09;ff…1. 最近实在头疼因为项目换了平台。折腾来折腾去到今天算是把很多坑踩完了。
RK上实现硬解码方案一共有一下几种方式
1opencvgstreamer插件采用硬解码只能解码出图像无法解出声音
2ffmpegRK的mpp方案转成cv::Mat或者QImage
3ffmpeg6.0x自带的rockitmpp硬解码器DRM渲染
4Rockit渲染框架
以上四种基本上都实现了。过程确实很艰辛掉了一大坨头发骂了无数次Rockit厂商。效果最好最完美的应属于Rockit方案。这套方案也是最难的最通用的。
话不多说上代码
方案1
#include GstDecoder.h
#include LogUtils.h
using namespace phm_decoder;
GstDecoder::GstDecoder(std::string root_ftp_url, std::string local_ftp_path): root_ftp_url_(root_ftp_url), local_ftp_path_(local_ftp_path) {}
GstDecoder::~GstDecoder() {}void GstDecoder::decode(const std::string rtsp, const int train_index, const int car_index, const int pos_index) {std::string pipeline rtspsrc location rtsp latency1000 protocolstcp ! rtph265depay ! h265parse ! mppvideodec ! videoconvert ! video/x-raw,format(string)BGR ! appsink ;Logger::info(pipeline : {}, pipeline);cv::VideoCapture video_capture;DecoderData data;data.online false;data.decoder_state false;data.train_index train_index;data.car_index car_index;data.pos_index pos_index;Logger::info(GstDecoder decode train_index{}, train_index);data.device_type 3;date_dirs_[rtsp] get_cur_date() /;video_capture.open(pipeline, cv::CAP_GSTREAMER);int reop 3;while (!video_capture.isOpened() reop 0) {Logger::error(video_capture open error reop times{}!, reop);std::unique_lockstd::mutex lock(image_mutex_);images_.push(data);reop--;if (reop 0) {return;} else {std::this_thread::sleep_for(std::chrono::milliseconds(1000));video_capture.release();video_capture.open(pipeline, cv::CAP_GSTREAMER);Logger::error(video_capture open error return);}}cv::Mat src;int count 0;int error_count 0;// 50 zhen biyou i zhenwhile (count 50) {// 读取摄像头内容也就是从摄像头拉流video_capture src;if (check_mat(src)) {count;} else {error_count;if (error_count 75) {Logger::info(75 frames is error ,but is ok!);break;}// Logger::info(mat is green read rtsp: {}, count:{}!, rtsp, count);}}std::string time_str get_cur_time();char car_index_str[12] {0};sprintf(car_index_str, _%d_%d.jpg, car_index, pos_index);std::string filename time_str std::string(car_index_str);std::string dir_str local_ftp_path_ date_dirs_[rtsp];std::string mkdir_cmd mkdir dir_str -p;system(mkdir_cmd.c_str());std::string local_path dir_str filename;Logger::info(gst img local_path:{}!, local_path);cv::imwrite(local_path.c_str(), src);system(sync);data.mat std::make_sharedcv::Mat(std::move(src));data.ftp_path root_ftp_url_ date_dirs_[rtsp] filename;data.online true;data.decoder_state true;Logger::info(data.ftp_path:{}!, data.ftp_path);// data.ftp_url root_ftp_url date_dirs_[rtsp] filename;std::unique_lockstd::mutex lock(image_mutex_);images_.push(data);std::unique_lockstd::mutex lock_c(decoder_mutex_);decoder_threads_.erase(rtsp);
}bool GstDecoder::check_mat(const cv::Mat mat) {// 遍历每个像素// sige jiaoint gree_count 0;int roi_col 200;int roi_row 200;int all_count 10000;for (int y 0; y roi_row; y) {for (int x 0; x roi_col; x) {// 获取像素点的 BGR 值cv::Vec3b pixel mat.atcv::Vec3b(y, x);int b (int)pixel[0];int g (int)pixel[1];int r (int)pixel[2];if (g 150 b 10 r 10) {gree_count;if (gree_count all_count) {return false;}}}}return true;
}这个在Rocki的文档里比较常见我这里只是为了拿到图片去检测。不用显示。所以没有后续处理了这个相对来说比较简单至于h264还是h265根据摄像机配置修改gstreamer插件名称就行了
方案2代码看个大概吧有一部分打开rtsp流的处理没有贴上来。这个方案的难点在与dump_mat这个函数就是拿到MppFrame后如何转成cv::mat或者QImage具体这个git上有其他兄弟写的源码可以参考他的https://github.com/MUZLATAN/ffmpeg_rtsp_mpp/blob/master/main.cpp
#pragma once
Rk3568mpp.h#include Demultiplex.h
#include rockchip/rk_mpi.h
#include iostream
#include memory
#include mutex
#include opencv2/opencv.hpp
#include queue
#include thread#pragma pack(1)
// 组播 234.1.1.2 端口 12306
typedef struct {unsigned char head; // 包头 0xFBunsigned char func; // 请求图片01图片上传成功02unsigned char ip[16]; // lcd ip地址unsigned char file_path[128]; // 图片上传至RK3568后上报完整路径unsigned char tail; //0XFC;
} lcd_img_msg;#pragma pack()namespace phm_decoder {class Rk3568mpp {// just for decode h264 or h265 videopublic:Rk3568mpp();~Rk3568mpp();bool init(MppCodingType type MPP_VIDEO_CodingHEVC);void quit();bool send_packet(AVPacket *pav_pack);bool read_frame(MppFrame frame);private:MppCtx ctx_;MppApi *mpi_;MppDecCfg cfg_;MppBufferGroup frm_grp_;
};
} // namespace phm_decoderRk3568mpp.cpp#include Rk3568mpp.h
#include LogUtils.h
extern C {
#include string.h
#include unistd.h
}
using namespace phm_decoder;
Rk3568mpp::Rk3568mpp() : ctx_(nullptr), cfg_(nullptr), frm_grp_(nullptr), mpi_(nullptr) {bool res init();if (!res) {Logger::error(Decoder init error!);return;}Logger::info(Decoder init success!);
}Rk3568mpp::~Rk3568mpp() {Logger::info( ~Rk3568mpp!);quit();
}bool Rk3568mpp::init(MppCodingType type) {MPP_RET ret MPP_OK;RK_U32 need_split 0;// split by h264 or h265 data typeret mpp_create(ctx_, mpi_);if (ret) {Logger::error(mpp_create failed);quit();return false;}MppParam param need_split;ret mpi_-control(ctx_, MPP_DEC_SET_PARSER_SPLIT_MODE, param);if (ret) {std::string info fmt::format(Failed to set MPP_DEC_SET_PARSER_SPLIT_MODE :{} ,ret:{}, (int)ret);Logger::error(info);quit();return false;}int fast 1;param fast;ret mpi_-control(ctx_, MPP_DEC_SET_PARSER_FAST_MODE, param);if (ret) {std::string info fmt::format(Failed to set MPP_DEC_SET_PARSER_SPLIT_MODE :{} ,ret:{}, (int)ret);Logger::error(info);quit();return false;}MppPollType timeout MPP_POLL_NON_BLOCK;param timeout;ret mpi_-control(ctx_, MPP_SET_OUTPUT_TIMEOUT, param);if (ret) {std::string info fmt::format(Failed to set output timeout :{} ,ret:{}, (int)timeout, (int)ret);Logger::error(info);quit();return false;}// MppCodingType type MPP_VIDEO_CodingAVC;ret mpp_init(ctx_, MPP_CTX_DEC, type);if (ret) {std::string info fmt::format(mpp_init failed type :{},ret:{}, (int)type, (int)ret);Logger::error(info);quit();return false;}// mpp_dec_cfg_init(cfg_);/* get default config from decoder context */// ret mpi_-control(ctx_, MPP_DEC_GET_CFG, cfg_);// if (ret) {// std::string info fmt::format(Failed mpi control ret:{}, (int)ret);// Logger::error(info);// quit();// return false;// }// /*// * split_parse is to enable mpp internal frame spliter when the input// * packet is not aplited into frames.// */// ret mpp_dec_cfg_set_u32(cfg_, base:split_parse, need_split);// if (ret) {// std::string info fmt::format(mpp_dec_cfg_set_u32ret ret:{}, (int)ret);// Logger::error(info);// quit();// return false;// }// ret mpi_-control(ctx_, MPP_DEC_SET_CFG, cfg_);// if (ret) {// // mpp_err(%p failed to set cfg %p ret %d\n, ctx, cfg, ret);// std::string info fmt::format(mpp_dec_cfg_set_u32ret ret:{}, (int)ret);// Logger::error(info);// quit();// return false;// }return true;
}void Rk3568mpp::quit() {if (ctx_) {MPP_RET ret mpi_-reset(ctx_);if (ret) {Logger::error(mpi-reset failed);}}if (ctx_) {mpp_destroy(ctx_);ctx_ nullptr;}if (cfg_) {mpp_dec_cfg_deinit(cfg_);cfg_ nullptr;}if (frm_grp_) {mpp_buffer_group_put(frm_grp_);frm_grp_ nullptr;}
}bool Rk3568mpp::send_packet(AVPacket *av_pack) {MPP_RET ret MPP_OK;MppPacket mpp_packet nullptr;std::string info_av fmt::format(decode av packet size:{}, av_pack-size);Logger::info(info_av);ret mpp_packet_init(mpp_packet, av_pack-data, av_pack-size);if (ret 0) {std::string init_inf fmt::format(mpp_packet_init fail ret:{}, (int)ret);Logger::error(init_inf);return false;}Logger::info(fmt::format(mpp_packet_init success));mpp_packet_set_pts(mpp_packet, av_pack-pts);int put_times 2;bool put_res false;do {MPP_RET ret mpi_-decode_put_packet(ctx_, mpp_packet);if (MPP_OK ret) {if (0 ! mpp_packet_get_length(mpp_packet)) {Logger::error(put data error!);}Logger::info(fmt::format(decode_put_packet success));put_res true;break;}usleep(2 * 1000);put_times--;Logger::info(fmt::format(mpi_-decode_put_packet error{}, (int)ret));} while (put_times 0);mpp_packet_deinit(mpp_packet);return put_res;
}bool Rk3568mpp::read_frame(MppFrame frame) {int get_frame_error_time 2;do {Logger::info(decode_get_frame start);RK_U32 frm_eos 0;MPP_RET ret mpi_-decode_get_frame(ctx_, frame);if (ret) {std::string info fmt::format(decodes_get_frame ret:{}, (int)ret);Logger::error(info);get_frame_error_time--;if (get_frame_error_time 0) {break;}usleep(2 * 1000);continue;}if (!frame) {Logger::info(read frame error,need send new pack);return false;}Logger::info(read decode_get_frame success!);if (mpp_frame_get_info_change(frame)) {Logger::info(mpp_frame_get_info_change);// found info change and create buffer group for decodingRK_U32 width mpp_frame_get_width(frame);RK_U32 height mpp_frame_get_height(frame);RK_U32 hor_stride mpp_frame_get_hor_stride(frame);RK_U32 ver_stride mpp_frame_get_ver_stride(frame);RK_U32 buf_size mpp_frame_get_buf_size(frame);Logger::info(decode_get_frame get info changed found);if (!frm_grp_) {/* If buffer group is not set create one and limit it */ret mpp_buffer_group_get_internal(frm_grp_, MPP_BUFFER_TYPE_ION);if (ret) {std::string error_info fmt::format(get mpp buffer group failed:{}, (int)ret);Logger::info(error_info);break;}/* Set buffer to mpp decoder */ret mpi_-control(ctx_, MPP_DEC_SET_EXT_BUF_GROUP, frm_grp_);if (ret) {std::string error_info fmt::format(set buffer group failed ret:{}, (int)ret);Logger::info(error_info);break;}} else {/* If old buffer group exist clear it */ret mpp_buffer_group_clear(frm_grp_);if (ret) {std::string error_info fmt::format(clear buffer group failed ret:{}, (int)ret);Logger::info(error_info);break;}}/* Use limit config to limit buffer count to 24 */ret mpp_buffer_group_limit_config(frm_grp_, buf_size, 24);if (ret) {std::string error_info fmt::format(limit buffer group failed ret:{}, (int)ret);Logger::info(error_info);break;}ret mpi_-control(ctx_, MPP_DEC_SET_INFO_CHANGE_READY, NULL);if (ret) {std::string error_info fmt::format(info change ready failed ret:{}, (int)ret);Logger::info(error_info);return false;}} else {std::string error_info fmt::format(read frame success);Logger::info(error_info);RK_U32 err_info mpp_frame_get_errinfo(frame);RK_U32 discard mpp_frame_get_discard(frame);if (!err_info !discard) {return true;} else {std::string error_info fmt::format(mpp_frame_get_errinfo err_info:{},discard:{}, (int)err_info, (int)discard);Logger::info(error_info);return false;}}} while (true);return false;
}// cv::Mat FfpDecoder::dump_mat(MppFrame frame, int ca_index) {// cv::Mat image_data;
// if (NULL frame)
// return image_data;
// RK_U32 width 0;
// RK_U32 height 0;
// MppFrameFormat fmt MPP_FMT_YUV420SP;
// MppBuffer buffer NULL;
// RK_U8 *base NULL;
// width mpp_frame_get_width(frame);
// height mpp_frame_get_height(frame);
// RK_U32 h_stride mpp_frame_get_hor_stride(frame);
// RK_U32 v_stride mpp_frame_get_ver_stride(frame);
// RK_U32 i 0;// fmt mpp_frame_get_fmt(frame);
// buffer mpp_frame_get_buffer(frame);
// std::string fmt_info fmt::format(video mpp_frame_get_fmt format :{}, (int)fmt);
// Logger::info(fmt_info);
// if (NULL buffer)
// return image_data;// base (RK_U8 *)mpp_buffer_get_ptr(buffer);// if (MPP_FRAME_FMT_IS_RGB(fmt) MPP_FRAME_FMT_IS_LE(fmt)) {
// fmt (MppFrameFormat)(fmt MPP_FRAME_FMT_MASK);
// std::string fmt_info fmt::format(video reset frame format :{}, (int)fmt);
// Logger::info(fmt_info);
// }// switch (fmt) {
// case MPP_FMT_YUV422SP: {// } break;
// case MPP_FMT_YUV420SP_VU: {
// RK_U8 *base_y base;
// RK_U8 *base_c base h_stride * v_stride;
// cv::Mat yuvMat;
// yuvMat.create(height * 3 / 2, width, CV_8UC1);
// //转为YUV420p格式
// int idx 0;
// for (i 0; i height; i, base_y h_stride) {
// memcpy(yuvMat.data idx, base_y, width);
// idx width;
// }
// for (i 0; i height / 2; i, base_c h_stride) {
// memcpy(yuvMat.data idx, base_c, width);
// idx width;
// }
// //这里的转码需要转为RGB 3通道 RGBA四通道则不能检测成功
// cv::cvtColor(yuvMat, image_data, cv::COLOR_YUV2BGR_NV12);
// } break;
// case MPP_FMT_YUV420SP: {
// RK_U8 *base_y base;
// RK_U8 *base_c base h_stride * v_stride;
// cv::Mat yuvMat;
// yuvMat.create(height * 3 / 2, width, CV_8UC1);
// //转为YUV420p格式
// int idx 0;
// for (i 0; i height; i, base_y h_stride) {
// memcpy(yuvMat.data idx, base_y, width);
// idx width;
// }
// for (i 0; i height / 2; i, base_c h_stride) {
// memcpy(yuvMat.data idx, base_c, width);
// idx width;
// }
// //这里的转码需要转为RGB 3通道 RGBA四通道则不能检测成功
// cv::cvtColor(yuvMat, image_data, cv::COLOR_YUV420sp2BGR);
// static int index 0;
// if (index 10) {
// index 0;
// }
// std::string filename fmt::format(./index{}_ca{}.jpg, index, ca_index);
// cv::imwrite(filename, image_data);// } break;
// case MPP_FMT_YUV420P: {
// RK_U8 *base_y base;
// RK_U8 *base_c base h_stride * v_stride;
// cv::Mat yuvMat;
// yuvMat.create(height * 3 / 2, width, CV_8UC1);
// //转为YUV420p格式
// int idx 0;
// for (i 0; i height; i, base_y h_stride) {
// memcpy(yuvMat.data idx, base_y, width);
// idx width;
// }
// for (i 0; i height / 2; i, base_c h_stride) {
// memcpy(yuvMat.data idx, base_c, width);
// idx width;
// }
// //这里的转码需要转为RGB 3通道 RGBA四通道则不能检测成功
// cv::cvtColor(yuvMat, image_data, cv::COLOR_YUV420p2BGR);// } break;
// case MPP_FMT_YUV444SP: {// } break;
// case MPP_FMT_YUV400: {
// } break;
// case MPP_FMT_ARGB8888:
// case MPP_FMT_ABGR8888:
// case MPP_FMT_BGRA8888:
// case MPP_FMT_RGBA8888: {
// } break;
// case MPP_FMT_RGB565:
// case MPP_FMT_BGR565:
// case MPP_FMT_RGB555:
// case MPP_FMT_BGR555:
// case MPP_FMT_RGB444:
// case MPP_FMT_BGR444: {
// } break;
// default: {
// std::string info fmt::format(not supported format :{}, (int)fmt);
// Logger::error(info);
// } break;
// }
// return image_data;
// }FfpDecoder.h
#pragma once#include DataDecoder.h
#include Demultiplex.h
#include rockchip/rk_mpi.h
#include iostream
#include memory
#include mutex
#include opencv2/opencv.hpp
#include queue
#include thread
namespace phm_decoder {class FfpDecoder : public DataDecoder {// just for decode h264 or h265 videopublic:FfpDecoder(std::string root_ftp_url, std::string local_ftp_path);~FfpDecoder();virtual void decode(const std::string rtsp, const int train_index, const int car_index, const int pos_index);cv::Mat dump_mat(MppFrame frame, int ca_index);private:std::string root_ftp_url_;std::string local_ftp_path_;std::mapstd::string, std::string date_dirs_;
};
} // namespace phm_decoderFfpDecoder.cpp
#include FfpDecoder.h
#include LogUtils.h
#include Rk3568mpp.h
extern C {
#include string.h
#include unistd.h
}
using namespace phm_decoder;
FfpDecoder::FfpDecoder(std::string root_ftp_url, std::string local_ftp_path): root_ftp_url_(root_ftp_url), local_ftp_path_(local_ftp_path) {}FfpDecoder::~FfpDecoder() {}void FfpDecoder::decode(const std::string rtsp, const int train_index, const int car_index, const int pos_index) {bool get_frame_ret false;DecoderData data;data.train_index train_index;data.car_index car_index;data.pos_index pos_index;data.device_type 3;std::string d_info fmt::format(decode rtsp url:{},camera_index:{}, rtsp, train_index, car_index, pos_index);Logger::info(d_info);bool decode_res false;Logger::error(rtsp:{}, rtsp);std::string rtsp_url rtsp;std::string info_url fmt::format(ready open rtsp_url rtsp:{}, rtsp_url);Logger::error(info_url);int reconnect_times 3;Demultiplex dp;dp.init();date_dirs_[rtsp] get_cur_date() /;while (active_ reconnect_times 0) {decode_res dp.open_rtsp(rtsp_url);if (!decode_res) {std::string info fmt::format(open rtsp error! reconnect_times:{}, rtsp:{}, reconnect_times, rtsp_url);Logger::error(info);usleep(100 * 1000);reconnect_times--;continue;} else {break;}}if (!decode_res) {data.online false;data.decoder_state false;std::unique_lockstd::mutex lock(image_mutex_);images_.push(data);std::unique_lockstd::mutex lock_c(decoder_mutex_);decoder_threads_.erase(rtsp);return;}Logger::info(open rtsp success!);int get_frame_count 2;dp.seed_i_frame();int read_times 4;Rk3568mpp rkmpp;bool res rkmpp.init();if (!res) {std::string info fmt::format(rkmpp init error);Logger::info(info);data.online false;data.decoder_state false;std::unique_lockstd::mutex lock(image_mutex_);images_.push(data);std::unique_lockstd::mutex lock_c(decoder_mutex_);decoder_threads_.erase(rtsp);return;}// read_times 0;while (active_ read_times 0) {read_times--;std::queueAVPacket * read_packet dp.read_packet(); // fasong I zhenwhile (!read_packet.empty() active_) {AVPacket *av_pack read_packet.front();if (av_pack-size 0) {read_packet.pop();av_packet_unref(av_pack);av_packet_free(av_pack);Logger::error(decode av packet size-size is error);continue;}res rkmpp.send_packet(av_pack);if (res) {read_packet.pop();av_packet_unref(av_pack);av_packet_free(av_pack);}int get_frame_error_time 2;Logger::info(fmt::format(decode_get_frame start));MppFrame frame nullptr;res rkmpp.read_frame(frame);if (!res) {Logger::info(read frame error,need send new pack);continue;}get_frame_count--;auto image std::make_sharedcv::Mat(std::move(dump_mat(frame, pos_index)));if (get_frame_count 0) {data.mat image;get_frame_ret true;std::string time_str get_cur_time();char car_index_str[12] {0};sprintf(car_index_str, _%d_%d.jpg, car_index, pos_index);std::string filename time_str std::string(car_index_str);std::string dir_str local_ftp_path_ date_dirs_[rtsp];std::string mkdir_cmd mkdir dir_str -p;system(mkdir_cmd.c_str());std::string local_path dir_str filename;Logger::info(gst img local_path:{}!, local_path);cv::imwrite(local_path.c_str(), *(data.mat.get()));system(sync);data.ftp_path root_ftp_url_ date_dirs_[rtsp] filename;Logger::info(ffmppeg get data.ftp_path:{}!, data.ftp_path);} else {image.reset();}mpp_frame_deinit(frame);if (get_frame_ret) {break;}}while (!read_packet.empty()) {AVPacket *pack read_packet.front();read_packet.pop();Logger::info(fmt::format(release read_packet data));av_packet_unref(pack);av_packet_free(pack);}if (get_frame_ret) {break;} else {Logger::info(fmt::format(read pack times :{}, read_times));usleep(200 * 1000);}}dp.close();data.online true;data.decoder_state get_frame_ret;if (!get_frame_ret) {Logger::info(send error to server);} else {Logger::info(capture camera image success!);}std::unique_lockstd::mutex lock(image_mutex_);images_.push(data);std::unique_lockstd::mutex lock_c(decoder_mutex_);decoder_threads_.erase(rtsp);
}cv::Mat FfpDecoder::dump_mat(MppFrame frame, int ca_index) {cv::Mat image_data;if (NULL frame) {Logger::error(dump_mat error:frame is null);return image_data;}RK_U32 width 0;RK_U32 height 0;MppFrameFormat fmt MPP_FMT_YUV420SP;MppBuffer buffer NULL;RK_U8 *base NULL;width mpp_frame_get_width(frame);height mpp_frame_get_height(frame);RK_U32 h_stride mpp_frame_get_hor_stride(frame);RK_U32 v_stride mpp_frame_get_ver_stride(frame);RK_U32 i 0;fmt mpp_frame_get_fmt(frame);buffer mpp_frame_get_buffer(frame);std::string fmt_info fmt::format(video mpp_frame_get_fmt format :{}, (int)fmt);Logger::info(fmt_info);if (NULL buffer) {Logger::error(dump_mat error:frame is null);return image_data;}base (RK_U8 *)mpp_buffer_get_ptr(buffer);if (MPP_FRAME_FMT_IS_RGB(fmt) MPP_FRAME_FMT_IS_LE(fmt)) {fmt (MppFrameFormat)(fmt MPP_FRAME_FMT_MASK);std::string fmt_info fmt::format(video reset frame format :{}, (int)fmt);Logger::info(fmt_info);}switch (fmt) {case MPP_FMT_YUV422SP: {} break;case MPP_FMT_YUV420SP_VU: {RK_U8 *base_y base;RK_U8 *base_c base h_stride * v_stride;cv::Mat yuvMat;yuvMat.create(height * 3 / 2, width, CV_8UC1);// 转为YUV420p格式int idx 0;for (i 0; i height; i, base_y h_stride) {memcpy(yuvMat.data idx, base_y, width);idx width;}for (i 0; i height / 2; i, base_c h_stride) {memcpy(yuvMat.data idx, base_c, width);idx width;}// 这里的转码需要转为RGB 3通道 RGBA四通道则不能检测成功cv::cvtColor(yuvMat, image_data, cv::COLOR_YUV2BGR_NV12);} break;case MPP_FMT_YUV420SP: {RK_U8 *base_y base;RK_U8 *base_c base h_stride * v_stride;cv::Mat yuvMat;yuvMat.create(height * 3 / 2, width, CV_8UC1);// 转为YUV420p格式int idx 0;for (i 0; i height; i, base_y h_stride) {memcpy(yuvMat.data idx, base_y, width);idx width;}for (i 0; i height / 2; i, base_c h_stride) {memcpy(yuvMat.data idx, base_c, width);idx width;}// 这里的转码需要转为RGB 3通道 RGBA四通道则不能检测成功cv::cvtColor(yuvMat, image_data, cv::COLOR_YUV420sp2BGR);static int index 0;if (index 10) {index 0;}// std::string filename fmt::format(./index{}_ca{}.jpg, index, ca_index);// cv::imwrite(filename, image_data);} break;case MPP_FMT_YUV420P: {RK_U8 *base_y base;RK_U8 *base_c base h_stride * v_stride;cv::Mat yuvMat;yuvMat.create(height * 3 / 2, width, CV_8UC1);// 转为YUV420p格式int idx 0;for (i 0; i height; i, base_y h_stride) {memcpy(yuvMat.data idx, base_y, width);idx width;}for (i 0; i height / 2; i, base_c h_stride) {memcpy(yuvMat.data idx, base_c, width);idx width;}// 这里的转码需要转为RGB 3通道 RGBA四通道则不能检测成功cv::cvtColor(yuvMat, image_data, cv::COLOR_YUV420p2BGR);} break;case MPP_FMT_YUV444SP: {} break;case MPP_FMT_YUV400: {} break;case MPP_FMT_ARGB8888:case MPP_FMT_ABGR8888:case MPP_FMT_BGRA8888:case MPP_FMT_RGBA8888: {} break;case MPP_FMT_RGB565:case MPP_FMT_BGR565:case MPP_FMT_RGB555:case MPP_FMT_BGR555:case MPP_FMT_RGB444:case MPP_FMT_BGR444: {} break;default: {std::string info fmt::format(not supported format :{}, (int)fmt);Logger::error(info);} break;}Logger::error(dump_mat success!!);return image_data;
}#include FfpDecoder.h
#include LogUtils.h
#include Rk3568mpp.h
extern C {
#include string.h
#include unistd.h
}
using namespace phm_decoder;
FfpDecoder::FfpDecoder(std::string root_ftp_url, std::string local_ftp_path): root_ftp_url_(root_ftp_url), local_ftp_path_(local_ftp_path) {}FfpDecoder::~FfpDecoder() {}void FfpDecoder::decode(const std::string rtsp, const int train_index, const int car_index, const int pos_index) {bool get_frame_ret false;DecoderData data;data.train_index train_index;data.car_index car_index;data.pos_index pos_index;data.device_type 3;std::string d_info fmt::format(decode rtsp url:{},camera_index:{}, rtsp, train_index, car_index, pos_index);Logger::info(d_info);bool decode_res false;Logger::error(rtsp:{}, rtsp);std::string rtsp_url rtsp;std::string info_url fmt::format(ready open rtsp_url rtsp:{}, rtsp_url);Logger::error(info_url);int reconnect_times 3;Demultiplex dp;dp.init();date_dirs_[rtsp] get_cur_date() /;while (active_ reconnect_times 0) {decode_res dp.open_rtsp(rtsp_url);if (!decode_res) {std::string info fmt::format(open rtsp error! reconnect_times:{}, rtsp:{}, reconnect_times, rtsp_url);Logger::error(info);usleep(100 * 1000);reconnect_times--;continue;} else {break;}}if (!decode_res) {data.online false;data.decoder_state false;std::unique_lockstd::mutex lock(image_mutex_);images_.push(data);std::unique_lockstd::mutex lock_c(decoder_mutex_);decoder_threads_.erase(rtsp);return;}Logger::info(open rtsp success!);int get_frame_count 2;dp.seed_i_frame();int read_times 4;Rk3568mpp rkmpp;bool res rkmpp.init();if (!res) {std::string info fmt::format(rkmpp init error);Logger::info(info);data.online false;data.decoder_state false;std::unique_lockstd::mutex lock(image_mutex_);images_.push(data);std::unique_lockstd::mutex lock_c(decoder_mutex_);decoder_threads_.erase(rtsp);return;}// read_times 0;while (active_ read_times 0) {read_times--;std::queueAVPacket * read_packet dp.read_packet(); // fasong I zhenwhile (!read_packet.empty() active_) {AVPacket *av_pack read_packet.front();if (av_pack-size 0) {read_packet.pop();av_packet_unref(av_pack);av_packet_free(av_pack);Logger::error(decode av packet size-size is error);continue;}res rkmpp.send_packet(av_pack);if (res) {read_packet.pop();av_packet_unref(av_pack);av_packet_free(av_pack);}int get_frame_error_time 2;Logger::info(fmt::format(decode_get_frame start));MppFrame frame nullptr;res rkmpp.read_frame(frame);if (!res) {Logger::info(read frame error,need send new pack);continue;}get_frame_count--;auto image std::make_sharedcv::Mat(std::move(dump_mat(frame, pos_index)));if (get_frame_count 0) {data.mat image;get_frame_ret true;std::string time_str get_cur_time();char car_index_str[12] {0};sprintf(car_index_str, _%d_%d.jpg, car_index, pos_index);std::string filename time_str std::string(car_index_str);std::string dir_str local_ftp_path_ date_dirs_[rtsp];std::string mkdir_cmd mkdir dir_str -p;system(mkdir_cmd.c_str());std::string local_path dir_str filename;Logger::info(gst img local_path:{}!, local_path);cv::imwrite(local_path.c_str(), *(data.mat.get()));system(sync);data.ftp_path root_ftp_url_ date_dirs_[rtsp] filename;Logger::info(ffmppeg get data.ftp_path:{}!, data.ftp_path);} else {image.reset();}mpp_frame_deinit(frame);if (get_frame_ret) {break;}}while (!read_packet.empty()) {AVPacket *pack read_packet.front();read_packet.pop();Logger::info(fmt::format(release read_packet data));av_packet_unref(pack);av_packet_free(pack);}if (get_frame_ret) {break;} else {Logger::info(fmt::format(read pack times :{}, read_times));usleep(200 * 1000);}}dp.close();data.online true;data.decoder_state get_frame_ret;if (!get_frame_ret) {Logger::info(send error to server);} else {Logger::info(capture camera image success!);}std::unique_lockstd::mutex lock(image_mutex_);images_.push(data);std::unique_lockstd::mutex lock_c(decoder_mutex_);decoder_threads_.erase(rtsp);
}cv::Mat FfpDecoder::dump_mat(MppFrame frame, int ca_index) {cv::Mat image_data;if (NULL frame) {Logger::error(dump_mat error:frame is null);return image_data;}RK_U32 width 0;RK_U32 height 0;MppFrameFormat fmt MPP_FMT_YUV420SP;MppBuffer buffer NULL;RK_U8 *base NULL;width mpp_frame_get_width(frame);height mpp_frame_get_height(frame);RK_U32 h_stride mpp_frame_get_hor_stride(frame);RK_U32 v_stride mpp_frame_get_ver_stride(frame);RK_U32 i 0;fmt mpp_frame_get_fmt(frame);buffer mpp_frame_get_buffer(frame);std::string fmt_info fmt::format(video mpp_frame_get_fmt format :{}, (int)fmt);Logger::info(fmt_info);if (NULL buffer) {Logger::error(dump_mat error:frame is null);return image_data;}base (RK_U8 *)mpp_buffer_get_ptr(buffer);if (MPP_FRAME_FMT_IS_RGB(fmt) MPP_FRAME_FMT_IS_LE(fmt)) {fmt (MppFrameFormat)(fmt MPP_FRAME_FMT_MASK);std::string fmt_info fmt::format(video reset frame format :{}, (int)fmt);Logger::info(fmt_info);}switch (fmt) {case MPP_FMT_YUV422SP: {} break;case MPP_FMT_YUV420SP_VU: {RK_U8 *base_y base;RK_U8 *base_c base h_stride * v_stride;cv::Mat yuvMat;yuvMat.create(height * 3 / 2, width, CV_8UC1);// 转为YUV420p格式int idx 0;for (i 0; i height; i, base_y h_stride) {memcpy(yuvMat.data idx, base_y, width);idx width;}for (i 0; i height / 2; i, base_c h_stride) {memcpy(yuvMat.data idx, base_c, width);idx width;}// 这里的转码需要转为RGB 3通道 RGBA四通道则不能检测成功cv::cvtColor(yuvMat, image_data, cv::COLOR_YUV2BGR_NV12);} break;case MPP_FMT_YUV420SP: {RK_U8 *base_y base;RK_U8 *base_c base h_stride * v_stride;cv::Mat yuvMat;yuvMat.create(height * 3 / 2, width, CV_8UC1);// 转为YUV420p格式int idx 0;for (i 0; i height; i, base_y h_stride) {memcpy(yuvMat.data idx, base_y, width);idx width;}for (i 0; i height / 2; i, base_c h_stride) {memcpy(yuvMat.data idx, base_c, width);idx width;}// 这里的转码需要转为RGB 3通道 RGBA四通道则不能检测成功cv::cvtColor(yuvMat, image_data, cv::COLOR_YUV420sp2BGR);static int index 0;if (index 10) {index 0;}// std::string filename fmt::format(./index{}_ca{}.jpg, index, ca_index);// cv::imwrite(filename, image_data);} break;case MPP_FMT_YUV420P: {RK_U8 *base_y base;RK_U8 *base_c base h_stride * v_stride;cv::Mat yuvMat;yuvMat.create(height * 3 / 2, width, CV_8UC1);// 转为YUV420p格式int idx 0;for (i 0; i height; i, base_y h_stride) {memcpy(yuvMat.data idx, base_y, width);idx width;}for (i 0; i height / 2; i, base_c h_stride) {memcpy(yuvMat.data idx, base_c, width);idx width;}// 这里的转码需要转为RGB 3通道 RGBA四通道则不能检测成功cv::cvtColor(yuvMat, image_data, cv::COLOR_YUV420p2BGR);} break;case MPP_FMT_YUV444SP: {} break;case MPP_FMT_YUV400: {} break;case MPP_FMT_ARGB8888:case MPP_FMT_ABGR8888:case MPP_FMT_BGRA8888:case MPP_FMT_RGBA8888: {} break;case MPP_FMT_RGB565:case MPP_FMT_BGR565:case MPP_FMT_RGB555:case MPP_FMT_BGR555:case MPP_FMT_RGB444:case MPP_FMT_BGR444: {} break;default: {std::string info fmt::format(not supported format :{}, (int)fmt);Logger::error(info);} break;}Logger::error(dump_mat success!!);return image_data;
}方案3
比较头疼的是如何编译带h264_rkmpp与hevc_rkmpp硬解码器的ffmpeg6.0版本。这个自行编译吧。我也是别人给的。有小伙伴私信找我要我没给。所以别找我要了。应该不难。
这个难点除了ffmpeg6.0外就是DRM渲染了重点是plane的层级选择。一般设备是crtc和conn_id就一个但是图层是有多个的具体渲染到哪个层需要自己指定。
#include player_th.h
#include stdio.h
#include QDebug
#include form_video_player.h
player_th::player_th(QObject *parent,int x,int y,int width,int height) :QThread(parent),x_(x),y_(y),width_(width),height_(height)
{video_stream_idx -1;audio_stream_idx-1;quit_false;frameNULL;scale1;last_scale0;seek_times0;last_seek_times0;need_switch_ true;fmt_ctxNULL;video_dec_ctxNULL;pSwsContextNULL;qDebug()player_th;video_track_new VideoTrack();video_track_-set_display_area(x_,y_,width_,height_);Init();parent_ (form_video_player*)(parent);
}player_th::~player_th()
{stop_ true;quit_ true;this-msleep(1000);
}
void player_th::set_scale(float s)
{cmd_idCMD_SCALE;scales;
}QString player_th::ffmpeg_error_string(int errnum)
{char error_buf[AV_ERROR_MAX_STRING_SIZE] {0};return QString(QLatin1String(av_make_error_string(error_buf, AV_ERROR_MAX_STRING_SIZE, errnum)));
}void player_th::SetUrl(QString str){QMutexLocker locker(url_lock_);cur_rtsp_url_str;if(cur_rtsp_url_!str){need_switch_ true;}else{need_switch_ false;}
}bool player_th::Init()
{qDebug()start Init;total_playtime_s0;video_streamNULL;cmd_idCMD_PLAY;last_cmd_idCMD_PLAY;if(fmt_ctxNULL){fmt_ctx avformat_alloc_context();//申请一个AVFormatContext结构的内存,并进行简单初始化}return true;
}bool player_th::open_rtsp(QString rtsp_url){AVDictionary* dco NULL;av_dict_set(dco, rtsp_transport, tcp, 0);av_dict_set(dco, max_analyze_duration, 3, 0);av_dict_set(dco, stimeout, 3000000, 0);if (avformat_open_input(fmt_ctx, rtsp_url.toStdString().c_str(), NULL, dco) 0){fprintf(stderr, Could not open source file %s\n,rtsp_url.toStdString().c_str());return false;}av_dict_free(dco);/* retrieve stream information */if (avformat_find_stream_info(fmt_ctx, NULL) 0){fprintf(stderr, Could not find stream information\n);return false;}if (open_codec_context(video_stream_idx, video_dec_ctx, fmt_ctx, AVMEDIA_TYPE_VIDEO) 0){video_stream fmt_ctx-streams[video_stream_idx];/* allocate image where the decoded image will be put */video_width video_dec_ctx-width;video_height video_dec_ctx-height;pix_fmt video_dec_ctx-pix_fmt;}else{return false;}if (!video_stream){fprintf(stderr, Could not find audio or video stream in the input, aborting\n);return false;}if(fmt_ctx-duration!AV_NOPTS_VALUE){qDebug()pAVFormatContext-durationfmt_ctx-duration/AV_TIME_BASE;total_playtime_sfmt_ctx-duration/AV_TIME_BASE;emit realplay_time(0,total_playtime_s);}if(frameNULL){frameav_frame_alloc();}pkt av_packet_alloc();if (!pkt){fprintf(stderr, Could not allocate packet\n);return false;}if(!video_track_){video_track_new VideoTrack();video_track_-set_display_area(x_,y_,width_,height_);}qDebug()fmt_ctxduration:fmt_ctx-duration;qDebug()width:video_widthheightvideo_heighttotal_playtime_s:total_playtime_sAV_NOPTS_VALUEAV_NOPTS_VALUE;return true;
}void player_th::run()
{quit_false;while(!quit_){if(stop_ || !parent_-isVisible()){msleep(1000);continue;}QMutexLocker locker(url_lock_);if(cmd_idCMD_PLAY !open_rtsp(cur_rtsp_url_)){msleep(1000);continue;}need_switch_ false;while (!quit_!stop_ !need_switch_ parent_-isVisible()){if(cmd_idCMD_PAUSElast_cmd_id!CMD_PAUSE){av_read_pause(fmt_ctx);last_cmd_idCMD_PAUSE;qDebug()pause!!!!!!!!!endl;}else if(cmd_idCMD_PLAYlast_cmd_id!CMD_PLAY){av_read_play(fmt_ctx);last_cmd_idCMD_PLAY;}else if(cmd_idCMD_SEEK)//last_cmd_id!CMD_PAUSE{if(seek_times!last_seek_times){seek_replay_sec(seek_times);last_seek_timesseek_times;// qDebug()seek change:seek_times;}}else if (cmd_id CMD_SCALE last_cmd_id!CMD_PAUSE){if(last_scale!scale){qDebug()cmd_id CMD_SCALE :scale;av_read_pause(fmt_ctx);fmt_ctx-scalescale;av_read_play(fmt_ctx);last_scalescale;}}else if(cmd_idCMD_STOP){stop_true;break;}else if(cmd_id CMD_RESUME){av_read_play(fmt_ctx);last_cmd_idCMD_RESUME;}if(cmd_idCMD_PAUSE){msleep(25);continue;}if (av_read_frame(fmt_ctx, pkt) 0){int ret0;if (pkt-stream_index video_stream_idx){ret decode_packet(video_dec_ctx, pkt);float play_positionpkt-pts*r2d(video_stream-time_base);emit realplay_time(play_position,total_playtime_s);}av_packet_unref(pkt);if (ret 0){qDebug()break!!!!ret;break;}msleep(5);//play local video need control speed.}else{break;}}free_content();}
}bool player_th::seek_replay_sec(int seek_time)
{//seek到指定的秒qDebug()seek_replay_sec:seek_time;cmd_idCMD_SEEK;int64_t seek_target seek_time * AV_TIME_BASE;int stream_indexvideo_stream_idx;AVStream * s NULL;svideo_stream;//seek_target av_rescale_q(seek_target, AV_TIME_BASE_Q,s-time_base);AVRational bq {1, AV_TIME_BASE};seek_target av_rescale_q(seek_target, bq,s-time_base);if (s-start_time ! AV_NOPTS_VALUE)seek_target s-start_time;// start_io_timeout_check();int ret av_seek_frame(fmt_ctx, stream_index, seek_target, AVSEEK_FLAG_BACKWARD);// stop_io_timeout_check();if (ret 0){// LOG_ERROR() av_seek_frame failed, ffmpeg_error_string(ret);qDebug()av_seek_frame failed!!!!;return false;}else{return true;}}void player_th::stop_play(){stop_ true;
}void player_th::start_play()
{stop_ false;}
int player_th::open_codec_context(int *stream_idx,AVCodecContext **dec_ctx, AVFormatContext *fmt_ctx, enum AVMediaType type)
{int ret, stream_index;AVStream *st;const AVCodec *dec NULL;ret av_find_best_stream(fmt_ctx, type, -1, -1, NULL, 0);if (ret 0) {fprintf(stderr, Could not find %s stream in input file\n,av_get_media_type_string(type));return ret;} else {stream_index ret;st fmt_ctx-streams[stream_index];/* find decoder for the stream */// dec avcodec_find_decoder(st-codecpar-codec_id);dec avcodec_find_decoder_by_name(h264_rkmpp);//hevc_rkmpp// dec avcodec_find_decoder_by_name(hevc_rkmpp);if (!dec) {fprintf(stderr, Failed to find %s codec\n,av_get_media_type_string(type));return AVERROR(EINVAL);}/* Allocate a codec context for the decoder */*dec_ctx avcodec_alloc_context3(dec);if (!*dec_ctx) {fprintf(stderr, Failed to allocate the %s codec context\n,av_get_media_type_string(type));return AVERROR(ENOMEM);}/* Copy codec parameters from input stream to output codec context */if ((ret avcodec_parameters_to_context(*dec_ctx, st-codecpar)) 0) {fprintf(stderr, Failed to copy %s codec parameters to decoder context\n,av_get_media_type_string(type));return ret;}/* Init the decoders */if ((ret avcodec_open2(*dec_ctx, dec, NULL)) 0) {fprintf(stderr, Failed to open %s codec\n,av_get_media_type_string(type));return ret;}*stream_idx stream_index;}return 0;
}int player_th::decode_packet(AVCodecContext *dec, const AVPacket *pkt)
{int ret -1;AVFrame *pFrameOK NULL;// submit the packet to the decoderret avcodec_send_packet(dec, pkt);if (ret 0){qDebug()avcodec_send_packet error:ffmpeg_error_string(ret);}// get all the available frames from the decoderwhile (!quit_!stop_ret 0){if (!(pFrameOK av_frame_alloc())){fprintf(stderr, Can not alloc frame\n);ret AVERROR(ENOMEM);qDebug()av_frame_alloc error;// goto fail;}ret avcodec_receive_frame(dec, frame);if (ret 0){// those two return values are special and mean there is no output// frame available, but there were no errors during decodingif (ret AVERROR_EOF || ret AVERROR(EAGAIN)){//无可用帧//qDebug()avcodec_receive_frame error:ffmpeg_error_string(ret);return 0;}elseqDebug()avcodec_receive_frame error:ffmpeg_error_string(ret);return ret;}else{//开始解码// qDebug()frame format:frame-formatAV_PIX_FMT_NV12:AV_PIX_FMT_NV12AV_PIX_FMT_DRM_PRIMEAV_PIX_FMT_DRM_PRIME;;if(video_track_!NULL)video_track_-DisplayImage(frame);}av_frame_unref(frame);av_frame_free(pFrameOK);if (ret 0)return ret;}return 0;
}void player_th::free_content()
{qDebug()free_content;if(video_dec_ctx!NULL){qDebug()avcodec_free_context;avcodec_free_context(video_dec_ctx);video_dec_ctxNULL;}if(fmt_ctx!NULL){qDebug()fmt_ctx free;avformat_close_input(fmt_ctx);}if(pkt!NULL){qDebug()pkt free;av_packet_free(pkt);pktNULL;}if(frame!NULL){qDebug()frame free;av_frame_free(frame);frameNULL;}if(video_track_!NULL){delete video_track_;video_track_ NULL;}
}void player_th::stop_display()
{quit_true;
}DRM渲染
extern C{
#include libavutil/hwcontext_drm.h
}
#include unistd.h
#include sys/types.h
#include fcntl.h
#include DirectKmsSink.h
//#include log.hbool DirectKmsSink::Init(int w, int h, AVPixelFormat format)
{if (inited_)return true;pix_fmt_ format;int ret;fd_ open(/dev/dri/card0, O_RDWR | O_CLOEXEC);if (fd_ 0) {printf(failed to open card0, %s, strerror(errno));goto err;}res_ drmModeGetResources(fd_);if (!res_) {printf(drmModeGetResources failed: %s (%d)\n, strerror(errno), errno);goto err;}printf(!!!!!res_-count_crtcs:%d,!!!!res_-count_connectors:%d\n,res_-count_crtcs,res_-count_connectors);if (!res_-count_crtcs || !res_-count_connectors) {printf(count_crtcs(%d) or count_connectors(%d) failed\n, res_-count_crtcs, res_-count_connectors);goto err;}crtc_id_ res_-crtcs[0];//3conn_id_ res_-connectors[0];printf(kms crtc_id%u conn_id%u\n, crtc_id_, conn_id_);crtc_ drmModeGetCrtc (fd_, crtc_id_);if (!crtc_) {printf(drmModeGetCrtc failed: %s (%d), strerror(errno), errno);goto err;}mode_display_w_ crtc_-mode.hdisplay;mode_display_h_ crtc_-mode.vdisplay;printf(kms mode_display_w_%u mode_display_h_%u\n, mode_display_w_, mode_display_h_);plane_res_ drmModeGetPlaneResources(fd_);if (!plane_res_) {printf(drmModeGetPlaneResources failed: %s (%d), strerror(errno), errno);goto err;}plane_id_ FindPlaneForCrtc();if (plane_id_ -1){printf(FindPlaneForCrtc failed: %u, plane_id_);goto err;}printf(kms plane_id%u\n, plane_id_);inited_ true;return true;err:if(plane_res_){drmModeFreePlaneResources(plane_res_);plane_res_ nullptr;} if(crtc_){drmModeFreeCrtc(crtc_);crtc_ nullptr;} if(res_){drmModeFreeResources(res_);res_ nullptr;} if(fd_ 0){close(fd_);fd_ -1;} return false;
}DirectKmsSink::~DirectKmsSink()
{for (int i 0; i kBufferCount; i){DrmBuffer* next_buf buffers_[i];if (next_buf-fb_id ! -1){if (drmModeRmFB(fd_, next_buf-fb_id) ! 0)printf(cant remove fb %d\n, next_buf-fb_id);}}if(plane_res_)drmModeFreePlaneResources(plane_res_);if(crtc_)drmModeFreeCrtc(crtc_);if(res_)drmModeFreeResources(res_);if(fd_ 0)close(fd_);
}uint32_t DirectKmsSink::FindPlaneForCrtc()
{drmModePlane *plane;int i, pipe;plane NULL;pipe -1;for (i 0; i res_-count_crtcs; i) {if (crtc_id_ res_-crtcs[i]) {pipe i;break;}}if (pipe -1)return -1;printf(-----plane_res_-count_planes:%d\n,plane_res_-count_planes);for (i 0; i plane_res_-count_planes; i){plane drmModeGetPlane (fd_, plane_res_-planes[i]);if(plane){printf(-----drmModeGetPlane success:plane crtc_id %d\n,plane-crtc_id);}}for (i 0; i plane_res_-count_planes; i) {plane drmModeGetPlane (fd_, plane_res_-planes[i]);if (plane ){printf(-----drmModeGetPlane success:planes pipe %d\n,pipe);if (plane-possible_crtcs (1 pipe) plane-plane_id101){uint32_t plane_id plane-plane_id;drmModeFreePlane(plane);return plane_id;} drmModeFreePlane(plane);}}return -1;
}void DirectKmsSink::SetDisplayArea(int x, int y, int w, int h)
{display_x_ x;display_y_ y;display_w_ w;display_h_ h;// display_x_ 0;
// display_y_ 0;
// display_w_ 1024;
// display_h_ 768;//超出屏幕大小时错误处理if ((display_x_ display_w_) mode_display_w_)display_w_ mode_display_w_ - display_x_;if ((display_y_ display_h_) mode_display_h_)display_h_ mode_display_h_ - display_y_;printf(set display area x:%d y:%d w:%d h:%d\n, display_x_, display_y_, display_w_, display_h_);
}bool DirectKmsSink::ShowFrame(AVFrame *frame)
{DrmBuffer* drm_buf buffers_[drm_buf_idx_];AVDRMFrameDescriptor* desc (AVDRMFrameDescriptor *) frame-data[0];drm_buf-dbuf_fd desc-objects[0].fd;for (int i 0; i desc-layers-nb_planes i 4; i ) {drm_buf-pitches[i] desc-layers-planes[i].pitch;drm_buf-offsets[i] desc-layers-planes[i].offset;}drm_format_ desc-layers[0].format;drm_buf-fourcc drm_format_;if (!Display(drm_buf, frame-width, frame-height))return false;drm_buf_idx_ next_buffer_index();return true;}bool DirectKmsSink::Display(DrmBuffer* drm_buf, int width, int height)
{int ret;uint32_t bo_handle;ret drmPrimeFDToHandle(fd_, drm_buf-dbuf_fd, bo_handle);if (ret 0) {printf(cannot import dmabuf %d, fd%d\n, ret, drm_buf-dbuf_fd);return false;}uint32_t bo_handles[4];bo_handles[0] bo_handle;bo_handles[1] bo_handle;bo_handles[2] 0;bo_handles[3] 0;ret drmModeAddFB2(fd_, width, height, drm_buf-fourcc, bo_handles,drm_buf-pitches, drm_buf-offsets, drm_buf-fb_id, 0);if (ret 0) {printf(cannot add framebuffer %d\n, ret);}struct drm_gem_close gem_close;memset(gem_close, 0, sizeof gem_close);gem_close.handle bo_handle;if (drmIoctl(fd_, DRM_IOCTL_GEM_CLOSE, gem_close) 0)printf(cant close gem: %s, strerror(errno));if (ret 0)return false;ret drmModeSetPlane(fd_, plane_id_, crtc_id_,drm_buf-fb_id, 0,display_x_, display_y_, display_w_, display_h_,0, 0, width 16, height 16);if (ret ! 0){printf(drmModeSetPlane failed %d\n, ret);}DrmBuffer* next_buf buffers_[next_buffer_index()];if (next_buf-fb_id ! -1){if (drmModeRmFB(fd_, next_buf-fb_id) ! 0)printf(cant remove fb %d\n, next_buf-fb_id);elsenext_buf-fb_id -1;}return ret 0;}int DirectKmsSink::next_buffer_index()
{int index drm_buf_idx_ 1;if (index kBufferCount)index 0;return index;
}方案4这个才是解决问题的终极奥义。虽然RK官方文档很垃圾当初拿到的demo还是旧版的。甚至调了很久发现不行去问RK官方才知道他们是有新版的就是linux5.1还有rockit最新版。其实内核不用更新只需要用最新的rockit库就行了。
这里就不放大量代码了代码太多具体可参考demo新版demo已经很详细了。
重点说一下遇到的问题和调试方法
视频一开始看不到发送码流失败可能是初始化解码缓冲区的问题。按照官方的demo这样干是可以的主要是注意解码的格式和缓存区大小还有h264,h265以及送帧方式。 RK_S32 rk_vdec::mpi_create_vdec(RK_S32 s32Ch, VIDEO_MODE_E enMode, int max_pic_width, int max_pic_height,bool enable_h264)
{RK_S32 s32Ret RK_SUCCESS;VDEC_CHN_ATTR_S stAttr;VDEC_CHN_PARAM_S stVdecParam;MB_POOL_CONFIG_S stMbPoolCfg;VDEC_PIC_BUF_ATTR_S stVdecPicBufAttr;MB_PIC_CAL_S stMbPicCalResult;VDEC_MOD_PARAM_S stModParam;std::cout RK_S32 rk_vdec::mpi_create_vdec s32Ch: s32Ch std::endl;memset(stAttr, 0, sizeof(VDEC_CHN_ATTR_S));memset(stVdecParam, 0, sizeof(VDEC_CHN_PARAM_S));memset(stModParam, 0, sizeof(VDEC_MOD_PARAM_S));RK_BOOL bEnableMbPool RK_TRUE;if (bEnableMbPool){stModParam.enVdecMBSource MB_SOURCE_USER;s32Ret RK_MPI_VDEC_SetModParam(stModParam);if (s32Ret ! RK_SUCCESS){RK_LOGE(vdec %d SetModParam failed! errcode %x, s32Ch, s32Ret);std::cout RK_MPI_VDEC_SetModParam failed std::endl;return s32Ret;}}stAttr.enMode VIDEO_MODE_FRAME;if (enable_h264){stAttr.enType RK_VIDEO_ID_AVC;}else{stAttr.enType RK_VIDEO_ID_HEVC;}stAttr.u32PicWidth max_pic_width;stAttr.u32PicHeight max_pic_height;stAttr.u32PicVirWidth max_pic_width;stAttr.u32PicVirHeight max_pic_height;stAttr.stVdecVideoAttr.u32TmvBufSize max_pic_width * max_pic_height * 4;//*3 / 2;stAttr.stVdecVideoAttr.bTemporalMvpEnable RK_TRUE;stAttr.stVdecVideoAttr.u32RefFrameNum 5;stAttr.u32FrameBufSize max_pic_width * max_pic_height * 4;// / 2;stAttr.u32FrameBufCnt 8;stAttr.u32StreamBufCnt 8;stVdecParam.stVdecVideoParam.enOutputOrder VIDEO_OUTPUT_ORDER_DISP;chn_ s32Ch;s32Ret RK_MPI_VDEC_CreateChn(chn_, stAttr);if (s32Ret ! RK_SUCCESS){RK_LOGE(create chn_ %d vdec failed! , chn_);std::cout RK_MPI_VDEC_CreateChn failed std::endl;return s32Ret;}if (enable_h264){stVdecParam.enType RK_VIDEO_ID_AVC;}else{stVdecParam.enType RK_VIDEO_ID_HEVC;}stVdecParam.stVdecVideoParam.enDecMode VIDEO_DEC_MODE_IPB;// VIDEO_OUTPUT_ORDER_DISPstVdecParam.stVdecVideoParam.enOutputOrder VIDEO_OUTPUT_ORDER_DEC;stVdecParam.stVdecVideoParam.enCompressMode COMPRESS_MODE_NONE; //(COMPRESS_MODE_E)ctx-u32CompressMode;s32Ret RK_MPI_VDEC_SetChnParam(chn_, stVdecParam);if (s32Ret ! RK_SUCCESS){RK_LOGE(set chn %d param failed %x! , chn_, s32Ret);std::cout RK_MPI_VDEC_SetChnParam failed std::endl;return s32Ret;}int s32ChnFd RK_MPI_VDEC_GetFd(chn_);if (s32ChnFd 0){RK_LOGE(get fd chn %d failed %d, chn_, s32ChnFd);std::cout RK_MPI_VDEC_GetFd failed std::endl;return s32Ret;}if (bEnableMbPool){memset(stMbPoolCfg, 0, sizeof(MB_POOL_CONFIG_S));if (enable_h264){stVdecPicBufAttr.enCodecType RK_VIDEO_ID_AVC;}else{stVdecPicBufAttr.enCodecType RK_VIDEO_ID_HEVC;}stVdecPicBufAttr.stPicBufAttr.u32Width max_pic_width;stVdecPicBufAttr.stPicBufAttr.u32Height max_pic_height;std::cout stVdecPicBufAttr :width max_pic_width height max_pic_height std::endl;stVdecPicBufAttr.stPicBufAttr.enPixelFormat RK_FMT_RGBA8888;// RK_FMT_YUV420SP;stVdecPicBufAttr.stPicBufAttr.enCompMode COMPRESS_MODE_NONE;s32Ret RK_MPI_CAL_VDEC_GetPicBufferSize(stVdecPicBufAttr, stMbPicCalResult);if (s32Ret ! RK_SUCCESS){RK_LOGE(get picture buffer size failed. err 0x%x, s32Ret);std::cout RK_MPI_CAL_VDEC_GetPicBufferSize failed std::endl;return s32Ret;}std::cout stMbPoolCfg.u32MBCnt stMbPoolCfg.u32MBCnt std::endl;stMbPoolCfg.u64MBSize stMbPicCalResult.u32MBSize;stMbPoolCfg.u32MBCnt 10;stMbPoolCfg.enAllocType MB_ALLOC_TYPE_DMA;stMbPoolCfg.enDmaType MB_DMA_TYPE_NONE;stMbPoolCfg.enRemapMode MB_REMAP_MODE_CACHED;stMbPoolCfg.bPreAlloc RK_TRUE;MB_POOL s32Pool RK_MPI_MB_CreatePool(stMbPoolCfg);if (s32Pool MB_INVALID_POOLID){RK_LOGE(create pool failed!);std::cout RK_MPI_MB_CreatePool failed std::endl;return s32Ret;}s32Ret RK_MPI_VDEC_AttachMbPool(chn_, s32Pool);if (s32Ret ! RK_SUCCESS){std::cout RK_MPI_VDEC_AttachMbPool failed std::endl;RK_LOGE(attatc vdec mb pool %d failed! , chn_);return s32Ret;}}s32Ret RK_MPI_VDEC_StartRecvStream(chn_);if (s32Ret ! RK_SUCCESS){RK_LOGE(start recv chn %d failed %x! , chn_, s32Ret);std::cout RK_MPI_VDEC_StartRecvStream failed std::endl;return s32Ret;}std::cout RK_MPI_VDEC_StartRecvStream::RK_MPI_VDEC_StartRecvStream( std::endl;s32Ret RK_MPI_VDEC_SetDisplayMode(chn_, VIDEO_DISPLAY_MODE_PLAYBACK);if (s32Ret ! RK_SUCCESS){RK_LOGE(RK_MPI_VDEC_SetDisplayMode failed with %#x!, s32Ret);return s32Ret;}std::cout chn_ VDEC SUCESS ----------------- : chn_ std::endl;return RK_SUCCESS;
}
问题2视频卡顿这个问题很他妈的奇怪一开始是以为是RK的bug还是怎么。调试了很久画面出来了但是动不动就卡顿花屏。快要崩溃边缘。手上活太多搁置了两个礼拜。最后去问RK官方对方说你拉的是摄像机可能网络丢帧让我把拉流的画面存下来对比。
后来想想那我都改成TCP方式了还会丢帧?软解码没问题啊。日了狗。最后相信干脆播放本地文件吧。于是就播放本地文件调试了两天才能正常播放本地文件。因为一次性初始化通道数量太少居然不显示。这个问题没有深究反正每次初始化五个VO通道就行了。 本地视频没有花屏但是卡顿还有。 后来鬼使神差把双线程改成了单线程播放结果正常了我欣喜异常心喜妈卖批的终于好了。
以下是送帧示例仔细的伙伴已经发现问题了。那里那个回调函数free_stream_Data干什么用的
其实这个就是为什么卡顿的真实原因因为之前卡顿是每次从av_read_frame读到AVPackt的时候就立即调用这个函数发送数据解码了。发完以后RK不是立即使用过这段数据的。但是发完后跟着就把AVPacket的数据内存给释放了。所以就导致在实际解码使用的时候数据已经被释放了相当于是丢帧了。
而这个回调函数就是为了给你释放内存的。所以示例代码中特地做了一次内存复制等rk用完在释放。 bool rk_vdec::send_stream(const char *buf, int size, int64_t pts /* 0*/)
{VDEC_STREAM_S stStream;MB_EXT_CONFIG_S pstMbExtConfig;MB_BLK buffer RK_NULL;memset(stStream, 0, sizeof(VDEC_STREAM_S));memset(pstMbExtConfig, 0, sizeof(MB_EXT_CONFIG_S));char *data_copy new char[size];memcpy(data_copy,buf,size);pstMbExtConfig.pFreeCB free_stream_Data;//printf(send frame 0x%x\n,data_copy);pstMbExtConfig.pOpaque (RK_U8 *)data_copy;pstMbExtConfig.pu8VirAddr (RK_U8 *)data_copy;pstMbExtConfig.u64Size size;RK_MPI_SYS_CreateMB(buffer, pstMbExtConfig);stStream.u64PTS pts;stStream.pMbBlk buffer;stStream.u32Len size;stStream.bEndOfStream RK_FALSE;stStream.bBypassMbBlk RK_TRUE;int ret RK_MPI_VDEC_SendStream(chn_, stStream, HI_IO_BLOCK);if (ret ! RK_SUCCESS){//std::cout RK_MPI_VDEC_SendStream failed : 0x std::hex ret std::endl;RK_MPI_MB_ReleaseMB(stStream.pMbBlk);return false;}else{//std::cout RK_MPI_VDEC_SendStream SUCCESS 1111111111111111111111111111111 std::endl;RK_MPI_MB_ReleaseMB(stStream.pMbBlk);}return true;
} 最后说一说和Qt融合的方法主要说Rockit的方案其他方案可以交给QImage渲染效率低一些但足够用了。
由于RK平台的Qt一般基于weston显示的也就是wayland平台。融合的方式在rockit方案里提到了很多网上也有很多说用RGARGN什么鬼的或者VO send_frame的方法甚至还有用colorKey视频层在上Qt层在下通过设置Qt背景颜色什么鬼的。
我只想说以上方法都是垃圾。
正经的方法思路如下Qt一般是在主层显示而rockit的方案会用到其他一个层。融合的思路就是让两个层叠加显示Qt不需要显示的地方设置透明度让rockit层透出来。
也就是说Qt主层在上zpos比rockit显示层大。
然后设置Qt主层的背景颜色支持透明度默认的weston.ini是不支持的需要额外设置。 [output] nameLVDS-1 mode1920x108060 gbm-formatargb8888
这个透明度必须设置否则Qt层不透明无法显示下层的视频信息。这样Qt中的透明区域可以任意放置其他控件这样就保证Qt画面叠加在视频层上面了。
这个想法很好但是实现起来废了一周时间掉了不少头发少撩骚了几次妹子。最终还是被我找到了方法。
因为视频层在Qt层上一开始也是遮挡Qt的后来设置视频层的背景颜色格式是RGBA后就能看到Qt了所以反过来一定也可以。最终跟着驱动兄弟又是翻驱动代码又是怎么滴的。总之是改好了。
至此所以问题完美解决。 我以为完了但是后来发现一个首次启动视频画面只有几帧等Qt起来后就卡死了一样。
又把RK骂了一顿。花了一天时间。证明是weston启动时一定会打断rockit的渲染通道。 问了RK最终得知加个这个环境变量就i行了export rt_vo_disable_vop0 由此可见rock的官方埋了多少地雷 最后我把rockit layer和设备的初始化示例代码放出来啦有问题可以评论回复 void rk_video_player::init_rk_system()
{std::cout start RK_MPI_SYS_Init std::endl;RK_U32 s32Ret RK_MPI_SYS_Init();if (s32Ret ! RK_SUCCESS){std::cout RK_MPI_SYS_Init failed std::endl;}else{std::cout RK_MPI_SYS_Init SUCCESS std::endl;}init_device(0);init_device_layer_camera(layer_, 0, 32, 896, 552);set_qt_layer_priority();// init_device_layer_playback(layer2_, 200, 52, 700, 400);
}
bool rk_video_player::init_device(int dev)
{// enable_device(false);std::cout rk_vo::init_device dev std::endl;RK_S32 s32Ret;device_ dev;VO_LAYER VoLayer layer_;VO_PUB_ATTR_S VoPubAttr;VO_DEV VoDev dev;memset(VoPubAttr, 0, sizeof(VO_PUB_ATTR_S));std::cout RK_MPI_VO_BindLayer :layer_ dev layer_ layer_ std::endl;s32Ret RK_MPI_VO_BindLayer(VoLayer, VoDev, RK356X_LAYER_MODE);if (s32Ret ! RK_SUCCESS){std::cout RK_MPI_VO_BindLayer layer_ failed,s32Ret: std::hex s32Ret std::endl;return RK_FAILURE;}std::cout RK_MPI_VO_SetPubAttr::open VoPubAttr.enIntfType VoPubAttr.enIntfType VoPubAttr.enIntfSync VoPubAttr.enIntfSync std::endl;if (VoPubAttr.enIntfType ! VO_INTF_LVDS VoPubAttr.enIntfSync ! VO_OUTPUT_1024x768_60){std::cout RK_MPI_VO_SetPubAttr VO_INTF_LVDS dev std::hex s32Ret std::dec std::endl;VoPubAttr.enIntfType VO_INTF_LVDS; // VO_INTF_LVDS;VoPubAttr.enIntfSync VO_OUTPUT_1024x768_60; // VO_OUTPUT_1024x768_60;VoPubAttr.u32BgColor 0x00000000;s32Ret RK_MPI_VO_SetPubAttr(device_, VoPubAttr);if (s32Ret ! RK_SUCCESS){std::cout RK_MPI_VO_SetPubAttr false dev std::hex s32Ret std::dec std::endl;return false;}std::cout RK_MPI_VO_SetPubAttr success dev std::hex s32Ret std::dec std::endl;}std::cout RK_MPI_VO_enable_device success ,then RK_MPI_VO_BindLayer;std::cout enable_device true::open dev dev layer_ layer_ std::endl;if (!enable_device(true)){std::cout enable_device failed,s32Ret std::endl;return false;}
}bool rk_video_player::init_device_layer_camera(int layer1, int top_x, int top_y, int width, int height)
{RK_S32 s32Ret;VO_LAYER VoLayer layer1;s32Ret RK_MPI_VO_SetLayerPriority(layer1, 0);s32Ret RK_MPI_VO_SetLayerDispBufLen(layer1, 6);std::cout RK_MPI_VO_SetLayerDispBufLen success std::endl;std::cout RK_MPI_VO_GetLayerAttr :open dev device_ layer1 layer1 std::endl;VO_VIDEO_LAYER_ATTR_S stLayerAttr;s32Ret RK_MPI_VO_GetLayerAttr(VoLayer, stLayerAttr);stLayerAttr.enCompressMode COMPRESS_MODE_NONE;stLayerAttr.enPixFormat RK_FMT_RGBA8888;stLayerAttr.u32DispFrmRt 25;stLayerAttr.stDispRect.s32X top_x;stLayerAttr.stDispRect.s32Y top_y;stLayerAttr.stDispRect.u32Width width;stLayerAttr.stDispRect.u32Height height;stLayerAttr.stImageSize.u32Width width;stLayerAttr.stImageSize.u32Height height;stLayerAttr.bBypassFrame RK_TRUE;stLayerAttr.bLowDelay RK_FALSE;std::cout RK_MPI_VO_SetLayerAttr :width std::dec width height height std::endl;std::cout RK_MPI_VO_SetLayerAttr :open dev device_ layer1 layer1 std::endl;std::cout RK_MPI_SYS_Init height: height std::endl;s32Ret RK_MPI_VO_SetLayerAttr(VoLayer, stLayerAttr);if (s32Ret ! RK_SUCCESS){std::cout RK_MPI_VO_SetLayerAttr failed,s32Ret: std::hex s32Ret;return false;}RK_MPI_VO_SetLayerSpliceMode(VoLayer, VO_SPLICE_MODE_GPU);// VO_SPLICE_MODE_RGAstd::cout RK_MPI_VO_EnableLayer :open dev device_ layer1 layer1 std::endl;s32Ret RK_MPI_VO_EnableLayer(VoLayer);if (s32Ret ! RK_SUCCESS){std::cout RK_MPI_VO_EnableLayer failed,s32Ret:0x std::hex s32Ret std::endl;return false;}VO_CSC_S VideoCSC;memset(VideoCSC, 0, sizeof(VO_CSC_S));VideoCSC.enCscMatrix VO_CSC_MATRIX_IDENTITY;VideoCSC.u32Contrast 50;VideoCSC.u32Hue 50;VideoCSC.u32Luma 50;VideoCSC.u32Satuature 50;s32Ret RK_MPI_VO_SetLayerCSC(layer_, VideoCSC);if (s32Ret ! RK_SUCCESS){std::cout RK_MPI_VO_SetLayerCSC failed std::hex s32Ret std::endl;return false;}std::cout RK_MPI_VO_EnableLayer SUCCESS std::endl;return true;
}题外话
实际上还有第五种方案就是用Qt 的Mediaplayer或者video playback吧里边url直接设置gstream 方式和插件流的方式使用qtvideosink直接渲染在Qt的QML里边。可惜rockit提供编译的Qt版本没有这个插件想弄的话要自己搞。这个可以参考jeston英伟达板卡的那些示例。
