bool VideoDecoderCUDA::decode(const QByteArray &encoded)
{
if (!isAvailable())
return false;
DPTR_D(VideoDecoderCUDA);
if (!d.parser) {
qWarning("CUVID parser not ready");
return false;
}
CUVIDSOURCEDATAPACKET cuvid_pkt;
memset(&cuvid_pkt, 0, sizeof(CUVIDSOURCEDATAPACKET));
cuvid_pkt.payload = (unsigned char *)encoded.data();
cuvid_pkt.payload_size = encoded.size();
cuvid_pkt.flags = CUVID_PKT_TIMESTAMP;
cuvid_pkt.timestamp = 0;// ?
//TODO: fill NALU header for h264? https://devtalk.nvidia.com/default/topic/515571/what-the-data-format-34-cuvidparsevideodata-34-can-accept-/
{
//cuvidCtxUnlock(d.vid_ctx_lock, 0); //TODO: why wrong context?
CUresult cuStatus = cuvidParseVideoData(d.parser, &cuvid_pkt);
if (cuStatus != CUDA_SUCCESS) {
qWarning("cuMemcpyDtoH failed (%p, %s)", cuStatus, _cudaGetErrorEnum(cuStatus));
}
}
// callbacks are in the same thread as this. so no queue is required?
qDebug("frame queue size on decode: %d", d.frame_queue.size());
return !d.frame_queue.isEmpty();
}
bool cv::gpu::detail::VideoParser::parseVideoData(const unsigned char* data, size_t size, bool endOfStream)
{
CUVIDSOURCEDATAPACKET packet;
std::memset(&packet, 0, sizeof(CUVIDSOURCEDATAPACKET));
if (endOfStream)
packet.flags |= CUVID_PKT_ENDOFSTREAM;
packet.payload_size = static_cast<unsigned long>(size);
packet.payload = data;
if (cuvidParseVideoData(parser_, &packet) != CUDA_SUCCESS)
{
hasError_ = true;
frameQueue_->endDecode();
return false;
}
const int maxUnparsedPackets = 15;
++unparsedPackets_;
if (unparsedPackets_ > maxUnparsedPackets)
{
hasError_ = true;
frameQueue_->endDecode();
return false;
}
if (endOfStream)
frameQueue_->endDecode();
return !frameQueue_->isEndOfDecode();
}
int
VideoSource::HandleVideoData(void * pUserData, CUVIDSOURCEDATAPACKET * pPacket)
{
VideoSourceData * pVideoSourceData = (VideoSourceData *)pUserData;
// Parser calls back for decode & display within cuvidParseVideoData
CUresult oResult = cuvidParseVideoData(pVideoSourceData->hVideoParser, pPacket);
if ((pPacket->flags & CUVID_PKT_ENDOFSTREAM) || (oResult != CUDA_SUCCESS))
pVideoSourceData->pFrameQueue->endDecode();
return !pVideoSourceData->pFrameQueue->isEndOfDecode();
}
static int cuvid_decode_packet(AVCodecContext *avctx, const AVPacket *avpkt)
{
CuvidContext *ctx = avctx->priv_data;
AVHWDeviceContext *device_ctx = (AVHWDeviceContext*)ctx->hwdevice->data;
AVCUDADeviceContext *device_hwctx = device_ctx->hwctx;
CUcontext dummy, cuda_ctx = device_hwctx->cuda_ctx;
CUVIDSOURCEDATAPACKET cupkt;
AVPacket filter_packet = { 0 };
AVPacket filtered_packet = { 0 };
int ret = 0, eret = 0, is_flush = ctx->decoder_flushing;
av_log(avctx, AV_LOG_TRACE, "cuvid_decode_packet\n");
if (is_flush && avpkt && avpkt->size)
return AVERROR_EOF;
if (av_fifo_size(ctx->frame_queue) / sizeof(CuvidParsedFrame) > MAX_FRAME_COUNT - 2 && avpkt && avpkt->size)
return AVERROR(EAGAIN);
if (ctx->bsf && avpkt && avpkt->size) {
if ((ret = av_packet_ref(&filter_packet, avpkt)) < 0) {
av_log(avctx, AV_LOG_ERROR, "av_packet_ref failed\n");
return ret;
}
if ((ret = av_bsf_send_packet(ctx->bsf, &filter_packet)) < 0) {
av_log(avctx, AV_LOG_ERROR, "av_bsf_send_packet failed\n");
av_packet_unref(&filter_packet);
return ret;
}
if ((ret = av_bsf_receive_packet(ctx->bsf, &filtered_packet)) < 0) {
av_log(avctx, AV_LOG_ERROR, "av_bsf_receive_packet failed\n");
return ret;
}
avpkt = &filtered_packet;
}
ret = CHECK_CU(cuCtxPushCurrent(cuda_ctx));
if (ret < 0) {
av_packet_unref(&filtered_packet);
return ret;
}
memset(&cupkt, 0, sizeof(cupkt));
if (avpkt && avpkt->size) {
cupkt.payload_size = avpkt->size;
cupkt.payload = avpkt->data;
if (avpkt->pts != AV_NOPTS_VALUE) {
cupkt.flags = CUVID_PKT_TIMESTAMP;
if (avctx->pkt_timebase.num && avctx->pkt_timebase.den)
cupkt.timestamp = av_rescale_q(avpkt->pts, avctx->pkt_timebase, (AVRational){1, 10000000});
else
cupkt.timestamp = avpkt->pts;
}
} else {
cupkt.flags = CUVID_PKT_ENDOFSTREAM;
ctx->decoder_flushing = 1;
}
ret = CHECK_CU(cuvidParseVideoData(ctx->cuparser, &cupkt));
av_packet_unref(&filtered_packet);
if (ret < 0)
goto error;
// cuvidParseVideoData doesn't return an error just because stuff failed...
if (ctx->internal_error) {
av_log(avctx, AV_LOG_ERROR, "cuvid decode callback error\n");
ret = ctx->internal_error;
goto error;
}
error:
eret = CHECK_CU(cuCtxPopCurrent(&dummy));
if (eret < 0)
return eret;
else if (ret < 0)
return ret;
else if (is_flush)
return AVERROR_EOF;
else
return 0;
}
void NVdecPlugin::on_shmreader_data(void* data, size_t size) {
if (!ds_)
ds_ = std::make_unique<ThreadedWrapper<NVencDS>>(
devices_nv_ids_[devices_.get_current_index()], video_codec_, get_log_ptr());
if (!ds_->invoke<MPtr(&NVencDS::safe_bool_idiom)>()) {
warning("nvcuvid failed to create a decoding session (nvdec).");
ds_.reset(nullptr);
return;
}
ds_.get()->invoke<MPtr(&NVencDS::parse_data)>([&](CUvideoparser parser) {
CUVIDSOURCEDATAPACKET packet;
packet.payload_size = size;
packet.payload = (const unsigned char*)data;
cuvidParseVideoData(parser, &packet);
});
ds_.get()->invoke<MPtr(&NVencDS::process_decoded)>([&](const unsigned char* data_decoded,
const unsigned int& data_width,
const unsigned int& data_height,
const unsigned int& pitch) {
if (!pitch) return;
guint height = data_height * 3 / 2; // Only NV12 support for now, 12bpp.
{
if (!shmwriter_) {
size_t start_width = caps_.find("width=") + 6;
size_t end_width = caps_.find_first_of(',', start_width);
size_t start_height = caps_.find("height=") + 7;
size_t end_height = caps_.find_first_of(',', start_height);
caps_.replace(start_width, end_width - start_width, std::to_string(data_width));
caps_.replace(start_height, end_height - start_height, std::to_string(data_height));
writer_size_ = data_width * data_height * 3 / 2;
shmwriter_ =
std::make_unique<ShmdataWriter>(this, make_shmpath("video"), writer_size_, caps_);
}
std::unique_ptr<shmdata::OneWriteAccess> shm_ptr;
if (writer_size_ > shmwriter_->writer<MPtr(&shmdata::Writer::alloc_size)>())
shm_ptr =
shmwriter_->writer<MPtr(&shmdata::Writer::get_one_write_access_resize)>(writer_size_);
else
shm_ptr = shmwriter_->writer<MPtr(&shmdata::Writer::get_one_write_access)>();
shm_ptr->notify_clients(writer_size_);
std::cout << "data_width: " << data_width << "data_height: " << data_height
<< "height: " << height << std::endl;
auto dest = reinterpret_cast<guint8*>(shm_ptr->get_mem());
for (guint y = 0; y < height; ++y) {
memcpy(dest, data_decoded, data_width);
dest += data_width;
data_decoded += pitch;
}
}
shmwriter_->bytes_written(writer_size_);
});
}
static int cuvid_decode_frame(AVCodecContext *avctx, void *data, int *got_frame, AVPacket *avpkt)
{
CuvidContext *ctx = avctx->priv_data;
AVHWDeviceContext *device_ctx = (AVHWDeviceContext*)ctx->hwdevice->data;
AVCUDADeviceContext *device_hwctx = device_ctx->hwctx;
CUcontext dummy, cuda_ctx = device_hwctx->cuda_ctx;
AVFrame *frame = data;
CUVIDSOURCEDATAPACKET cupkt;
AVPacket filter_packet = { 0 };
AVPacket filtered_packet = { 0 };
CUdeviceptr mapped_frame = 0;
int ret = 0, eret = 0;
if (ctx->bsf && avpkt->size) {
if ((ret = av_packet_ref(&filter_packet, avpkt)) < 0) {
av_log(avctx, AV_LOG_ERROR, "av_packet_ref failed\n");
return ret;
}
if ((ret = av_bsf_send_packet(ctx->bsf, &filter_packet)) < 0) {
av_log(avctx, AV_LOG_ERROR, "av_bsf_send_packet failed\n");
av_packet_unref(&filter_packet);
return ret;
}
if ((ret = av_bsf_receive_packet(ctx->bsf, &filtered_packet)) < 0) {
av_log(avctx, AV_LOG_ERROR, "av_bsf_receive_packet failed\n");
return ret;
}
avpkt = &filtered_packet;
}
ret = CHECK_CU(cuCtxPushCurrent(cuda_ctx));
if (ret < 0) {
av_packet_unref(&filtered_packet);
return ret;
}
memset(&cupkt, 0, sizeof(cupkt));
if (avpkt->size) {
cupkt.payload_size = avpkt->size;
cupkt.payload = avpkt->data;
if (avpkt->pts != AV_NOPTS_VALUE) {
cupkt.flags = CUVID_PKT_TIMESTAMP;
if (avctx->pkt_timebase.num && avctx->pkt_timebase.den)
cupkt.timestamp = av_rescale_q(avpkt->pts, avctx->pkt_timebase, (AVRational){1, 10000000});
else
cupkt.timestamp = avpkt->pts;
}
} else {
cupkt.flags = CUVID_PKT_ENDOFSTREAM;
}
ret = CHECK_CU(cuvidParseVideoData(ctx->cuparser, &cupkt));
av_packet_unref(&filtered_packet);
if (ret < 0) {
goto error;
}
// cuvidParseVideoData doesn't return an error just because stuff failed...
if (ctx->internal_error) {
av_log(avctx, AV_LOG_ERROR, "cuvid decode callback error\n");
ret = ctx->internal_error;
goto error;
}
if (av_fifo_size(ctx->frame_queue)) {
CUVIDPARSERDISPINFO dispinfo;
CUVIDPROCPARAMS params;
unsigned int pitch = 0;
int offset = 0;
int i;
av_fifo_generic_read(ctx->frame_queue, &dispinfo, sizeof(CUVIDPARSERDISPINFO), NULL);
memset(¶ms, 0, sizeof(params));
params.progressive_frame = dispinfo.progressive_frame;
params.second_field = 0;
params.top_field_first = dispinfo.top_field_first;
ret = CHECK_CU(cuvidMapVideoFrame(ctx->cudecoder, dispinfo.picture_index, &mapped_frame, &pitch, ¶ms));
if (ret < 0)
goto error;
if (avctx->pix_fmt == AV_PIX_FMT_CUDA) {
ret = av_hwframe_get_buffer(ctx->hwframe, frame, 0);
if (ret < 0) {
av_log(avctx, AV_LOG_ERROR, "av_hwframe_get_buffer failed\n");
goto error;
}
ret = ff_decode_frame_props(avctx, frame);
if (ret < 0) {
av_log(avctx, AV_LOG_ERROR, "ff_decode_frame_props failed\n");
goto error;
}
for (i = 0; i < 2; i++) {
CUDA_MEMCPY2D cpy = {
.srcMemoryType = CU_MEMORYTYPE_DEVICE,
.dstMemoryType = CU_MEMORYTYPE_DEVICE,
.srcDevice = mapped_frame,
.dstDevice = (CUdeviceptr)frame->data[i],
.srcPitch = pitch,
.dstPitch = frame->linesize[i],
.srcY = offset,
.WidthInBytes = FFMIN(pitch, frame->linesize[i]),
.Height = avctx->coded_height >> (i ? 1 : 0),
};
ret = CHECK_CU(cuMemcpy2D(&cpy));
if (ret < 0)
goto error;
offset += avctx->coded_height;
}
} else if (avctx->pix_fmt == AV_PIX_FMT_NV12) {