size_t IntelH264Decoder::Reset(const void* buf, size_t bufSize)
//void IntelH264Decoder::Reset()
{
mfxVideoParam par = { 0 };
par.mfx.CodecId = MFX_CODEC_AVC;
mfxBitstream bs = { 0 };
bs.Data = (uint8_t*) buf;
bs.DataOffset = 0;
bs.DataLength = bufSize;
bs.MaxLength = bufSize;
bs.DecodeTimeStamp = MFX_TIMESTAMP_UNKNOWN;
bs.TimeStamp = MFX_TIMESTAMP_UNKNOWN;
//MakeBitstream(Buf, bs);
//bs.Data = (uint8_t*) tpair;
//bs.DataOffset = 0;
//bs.DataLength = sizeof(tpair);
//bs.MaxLength = sizeof(tpair);
mfxStatus err = MFXVideoDECODE_DecodeHeader(Session, &bs, &par);
if (err == MFX_ERR_NONE)
{
par.IOPattern = MFX_IOPATTERN_IN_SYSTEM_MEMORY | MFX_IOPATTERN_OUT_SYSTEM_MEMORY;
err = MFXVideoDECODE_Init(Session, &par);
if (err == MFX_ERR_NONE)
{
VideoParam = par;
CreateWorkSurface(WorkSurface);
IsInitialized = true;
}
else
{
Log()->Error("MFXVideoDECODE_Init failed: %d", err);
}
}
return bs.DataOffset;
}
int ff_qsv_decode_init(AVCodecContext *avctx, QSVContext *q, AVPacket *avpkt)
{
mfxVideoParam param = { { 0 } };
mfxBitstream bs = { { { 0 } } };
int ret;
enum AVPixelFormat pix_fmts[3] = { AV_PIX_FMT_QSV,
AV_PIX_FMT_NV12,
AV_PIX_FMT_NONE };
q->iopattern = MFX_IOPATTERN_OUT_SYSTEM_MEMORY;
if (!q->session) {
if (avctx->hwaccel_context) {
AVQSVContext *qsv = avctx->hwaccel_context;
q->session = qsv->session;
q->iopattern = qsv->iopattern;
q->ext_buffers = qsv->ext_buffers;
q->nb_ext_buffers = qsv->nb_ext_buffers;
}
if (!q->session) {
ret = ff_qsv_init_internal_session(avctx, &q->internal_qs,
q->load_plugins);
if (ret < 0)
return ret;
q->session = q->internal_qs.session;
}
}
if (avpkt->size) {
bs.Data = avpkt->data;
bs.DataLength = avpkt->size;
bs.MaxLength = bs.DataLength;
bs.TimeStamp = avpkt->pts;
} else
return AVERROR_INVALIDDATA;
ret = ff_qsv_codec_id_to_mfx(avctx->codec_id);
if (ret < 0) {
av_log(avctx, AV_LOG_ERROR, "Unsupported codec_id %08x\n", avctx->codec_id);
return ret;
}
param.mfx.CodecId = ret;
ret = MFXVideoDECODE_DecodeHeader(q->session, &bs, ¶m);
if (MFX_ERR_MORE_DATA==ret) {
/* this code means that header not found so we return packet size to skip
a current packet
*/
return avpkt->size;
} else if (ret < 0) {
av_log(avctx, AV_LOG_ERROR, "Decode header error %d\n", ret);
return ff_qsv_error(ret);
}
param.IOPattern = q->iopattern;
param.AsyncDepth = q->async_depth;
param.ExtParam = q->ext_buffers;
param.NumExtParam = q->nb_ext_buffers;
param.mfx.FrameInfo.BitDepthLuma = 8;
param.mfx.FrameInfo.BitDepthChroma = 8;
ret = MFXVideoDECODE_Init(q->session, ¶m);
if (ret < 0) {
if (MFX_ERR_INVALID_VIDEO_PARAM==ret) {
av_log(avctx, AV_LOG_ERROR,
"Error initializing the MFX video decoder, unsupported video\n");
} else {
av_log(avctx, AV_LOG_ERROR,
"Error initializing the MFX video decoder %d\n", ret);
}
return ff_qsv_error(ret);
}
ret = ff_get_format(avctx, pix_fmts);
if (ret < 0)
return ret;
avctx->pix_fmt = ret;
avctx->profile = param.mfx.CodecProfile;
avctx->level = param.mfx.CodecLevel;
avctx->coded_width = param.mfx.FrameInfo.Width;
avctx->coded_height = param.mfx.FrameInfo.Height;
avctx->width = param.mfx.FrameInfo.CropW - param.mfx.FrameInfo.CropX;
avctx->height = param.mfx.FrameInfo.CropH - param.mfx.FrameInfo.CropY;
/* maximum decoder latency should be not exceed max DPB size for h.264 and
HEVC which is 16 for both cases.
So weare pre-allocating fifo big enough for 17 elements:
*/
if (!q->async_fifo) {
q->async_fifo = av_fifo_alloc((1 + 16) *
(sizeof(mfxSyncPoint) + sizeof(QSVFrame*)));
if (!q->async_fifo)
return AVERROR(ENOMEM);
}
q->input_fifo = av_fifo_alloc(1024*16);
if (!q->input_fifo)
return AVERROR(ENOMEM);
q->engine_ready = 1;
return 0;
}
STDMETHODIMP CDecMSDKMVC::Decode(const BYTE *buffer, int buflen, REFERENCE_TIME rtStart, REFERENCE_TIME rtStop, BOOL bSyncPoint, BOOL bDiscontinuity)
{
if (!m_mfxSession)
return E_UNEXPECTED;
HRESULT hr = S_OK;
CBitstreamBuffer bsBuffer(&m_buff);
mfxStatus sts = MFX_ERR_NONE;
mfxBitstream bs = { 0 };
BOOL bFlush = (buffer == nullptr);
if (rtStart >= -TIMESTAMP_OFFSET && rtStart != AV_NOPTS_VALUE)
bs.TimeStamp = rtStart + TIMESTAMP_OFFSET;
else
bs.TimeStamp = MFX_TIMESTAMP_UNKNOWN;
bs.DecodeTimeStamp = MFX_TIMESTAMP_UNKNOWN;
if (!bFlush) {
if (m_pAnnexBConverter) {
BYTE *pOutBuffer = nullptr;
int pOutSize = 0;
hr = m_pAnnexBConverter->Convert(&pOutBuffer, &pOutSize, buffer, buflen);
if (FAILED(hr))
return hr;
bsBuffer.SetBuffer(pOutBuffer, pOutSize, true);
}
else {
bsBuffer.SetBuffer((BYTE *)buffer, buflen, false);
}
// Check the buffer for SEI NALU, and some unwanted NALUs that need filtering
// MSDK's SEI reading functionality is slightly buggy
CH264Nalu nalu;
nalu.SetBuffer(bsBuffer.GetBuffer(), bsBuffer.GetBufferSize(), 0);
BOOL bNeedFilter = FALSE;
while (nalu.ReadNext()) {
if (nalu.GetType() == NALU_TYPE_SEI) {
ParseSEI(nalu.GetDataBuffer() + 1, nalu.GetDataLength() - 1, bs.TimeStamp);
}
else if (nalu.GetType() == NALU_TYPE_EOSEQ) {
bsBuffer.EnsureWriteable();
// This is rather ugly, and relies on the bitstream being AnnexB, so simply overwriting the EOS NAL with zero works.
// In the future a more elaborate bitstream filter might be advised
memset(bsBuffer.GetBuffer() + nalu.GetNALPos(), 0, 4);
}
}
bs.Data = bsBuffer.GetBuffer();
bs.DataLength = mfxU32(bsBuffer.GetBufferSize());
bs.MaxLength = bs.DataLength;
AddFrameToGOP(bs.TimeStamp);
}
if (!m_bDecodeReady) {
sts = MFXVideoDECODE_DecodeHeader(m_mfxSession, &bs, &m_mfxVideoParams);
if (sts == MFX_ERR_NOT_ENOUGH_BUFFER) {
hr = AllocateMVCExtBuffers();
if (FAILED(hr))
return hr;
sts = MFXVideoDECODE_DecodeHeader(m_mfxSession, &bs, &m_mfxVideoParams);
}
if (sts == MFX_ERR_NONE) {
m_mfxVideoParams.IOPattern = MFX_IOPATTERN_OUT_SYSTEM_MEMORY;
m_mfxVideoParams.AsyncDepth = ASYNC_DEPTH;
sts = MFXVideoDECODE_Init(m_mfxSession, &m_mfxVideoParams);
if (sts != MFX_ERR_NONE) {
DbgLog((LOG_TRACE, 10, L"CDevMSDKMVC::Decode(): Error initializing the MSDK decoder (%d)", sts));
return E_FAIL;
}
if (m_mfxExtMVCSeq.NumView != 2) {
DbgLog((LOG_TRACE, 10, L"CDevMSDKMVC::Decode(): Only MVC with two views is supported"));
return E_FAIL;
}
DbgLog((LOG_TRACE, 10, L"CDevMSDKMVC::Decode(): Initialized MVC with View Ids %d, %d", m_mfxExtMVCSeq.View[0].ViewId, m_mfxExtMVCSeq.View[1].ViewId));
m_bDecodeReady = TRUE;
}
}
if (!m_bDecodeReady)
return S_FALSE;
mfxSyncPoint sync = nullptr;
// Loop over the decoder to ensure all data is being consumed
while (1) {
MVCBuffer *pInputBuffer = GetBuffer();
if (pInputBuffer == nullptr)
return E_OUTOFMEMORY;
mfxFrameSurface1 *outsurf = nullptr;
sts = MFXVideoDECODE_DecodeFrameAsync(m_mfxSession, bFlush ? nullptr : &bs, &pInputBuffer->surface, &outsurf, &sync);
if (sts == MFX_ERR_INCOMPATIBLE_VIDEO_PARAM) {
DbgLog((LOG_TRACE, 10, L"CDevMSDKMVC::Decode(): Incompatible video parameters detected, flushing decoder"));
bsBuffer.Clear();
bFlush = TRUE;
m_bDecodeReady = FALSE;
continue;
}
if (sync) {
MVCBuffer * pOutputBuffer = FindBuffer(outsurf);
pOutputBuffer->queued = 1;
pOutputBuffer->sync = sync;
HandleOutput(pOutputBuffer);
continue;
}
if (sts != MFX_ERR_MORE_SURFACE && sts < 0)
break;
}
if (!bs.DataOffset && !sync && !bFlush) {
DbgLog((LOG_TRACE, 10, L"CDevMSDKMVC::Decode(): Decoder did not consume any data, discarding"));
bs.DataOffset = mfxU32(bsBuffer.GetBufferSize());
}
bsBuffer.Consume(bs.DataOffset);
if (sts != MFX_ERR_MORE_DATA && sts < 0) {
DbgLog((LOG_TRACE, 10, L"CDevMSDKMVC::Decode(): Error from Decode call (%d)", sts));
return S_FALSE;
}
return S_OK;
}
int ff_qsv_decode_init(AVCodecContext *avctx, QSVContext *q, AVPacket *avpkt)
{
mfxVideoParam param = { { 0 } };
mfxBitstream bs = { { { 0 } } };
int ret;
q->iopattern = MFX_IOPATTERN_OUT_SYSTEM_MEMORY;
if (!q->session) {
if (avctx->hwaccel_context) {
AVQSVContext *qsv = avctx->hwaccel_context;
q->session = qsv->session;
q->iopattern = qsv->iopattern;
q->ext_buffers = qsv->ext_buffers;
q->nb_ext_buffers = qsv->nb_ext_buffers;
}
if (!q->session) {
ret = ff_qsv_init_internal_session(avctx, &q->internal_qs, NULL);
if (ret < 0)
return ret;
q->session = q->internal_qs.session;
}
}
if (avpkt->size) {
bs.Data = avpkt->data;
bs.DataLength = avpkt->size;
bs.MaxLength = bs.DataLength;
bs.TimeStamp = avpkt->pts;
} else
return AVERROR_INVALIDDATA;
ret = ff_qsv_codec_id_to_mfx(avctx->codec_id);
if (ret < 0) {
av_log(avctx, AV_LOG_ERROR, "Unsupported codec_id %08x\n", avctx->codec_id);
return ret;
}
param.mfx.CodecId = ret;
ret = MFXVideoDECODE_DecodeHeader(q->session, &bs, ¶m);
if (MFX_ERR_MORE_DATA==ret) {
return AVERROR(EAGAIN);
} else if (ret < 0) {
av_log(avctx, AV_LOG_ERROR, "Decode header error %d\n", ret);
return ff_qsv_error(ret);
}
param.IOPattern = q->iopattern;
param.AsyncDepth = q->async_depth;
param.ExtParam = q->ext_buffers;
param.NumExtParam = q->nb_ext_buffers;
param.mfx.FrameInfo.BitDepthLuma = 8;
param.mfx.FrameInfo.BitDepthChroma = 8;
ret = MFXVideoDECODE_Init(q->session, ¶m);
if (ret < 0) {
av_log(avctx, AV_LOG_ERROR, "Error initializing the MFX video decoder\n");
return ff_qsv_error(ret);
}
avctx->pix_fmt = AV_PIX_FMT_NV12;
avctx->profile = param.mfx.CodecProfile;
avctx->level = param.mfx.CodecLevel;
avctx->coded_width = param.mfx.FrameInfo.Width;
avctx->coded_height = param.mfx.FrameInfo.Height;
avctx->width = param.mfx.FrameInfo.CropW - param.mfx.FrameInfo.CropX;
avctx->height = param.mfx.FrameInfo.CropH - param.mfx.FrameInfo.CropY;
q->async_fifo = av_fifo_alloc((1 + q->async_depth) *
(sizeof(mfxSyncPoint) + sizeof(QSVFrame*)));
if (!q->async_fifo)
return AVERROR(ENOMEM);
return 0;
}
static GstFlowReturn
gst_msdkdec_handle_frame (GstVideoDecoder * decoder, GstVideoCodecFrame * frame)
{
GstMsdkDec *thiz = GST_MSDKDEC (decoder);
GstMsdkDecClass *klass = GST_MSDKDEC_GET_CLASS (thiz);
GstFlowReturn flow;
GstBuffer *buffer, *input_buffer = NULL;
GstVideoInfo alloc_info;
MsdkDecTask *task = NULL;
mfxBitstream bitstream;
MsdkSurface *surface = NULL;
mfxSession session;
mfxStatus status;
GstMapInfo map_info;
guint i;
gsize data_size;
gboolean hard_reset = FALSE;
/* configure the subclass in order to fill the CodecID field of
* mfxVideoParam and also to load the PluginID for some of the
* codecs which is mandatory to invoke the
* MFXVideoDECODE_DecodeHeader API.
*
* For non packetized formats (currently only vc1), there
* could be headers received as codec_data which are not available
* instream and in that case subclass implementation will
* push it to the internal adapter. We invoke the subclass configure
* well early to make sure the codec_data received has been correctly
* pushed to the adapter by the subclasses before doing
* the DecodeHeader() later on
*/
if (!thiz->initialized || thiz->do_renego) {
/* Clear the internal adapter in renegotiation for non-packetized
* formats */
if (!thiz->is_packetized)
gst_adapter_clear (thiz->adapter);
if (!klass->configure || !klass->configure (thiz)) {
flow = GST_FLOW_OK;
goto error;
}
}
/* Current frame-codec could be pushed and released before this
* function ends -- because msdkdec pushes the oldest frame,
* according its PTS, and it could be this very same frame-codec
* among others pending frame-codecs.
*
* Instead of copying the input data into the mfxBitstream, let's
* keep an extra reference to frame-codec's input buffer */
input_buffer = gst_buffer_ref (frame->input_buffer);
if (!gst_buffer_map (input_buffer, &map_info, GST_MAP_READ)) {
gst_buffer_unref (input_buffer);
return GST_FLOW_ERROR;
}
memset (&bitstream, 0, sizeof (bitstream));
if (thiz->is_packetized) {
/* Packetized stream: We prefer to have a parser as connected upstream
* element to the decoder */
bitstream.Data = map_info.data;
bitstream.DataLength = map_info.size;
bitstream.MaxLength = map_info.size;
bitstream.DataFlag = MFX_BITSTREAM_COMPLETE_FRAME;
} else {
/* Non packetized streams: eg: vc1 advanced profile with per buffer bdu */
gst_adapter_push (thiz->adapter, gst_buffer_ref (input_buffer));
data_size = gst_adapter_available (thiz->adapter);
bitstream.Data = (mfxU8 *) gst_adapter_map (thiz->adapter, data_size);
bitstream.DataLength = (mfxU32) data_size;
bitstream.MaxLength = bitstream.DataLength;
}
GST_INFO_OBJECT (thiz,
"mfxBitStream=> DataLength:%d DataOffset:%d MaxLength:%d",
bitstream.DataLength, bitstream.DataOffset, bitstream.MaxLength);
session = gst_msdk_context_get_session (thiz->context);
if (!thiz->initialized || thiz->do_renego) {
/* gstreamer caps will not bring all the necessary parameters
* required for optimal decode configuration. For eg: the required numbers
* of surfaces to be allocated can be calculated based on H264 SEI header
* and this information can't be retrieved from the negotiated caps.
* So instead of introducing the codecparser dependency to parse the headers
* inside msdk plugin, we simply use the mfx apis to extract header information */
status = MFXVideoDECODE_DecodeHeader (session, &bitstream, &thiz->param);
if (status == MFX_ERR_MORE_DATA) {
flow = GST_FLOW_OK;
goto done;
}
if (!thiz->initialized)
hard_reset = TRUE;
else if (thiz->allocation_caps) {
gst_video_info_from_caps (&alloc_info, thiz->allocation_caps);
/* Check whether we need complete reset for dynamic resolution change */
if (thiz->param.mfx.FrameInfo.Width > GST_VIDEO_INFO_WIDTH (&alloc_info)
|| thiz->param.mfx.FrameInfo.Height >
GST_VIDEO_INFO_HEIGHT (&alloc_info))
hard_reset = TRUE;
}
/* if subclass requested for the force reset */
if (thiz->force_reset_on_res_change)
hard_reset = TRUE;
/* Config changed dynamically and we are going to do a full reset,
* this will unref the input frame which has the new configuration.
* Keep a ref to the input_frame to keep it alive */
if (thiz->initialized && thiz->do_renego)
gst_video_codec_frame_ref (frame);
if (!gst_msdkdec_negotiate (thiz, hard_reset)) {
GST_ELEMENT_ERROR (thiz, CORE, NEGOTIATION,
("Could not negotiate the stream"), (NULL));
flow = GST_FLOW_ERROR;
goto error;
}
}
for (;;) {
task = &g_array_index (thiz->tasks, MsdkDecTask, thiz->next_task);
flow = gst_msdkdec_finish_task (thiz, task);
if (flow != GST_FLOW_OK)
goto error;
if (!surface) {
flow = allocate_output_buffer (thiz, &buffer);
if (flow != GST_FLOW_OK)
goto error;
surface = get_surface (thiz, buffer);
if (!surface) {
/* Can't get a surface for some reason, finish tasks to see if
a surface becomes available. */
for (i = 0; i < thiz->tasks->len - 1; i++) {
thiz->next_task = (thiz->next_task + 1) % thiz->tasks->len;
task = &g_array_index (thiz->tasks, MsdkDecTask, thiz->next_task);
flow = gst_msdkdec_finish_task (thiz, task);
if (flow != GST_FLOW_OK)
goto error;
surface = get_surface (thiz, buffer);
if (surface)
break;
}
if (!surface) {
GST_ERROR_OBJECT (thiz, "Couldn't get a surface");
flow = GST_FLOW_ERROR;
goto error;
}
}
}
status =
MFXVideoDECODE_DecodeFrameAsync (session, &bitstream, surface->surface,
&task->surface, &task->sync_point);
/* media-sdk requires complete reset since the surface is inadaquate to
* do further decoding */
if (status == MFX_ERR_INCOMPATIBLE_VIDEO_PARAM) {
/* Requires memory re-allocation, do a hard reset */
if (!gst_msdkdec_negotiate (thiz, TRUE))
goto error;
status =
MFXVideoDECODE_DecodeFrameAsync (session, &bitstream,
surface->surface, &task->surface, &task->sync_point);
}
if (G_LIKELY (status == MFX_ERR_NONE)
|| (status == MFX_WRN_VIDEO_PARAM_CHANGED)) {
thiz->next_task = (thiz->next_task + 1) % thiz->tasks->len;
if (surface->surface->Data.Locked > 0 || !thiz->use_video_memory)
surface = NULL;
if (bitstream.DataLength == 0) {
flow = GST_FLOW_OK;
break;
}
} else if (status == MFX_ERR_MORE_DATA) {
if (task->surface) {
task->decode_only = TRUE;
thiz->next_task = (thiz->next_task + 1) % thiz->tasks->len;
}
if (surface->surface->Data.Locked > 0)
surface = NULL;
flow = GST_VIDEO_DECODER_FLOW_NEED_DATA;
break;
} else if (status == MFX_ERR_MORE_SURFACE) {
surface = NULL;
continue;
} else if (status == MFX_WRN_DEVICE_BUSY) {
/* If device is busy, wait 1ms and retry, as per MSDK's recomendation */
g_usleep (1000);
/* If the current surface is still busy, we should do sync oepration
* then tries to decode again
*/
thiz->next_task = (thiz->next_task + 1) % thiz->tasks->len;
} else if (status < MFX_ERR_NONE) {
GST_ERROR_OBJECT (thiz, "DecodeFrameAsync failed (%s)",
msdk_status_to_string (status));
flow = GST_FLOW_ERROR;
break;
}
}
if (!thiz->is_packetized) {
/* flush out the data which is already consumed by msdk */
gst_adapter_flush (thiz->adapter, bitstream.DataOffset);
flow = GST_FLOW_OK;
}
done:
if (surface)
free_surface (thiz, surface);
gst_buffer_unmap (input_buffer, &map_info);
gst_buffer_unref (input_buffer);
return flow;
error:
if (input_buffer) {
gst_buffer_unmap (input_buffer, &map_info);
gst_buffer_unref (input_buffer);
}
gst_video_decoder_drop_frame (decoder, frame);
return flow;
}
