static pj_status_t check_decode_result(pjmedia_vid_codec *codec,
const vpx_image_t *img,
const pj_timestamp *ts) {
vpx_private *vpx = (vpx_private*) codec->codec_data;
pjmedia_video_apply_fmt_param *vafp = &vpx->dec_vafp;
pjmedia_event event;
int res, reference_updates = 0;
/* Check for format change.
*/
if (img->d_w != (int) vafp->size.w || img->d_h != (int) vafp->size.h) {
pj_status_t status;
/* Update decoder format in param */
vpx->param.dec_fmt.det.vid.size.w = img->d_w;
vpx->param.dec_fmt.det.vid.size.h = img->d_h;
/* Re-init format info and apply-param of decoder */
vpx->dec_vfi = pjmedia_get_video_format_info(NULL, vpx->param.dec_fmt.id);
if (!vpx->dec_vfi)
return PJ_ENOTSUP;
pj_bzero(&vpx->dec_vafp, sizeof(vpx->dec_vafp));
vpx->dec_vafp.size = vpx->param.dec_fmt.det.vid.size;
vpx->dec_vafp.buffer = NULL;
status = (*vpx->dec_vfi->apply_fmt)(vpx->dec_vfi, &vpx->dec_vafp);
if (status != PJ_SUCCESS)
return status;
/* Realloc buffer if necessary */
if (vpx->dec_vafp.framebytes > vpx->dec_buf_size) {
PJ_LOG(5,(THIS_FILE, "Reallocating decoding buffer %u --> %u",
(unsigned)vpx->dec_buf_size,
(unsigned)vpx->dec_vafp.framebytes));
vpx->dec_buf_size = (unsigned)vpx->dec_vafp.framebytes;
vpx->dec_buf = pj_pool_alloc(vpx->pool, vpx->dec_buf_size);
}
/* Broadcast format changed event */
pjmedia_event_init(&event, PJMEDIA_EVENT_FMT_CHANGED, ts, codec);
event.data.fmt_changed.dir = PJMEDIA_DIR_DECODING;
pj_memcpy(&event.data.fmt_changed.new_fmt, &vpx->param.dec_fmt, sizeof(vpx->param.dec_fmt));
pjmedia_event_publish(NULL, codec, &event, 0);
}
/* Check for found keyframe */
res = vpx_codec_control(&vpx->decoder, VP8D_GET_LAST_REF_UPDATES, &reference_updates);
if (res == VPX_CODEC_OK) {
pj_bool_t got_keyframe = (reference_updates & VP8_GOLD_FRAME);
if (got_keyframe) {
pj_get_timestamp(&vpx->last_dec_keyframe_ts);
/* Broadcast keyframe event */
pjmedia_event_init(&event, PJMEDIA_EVENT_KEYFRAME_FOUND, ts, codec);
pjmedia_event_publish(NULL, codec, &event, 0);
}
}
return PJ_SUCCESS;
}
static pj_status_t pj_vpx_codec_decode(pjmedia_vid_codec *codec,
pj_size_t pkt_count,
pjmedia_frame packets[],
unsigned out_size,
pjmedia_frame *output) {
vpx_private *vpx = (vpx_private*) codec->codec_data;
vpx_image_t *img;
vpx_codec_iter_t iter;
int i, res;
PJ_ASSERT_RETURN(codec && pkt_count > 0 && packets && output, PJ_EINVAL);
vpx->dec_frame_len = 0;
/* TODO : packet parsing is absolutely incomplete here !!!!
* We should manage extensions, partitions etc
* */
for (i = 0; i < pkt_count; ++i) {
pj_uint8_t *data;
pj_uint8_t extended_bit, s_bit, partition_id;
unsigned extension_len = 0;
unsigned payload_size = packets[i].size;
if(payload_size == 0) {
continue;
}
data = packets[i].buf;
extended_bit = (*data) & 0x80;
s_bit = (*data) & 0x20;
partition_id = (*data) & 0x1F;
PJ_UNUSED_ARG(s_bit);
PJ_UNUSED_ARG(partition_id);
/* First octet is for */
/* |X|R|N|S|PartID | */
if(extended_bit) {
pj_uint8_t i_bit, l_bit, t_bit, k_bit;
(data)++;
extension_len++;
i_bit = (*data) & 0x80;
l_bit = (*data) & 0x40;
t_bit = (*data) & 0x20;
k_bit = (*data) & 0x10;
if(payload_size <= 1) {
PJ_LOG(4, (THIS_FILE, "Error decoding VP8 extended attributes"));
continue;
}
/* We have extension in octet 2 */
/* |I|L|T|K| RSV | */
if (i_bit) {
data++;
if(payload_size <= 2){
PJ_LOG(4, (THIS_FILE, "Error decoding VP8 extended picture ID attribute"));
continue;
}
// I present check M flag for long picture ID
if ((*data) & 0x80) {
data++;
}
}
if (l_bit) {
data++;
}
if (t_bit || k_bit) {
data++;
}
}
data++;
payload_size = packets[i].size - (data - (pj_uint8_t*)packets[i].buf);
//PJ_LOG(4, (THIS_FILE, "Unpack RTP %d size %d, start %d", i, packets[i].size, s[0] & 0x10));
if((vpx->dec_frame_len + payload_size) < vpx->dec_buf_size) {
pj_memcpy(vpx->dec_buf + vpx->dec_frame_len, data, payload_size);
vpx->dec_frame_len += payload_size;
} else {
PJ_LOG(1, (THIS_FILE, "Buffer is too small"));
}
}
if(vpx->dec_frame_len == 0){
PJ_LOG(1, (THIS_FILE, "No content for these packets"));
return PJ_SUCCESS;
}
res = vpx_codec_decode(&vpx->decoder, vpx->dec_buf, vpx->dec_frame_len, NULL, VPX_DL_REALTIME);
switch (res) {
case VPX_CODEC_UNSUP_BITSTREAM:
case VPX_CODEC_UNSUP_FEATURE:
case VPX_CODEC_CORRUPT_FRAME:
/* Fatal errors to the stream, request a keyframe to see if we can recover */
PJ_LOG(4, (THIS_FILE, "Fatal error decoding stream: (%d) %s", res, vpx_codec_err_to_string(res)));
pjmedia_event event;
pjmedia_event_init(&event, PJMEDIA_EVENT_KEYFRAME_MISSING, NULL, codec);
pjmedia_event_publish(NULL, codec, &event, 0);
return PJMEDIA_CODEC_EBADBITSTREAM;
case VPX_CODEC_OK:
break;
default:
PJ_LOG(4, (THIS_FILE, "Failed to decode packets: (%d) %s", res, vpx_codec_err_to_string(res)));
return PJMEDIA_ERROR;
}
iter = NULL;
for (;;) {
pj_status_t status;
img = vpx_codec_get_frame(&vpx->decoder, &iter);
if (img == NULL)
break;
status = pj_vpx_codec_decode_whole(codec, img, &packets[0].timestamp, out_size, output);
if (status != PJ_SUCCESS) {
PJ_LOG(4, (THIS_FILE, "Failed to decode frame"));
/* XXX stop processing and request keyframe? */
}
}
return PJ_SUCCESS;
}
static pj_status_t pj_vpx_codec_decode(pjmedia_vid_codec *codec,
pj_size_t pkt_count, pjmedia_frame packets[], unsigned out_size,
pjmedia_frame *output) {
vpx_private *vpx = (vpx_private*) codec->codec_data;
pj_status_t status;
pj_uint8_t *p;
pj_uint8_t *s;
vpx_image_t *img;
vpx_codec_iter_t iter = NULL;
int i, res;
PJ_ASSERT_RETURN(codec && pkt_count > 0 && packets && output, PJ_EINVAL);
i = 0;
output->type = PJMEDIA_FRAME_TYPE_NONE;
output->size = 0;
s = packets[0].buf;
p = vpx->dec_buf;
vpx->dec_frame_len = 0;
/* TODO : packet parsing is absolutely incomplete here !!!!
* We should manage extensions, partitions etc
* */
for (i = 0; i < pkt_count; ++i) {
unsigned extension_len = 0;
unsigned payload_size = packets[i].size;
if(payload_size <= 0){
continue;
}
s = packets[i].buf;
/* First octet is for */
/* |X|R|N|S|PartID | */
if( s[0] & 0x80 ){
if(payload_size <= 1){
continue;
}
/* We have extension in octet 2 */
/* |I|L|T|K| RSV | */
if( s[1] & 0x80 ){
extension_len++;
if(payload_size <= 2){
continue;
}
// I present check M flag for long picture ID
if( s[2] & 0x80 ) extension_len++;
}
if( s[1] & 0x40 ) extension_len++;
if( (s[1] & 0x20) | (s[1] & 0x10) ) extension_len++;
}
s += (extension_len + 1);
payload_size -= (extension_len + 1);
//PJ_LOG(4, (THIS_FILE, "Unpack RTP %d size %d, start %d", i, packets[i].size, s[0] & 0x10));
if((vpx->dec_frame_len + payload_size) < vpx->dec_buf_size) {
pj_memcpy((p + vpx->dec_frame_len), s, payload_size);
vpx->dec_frame_len += payload_size;
} else {
PJ_LOG(1, (THIS_FILE, "Buffer is too small"));
}
}
if(vpx->dec_frame_len == 0){
PJ_LOG(1, (THIS_FILE, "No content for these packets"));
return PJ_SUCCESS;
}
res = vpx_codec_decode(&vpx->decoder,
vpx->dec_buf, vpx->dec_frame_len,
0, VPX_DL_REALTIME);
if (res == VPX_CODEC_UNSUP_BITSTREAM){
PJ_LOG(2, (THIS_FILE, "More likely we are missing a keyframe, request it"));
/* Broadcast missing keyframe event */
pjmedia_event event;
pjmedia_event_init(&event, PJMEDIA_EVENT_KEYFRAME_MISSING,
&packets[0].timestamp,
codec);
pjmedia_event_publish(NULL, codec, &event, 0);
} else if (res != VPX_CODEC_OK) {
PJ_LOG(1,
(THIS_FILE, "Failed to decode packet : %s of size %d", vpx_codec_err_to_string(res), vpx->dec_frame_len));
/* TODO : better error code. Map with res? */
return PJ_SUCCESS;
}
/* No need to loop here for first implementation */
img = vpx_codec_get_frame(&vpx->decoder, &iter);
if (img != NULL) {
/* We have a frame available */
pj_vpx_codec_decode_whole(codec, img, &packets[0].timestamp, out_size, output);
}
return PJ_SUCCESS;
}
/*
* Decode frame.
*/
static pj_status_t openh264_codec_decode_whole(pjmedia_vid_codec *codec,
const pjmedia_frame *input,
unsigned output_buf_len,
pjmedia_frame *output)
{
openh264_private *ff = (openh264_private*)codec->codec_data;
void* pData[3] = {NULL};
SBufferInfo pDstInfo;
static pj_bool_t got_keyframe = PJ_FALSE;
DECODING_STATE decodingState;
/* Check if decoder has been opened */
// PJ_ASSERT_RETURN(ff->dec_ctx, PJ_EINVALIDOP);
PJ_ASSERT_RETURN(ff->dec, PJ_EINVALIDOP);
/* Reset output frame bit info */
output->bit_info = 0;
output->timestamp = input->timestamp;
pData[0] = NULL;
pData[1] = NULL;
pData[2] = NULL;
memset(&pDstInfo, 0x00, sizeof(SBufferInfo));
decodingState = callWelsDecoderFn(ff->dec)->DecodeFrame2(ff->dec, input->buf, input->size, pData, &pDstInfo);
if (pDstInfo.iBufferStatus == 1) {
pjmedia_video_apply_fmt_param *vafp = &ff->dec_vafp;
pj_uint8_t *q = (pj_uint8_t*)output->buf;
unsigned i;
pj_status_t status;
/* Check decoding result, e.g: see if the format got changed,
* keyframe found/missing.
*/
got_keyframe = PJ_TRUE;
status = check_new_decode_result(codec, &pDstInfo, &input->timestamp, got_keyframe);
if (status != PJ_SUCCESS)
return status;
/* Check provided buffer size */
if (vafp->framebytes > output_buf_len)
return PJ_ETOOSMALL;
/* Get the decoded data */
for (i = 0; i < ff->dec_vfi->plane_cnt; ++i) {
pj_uint8_t *p = pData[i];
/* The decoded data may contain padding */
if (pDstInfo.UsrData.sSystemBuffer.iStride[(i==0)?0:1]!=vafp->strides[i]) {
/* Padding exists, copy line by line */
pj_uint8_t *q_end;
q_end = q+vafp->plane_bytes[i];
while(q < q_end) {
pj_memcpy(q, p, vafp->strides[i]);
q += vafp->strides[i];
p += pDstInfo.UsrData.sSystemBuffer.iStride[(i==0)?0:1];
}
} else {
/* No padding, copy the whole plane */
pj_memcpy(q, p, vafp->plane_bytes[i]);
q += vafp->plane_bytes[i];
}
}
output->type = PJMEDIA_FRAME_TYPE_VIDEO;
output->size = vafp->framebytes;
} else {
output->type = PJMEDIA_FRAME_TYPE_NONE;
output->size = 0;
got_keyframe = PJ_FALSE;
if (decodingState == dsNoParamSets) {
pjmedia_event event;
/* Broadcast missing keyframe event */
pjmedia_event_init(&event, PJMEDIA_EVENT_KEYFRAME_MISSING,
&input->timestamp, codec);
pjmedia_event_publish(NULL, codec, &event, 0);
}
return PJMEDIA_CODEC_EBADBITSTREAM;
}
return PJ_SUCCESS;
}
static pj_status_t check_new_decode_result(pjmedia_vid_codec *codec,
SBufferInfo *bufferInfo,
const pj_timestamp *ts,
pj_bool_t got_keyframe)
{
openh264_private *ff = (openh264_private*)codec->codec_data;
pjmedia_video_apply_fmt_param *vafp = &ff->dec_vafp;
pjmedia_event event;
/* Check for format change.
* Decoder output format is set by libavcodec, in case it is different
* to the configured param.
*/
if (bufferInfo->UsrData.sSystemBuffer.iWidth != (int)vafp->size.w ||
bufferInfo->UsrData.sSystemBuffer.iHeight != (int)vafp->size.h)
{
pj_status_t status;
/* Update decoder format in param */
ff->param.dec_fmt.id = PJMEDIA_FORMAT_I420;
ff->param.dec_fmt.det.vid.size.w = bufferInfo->UsrData.sSystemBuffer.iWidth;
ff->param.dec_fmt.det.vid.size.h = bufferInfo->UsrData.sSystemBuffer.iHeight;
/* Re-init format info and apply-param of decoder */
ff->dec_vfi = pjmedia_get_video_format_info(NULL, ff->param.dec_fmt.id);
if (!ff->dec_vfi)
return PJ_ENOTSUP;
pj_bzero(&ff->dec_vafp, sizeof(ff->dec_vafp));
ff->dec_vafp.size = ff->param.dec_fmt.det.vid.size;
ff->dec_vafp.buffer = NULL;
status = (*ff->dec_vfi->apply_fmt)(ff->dec_vfi, &ff->dec_vafp);
if (status != PJ_SUCCESS)
return status;
/* Realloc buffer if necessary */
if (ff->dec_vafp.framebytes > ff->dec_buf_size) {
PJ_LOG(5,(THIS_FILE, "Reallocating decoding buffer %u --> %u",
(unsigned)ff->dec_buf_size,
(unsigned)ff->dec_vafp.framebytes));
ff->dec_buf_size = ff->dec_vafp.framebytes;
ff->unaligned_dec_buf = pj_pool_alloc(ff->pool, ff->dec_buf_size);
ff->dec_buf = align_buffer_16(ff->unaligned_dec_buf);
}
/* Broadcast format changed event */
pjmedia_event_init(&event, PJMEDIA_EVENT_FMT_CHANGED, ts, codec);
event.data.fmt_changed.dir = PJMEDIA_DIR_DECODING;
pj_memcpy(&event.data.fmt_changed.new_fmt, &ff->param.dec_fmt,
sizeof(ff->param.dec_fmt));
pjmedia_event_publish(NULL, codec, &event, 0);
}
/* Check for missing/found keyframe */
if (got_keyframe) {
pj_get_timestamp(&ff->last_dec_keyframe_ts);
/* Broadcast keyframe event */
pjmedia_event_init(&event, PJMEDIA_EVENT_KEYFRAME_FOUND, ts, codec);
pjmedia_event_publish(NULL, codec, &event, 0);
} else if (ff->last_dec_keyframe_ts.u64 == 0) {
/* Broadcast missing keyframe event */
// pjmedia_event_init(&event, PJMEDIA_EVENT_KEYFRAME_MISSING, ts, codec);
// pjmedia_event_publish(NULL, codec, &event, 0);
}
return PJ_SUCCESS;
}
/*
* Decode frame.
*/
static pj_status_t ffmpeg_codec_decode_whole(pjmedia_vid_codec *codec,
const pjmedia_frame *input,
unsigned output_buf_len,
pjmedia_frame *output)
{
ffmpeg_private *ff = (ffmpeg_private*)codec->codec_data;
AVFrame avframe;
AVPacket avpacket;
int err, got_picture;
/* Check if decoder has been opened */
PJ_ASSERT_RETURN(ff->dec_ctx, PJ_EINVALIDOP);
/* Reset output frame bit info */
output->bit_info = 0;
/* Validate output buffer size */
// Do this validation later after getting decoding result, where the real
// decoded size will be assured.
//if (ff->dec_vafp.framebytes > output_buf_len)
//return PJ_ETOOSMALL;
/* Init frame to receive the decoded data, the ffmpeg codec context will
* automatically provide the decoded buffer (single buffer used for the
* whole decoding session, and seems to be freed when the codec context
* closed).
*/
avcodec_get_frame_defaults(&avframe);
/* Init packet, the container of the encoded data */
av_init_packet(&avpacket);
avpacket.data = (pj_uint8_t*)input->buf;
avpacket.size = input->size;
/* ffmpeg warns:
* - input buffer padding, at least FF_INPUT_BUFFER_PADDING_SIZE
* - null terminated
* Normally, encoded buffer is allocated more than needed, so lets just
* bzero the input buffer end/pad, hope it will be just fine.
*/
pj_bzero(avpacket.data+avpacket.size, FF_INPUT_BUFFER_PADDING_SIZE);
output->bit_info = 0;
output->timestamp = input->timestamp;
#if LIBAVCODEC_VER_AT_LEAST(52,72)
//avpacket.flags = AV_PKT_FLAG_KEY;
#else
avpacket.flags = 0;
#endif
#if LIBAVCODEC_VER_AT_LEAST(52,72)
err = avcodec_decode_video2(ff->dec_ctx, &avframe,
&got_picture, &avpacket);
#else
err = avcodec_decode_video(ff->dec_ctx, &avframe,
&got_picture, avpacket.data, avpacket.size);
#endif
if (err < 0) {
pjmedia_event event;
output->type = PJMEDIA_FRAME_TYPE_NONE;
output->size = 0;
print_ffmpeg_err(err);
/* Broadcast missing keyframe event */
pjmedia_event_init(&event, PJMEDIA_EVENT_KEYFRAME_MISSING,
&input->timestamp, codec);
pjmedia_event_publish(NULL, codec, &event, 0);
return PJMEDIA_CODEC_EBADBITSTREAM;
} else if (got_picture) {
pjmedia_video_apply_fmt_param *vafp = &ff->dec_vafp;
pj_uint8_t *q = (pj_uint8_t*)output->buf;
unsigned i;
pj_status_t status;
/* Check decoding result, e.g: see if the format got changed,
* keyframe found/missing.
*/
status = check_decode_result(codec, &input->timestamp,
avframe.key_frame);
if (status != PJ_SUCCESS)
return status;
/* Check provided buffer size */
if (vafp->framebytes > output_buf_len)
return PJ_ETOOSMALL;
/* Get the decoded data */
for (i = 0; i < ff->dec_vfi->plane_cnt; ++i) {
pj_uint8_t *p = avframe.data[i];
/* The decoded data may contain padding */
if (avframe.linesize[i]!=vafp->strides[i]) {
/* Padding exists, copy line by line */
pj_uint8_t *q_end;
q_end = q+vafp->plane_bytes[i];
while(q < q_end) {
pj_memcpy(q, p, vafp->strides[i]);
q += vafp->strides[i];
p += avframe.linesize[i];
}
} else {
/* No padding, copy the whole plane */
pj_memcpy(q, p, vafp->plane_bytes[i]);
q += vafp->plane_bytes[i];
}
}
output->type = PJMEDIA_FRAME_TYPE_VIDEO;
output->size = vafp->framebytes;
} else {
output->type = PJMEDIA_FRAME_TYPE_NONE;
output->size = 0;
}
return PJ_SUCCESS;
}
static pj_status_t check_decode_result(pjmedia_vid_codec *codec,
const pj_timestamp *ts,
pj_bool_t got_keyframe)
{
ffmpeg_private *ff = (ffmpeg_private*)codec->codec_data;
pjmedia_video_apply_fmt_param *vafp = &ff->dec_vafp;
pjmedia_event event;
/* Check for format change.
* Decoder output format is set by libavcodec, in case it is different
* to the configured param.
*/
if (ff->dec_ctx->pix_fmt != ff->expected_dec_fmt ||
ff->dec_ctx->width != (int)vafp->size.w ||
ff->dec_ctx->height != (int)vafp->size.h)
{
pjmedia_format_id new_fmt_id;
pj_status_t status;
/* Get current raw format id from ffmpeg decoder context */
status = PixelFormat_to_pjmedia_format_id(ff->dec_ctx->pix_fmt,
&new_fmt_id);
if (status != PJ_SUCCESS)
return status;
/* Update decoder format in param */
ff->param.dec_fmt.id = new_fmt_id;
ff->param.dec_fmt.det.vid.size.w = ff->dec_ctx->width;
ff->param.dec_fmt.det.vid.size.h = ff->dec_ctx->height;
ff->expected_dec_fmt = ff->dec_ctx->pix_fmt;
/* Re-init format info and apply-param of decoder */
ff->dec_vfi = pjmedia_get_video_format_info(NULL, ff->param.dec_fmt.id);
if (!ff->dec_vfi)
return PJ_ENOTSUP;
pj_bzero(&ff->dec_vafp, sizeof(ff->dec_vafp));
ff->dec_vafp.size = ff->param.dec_fmt.det.vid.size;
ff->dec_vafp.buffer = NULL;
status = (*ff->dec_vfi->apply_fmt)(ff->dec_vfi, &ff->dec_vafp);
if (status != PJ_SUCCESS)
return status;
/* Realloc buffer if necessary */
if (ff->dec_vafp.framebytes > ff->dec_buf_size) {
PJ_LOG(5,(THIS_FILE, "Reallocating decoding buffer %u --> %u",
(unsigned)ff->dec_buf_size,
(unsigned)ff->dec_vafp.framebytes));
ff->dec_buf_size = ff->dec_vafp.framebytes;
ff->dec_buf = pj_pool_alloc(ff->pool, ff->dec_buf_size);
}
/* Broadcast format changed event */
pjmedia_event_init(&event, PJMEDIA_EVENT_FMT_CHANGED, ts, codec);
event.data.fmt_changed.dir = PJMEDIA_DIR_DECODING;
pj_memcpy(&event.data.fmt_changed.new_fmt, &ff->param.dec_fmt,
sizeof(ff->param.dec_fmt));
pjmedia_event_publish(NULL, codec, &event, 0);
}
/* Check for missing/found keyframe */
if (got_keyframe) {
pj_get_timestamp(&ff->last_dec_keyframe_ts);
/* Broadcast keyframe event */
pjmedia_event_init(&event, PJMEDIA_EVENT_KEYFRAME_FOUND, ts, codec);
pjmedia_event_publish(NULL, codec, &event, 0);
} else if (ff->last_dec_keyframe_ts.u64 == 0) {
/* Broadcast missing keyframe event */
pjmedia_event_init(&event, PJMEDIA_EVENT_KEYFRAME_MISSING, ts, codec);
pjmedia_event_publish(NULL, codec, &event, 0);
}
return PJ_SUCCESS;
}