nsresult
AppleVTDecoder::SubmitFrame(MediaRawData* aSample)
{
MOZ_ASSERT(mTaskQueue->IsCurrentThreadIn());
mInputIncoming--;
// For some reason this gives me a double-free error with stagefright.
AutoCFRelease<CMBlockBufferRef> block = nullptr;
AutoCFRelease<CMSampleBufferRef> sample = nullptr;
VTDecodeInfoFlags infoFlags;
OSStatus rv;
// FIXME: This copies the sample data. I think we can provide
// a custom block source which reuses the aSample buffer.
// But note that there may be a problem keeping the samples
// alive over multiple frames.
rv = CMBlockBufferCreateWithMemoryBlock(kCFAllocatorDefault, // Struct allocator.
const_cast<uint8_t*>(aSample->Data()),
aSample->Size(),
kCFAllocatorNull, // Block allocator.
NULL, // Block source.
0, // Data offset.
aSample->Size(),
false,
block.receive());
if (rv != noErr) {
NS_ERROR("Couldn't create CMBlockBuffer");
return NS_ERROR_FAILURE;
}
CMSampleTimingInfo timestamp = TimingInfoFromSample(aSample);
rv = CMSampleBufferCreate(kCFAllocatorDefault, block, true, 0, 0, mFormat, 1, 1, ×tamp, 0, NULL, sample.receive());
if (rv != noErr) {
NS_ERROR("Couldn't create CMSampleBuffer");
return NS_ERROR_FAILURE;
}
mQueuedSamples++;
VTDecodeFrameFlags decodeFlags =
kVTDecodeFrame_EnableAsynchronousDecompression;
rv = VTDecompressionSessionDecodeFrame(mSession,
sample,
decodeFlags,
CreateAppleFrameRef(aSample),
&infoFlags);
if (rv != noErr && !(infoFlags & kVTDecodeInfo_FrameDropped)) {
LOG("AppleVTDecoder: Error %d VTDecompressionSessionDecodeFrame", rv);
NS_WARNING("Couldn't pass frame to decoder");
mCallback->Error();
return NS_ERROR_FAILURE;
}
// Ask for more data.
if (!mInputIncoming && mQueuedSamples <= mMaxRefFrames) {
LOG("AppleVTDecoder task queue empty; requesting more data");
mCallback->InputExhausted();
}
return NS_OK;
}
nsresult
AppleVTDecoder::SubmitFrame(mp4_demuxer::MP4Sample* aSample)
{
// For some reason this gives me a double-free error with stagefright.
AutoCFRelease<CMBlockBufferRef> block = nullptr;
AutoCFRelease<CMSampleBufferRef> sample = nullptr;
VTDecodeInfoFlags flags;
OSStatus rv;
// FIXME: This copies the sample data. I think we can provide
// a custom block source which reuses the aSample buffer.
// But note that there may be a problem keeping the samples
// alive over multiple frames.
rv = CMBlockBufferCreateWithMemoryBlock(kCFAllocatorDefault // Struct allocator.
,aSample->data
,aSample->size
,kCFAllocatorNull // Block allocator.
,NULL // Block source.
,0 // Data offset.
,aSample->size
,false
,block.receive());
if (rv != noErr) {
NS_ERROR("Couldn't create CMBlockBuffer");
return NS_ERROR_FAILURE;
}
CMSampleTimingInfo timestamp = TimingInfoFromSample(aSample);
rv = CMSampleBufferCreate(kCFAllocatorDefault, block, true, 0, 0, mFormat, 1, 1, ×tamp, 0, NULL, sample.receive());
if (rv != noErr) {
NS_ERROR("Couldn't create CMSampleBuffer");
return NS_ERROR_FAILURE;
}
rv = VTDecompressionSessionDecodeFrame(mSession,
sample,
0,
CreateAppleFrameRef(aSample),
&flags);
if (rv != noErr) {
NS_WARNING("Couldn't pass frame to decoder");
return NS_ERROR_FAILURE;
}
// Ask for more data.
if (mTaskQueue->IsEmpty()) {
LOG("AppleVTDecoder task queue empty; requesting more data");
mCallback->InputExhausted();
}
return NS_OK;
}
static CMSampleBufferRef
gst_vtdec_sample_buffer_from (GstVTDec * self, GstBuffer * buf)
{
OSStatus status;
CMBlockBufferRef bbuf = NULL;
CMSampleBufferRef sbuf = NULL;
GstMapInfo map;
CMSampleTimingInfo sample_timing;
CMSampleTimingInfo time_array[1];
g_assert (self->fmt_desc != NULL);
gst_buffer_map (buf, &map, GST_MAP_READ);
status = CMBlockBufferCreateWithMemoryBlock (NULL,
map.data, (gint64) map.size, kCFAllocatorNull, NULL, 0, (gint64) map.size,
FALSE, &bbuf);
gst_buffer_unmap (buf, &map);
if (status != noErr)
goto error;
sample_timing.duration = CMTimeMake (GST_BUFFER_DURATION (buf), 1);
sample_timing.presentationTimeStamp = CMTimeMake (GST_BUFFER_PTS (buf), 1);
sample_timing.decodeTimeStamp = CMTimeMake (GST_BUFFER_DTS (buf), 1);
time_array[0] = sample_timing;
status = CMSampleBufferCreate (NULL, bbuf, TRUE, 0, 0, self->fmt_desc,
1, 1, time_array, 0, NULL, &sbuf);
if (status != noErr)
goto error;
beach:
CFRelease (bbuf);
return sbuf;
error:
GST_ERROR_OBJECT (self, "err %d", status);
goto beach;
}
static void
vtb_decode(struct media_codec *mc, struct video_decoder *vd,
struct media_queue *mq, struct media_buf *mb, int reqsize)
{
vtb_decoder_t *vtbd = mc->opaque;
VTDecodeInfoFlags infoflags;
int flags = kVTDecodeFrame_EnableAsynchronousDecompression | kVTDecodeFrame_EnableTemporalProcessing;
OSStatus status;
CMBlockBufferRef block_buf;
CMSampleBufferRef sample_buf;
vtbd->vtbd_vd = vd;
status =
CMBlockBufferCreateWithMemoryBlock(kCFAllocatorDefault,
mb->mb_data, mb->mb_size,
kCFAllocatorNull,
NULL, 0, mb->mb_size, 0, &block_buf);
if(status) {
TRACE(TRACE_ERROR, "VTB", "Data buffer allocation error %d", status);
return;
}
CMSampleTimingInfo ti;
ti.duration = CMTimeMake(mb->mb_duration, 1000000);
ti.presentationTimeStamp = CMTimeMake(mb->mb_pts, 1000000);
ti.decodeTimeStamp = CMTimeMake(mb->mb_dts, 1000000);
status =
CMSampleBufferCreate(kCFAllocatorDefault,
block_buf, TRUE, 0, 0, vtbd->vtbd_fmt,
1, 1, &ti, 0, NULL, &sample_buf);
CFRelease(block_buf);
if(status) {
TRACE(TRACE_ERROR, "VTB", "Sample buffer allocation error %d", status);
return;
}
void *frame_opaque = &vd->vd_reorder[vd->vd_reorder_ptr];
copy_mbm_from_mb(frame_opaque, mb);
vd->vd_reorder_ptr = (vd->vd_reorder_ptr + 1) & VIDEO_DECODER_REORDER_MASK;
if(mb->mb_skip)
flags |= kVTDecodeFrame_DoNotOutputFrame;
status =
VTDecompressionSessionDecodeFrame(vtbd->vtbd_session, sample_buf, flags,
frame_opaque, &infoflags);
CFRelease(sample_buf);
if(status) {
TRACE(TRACE_ERROR, "VTB", "Decoding error %d", status);
}
hts_mutex_lock(&vtbd->vtbd_mutex);
if(vtbd->vtbd_flush_to != PTS_UNSET) {
vtb_frame_t *vf;
while((vf = LIST_FIRST(&vtbd->vtbd_frames)) != NULL) {
if(vtbd->vtbd_flush_to < vf->vf_mbm.mbm_pts)
break;
LIST_REMOVE(vf, vf_link);
hts_mutex_unlock(&vtbd->vtbd_mutex);
emit_frame(vtbd, vf, mq);
hts_mutex_lock(&vtbd->vtbd_mutex);
CFRelease(vf->vf_buf);
free(vf);
}
}
hts_mutex_unlock(&vtbd->vtbd_mutex);
}
static void h264_dec_process(MSFilter *f) {
VTH264DecCtx *ctx = (VTH264DecCtx *)f->data;
mblk_t *pkt;
mblk_t *nalu;
mblk_t *pixbuf;
MSQueue q_nalus;
MSQueue q_nalus2;
CMBlockBufferRef stream = NULL;
CMSampleBufferRef sample = NULL;
CMSampleTimingInfo timing_info;
MSPicture pixbuf_desc;
OSStatus status;
MSList *parameter_sets = NULL;
bool_t unpacking_failed;
ms_queue_init(&q_nalus);
ms_queue_init(&q_nalus2);
// unpack RTP packet
unpacking_failed = FALSE;
while((pkt = ms_queue_get(f->inputs[0]))) {
unpacking_failed |= (rfc3984_unpack(&ctx->unpacker, pkt, &q_nalus) != 0);
}
if(unpacking_failed) {
ms_error("VideoToolboxDecoder: error while unpacking RTP packets");
goto fail;
}
// Pull out SPSs and PPSs and put them into the filter context if necessary
while((nalu = ms_queue_get(&q_nalus))) {
MSH264NaluType nalu_type = ms_h264_nalu_get_type(nalu);
if(nalu_type == MSH264NaluTypeSPS || nalu_type == MSH264NaluTypePPS) {
parameter_sets = ms_list_append(parameter_sets, nalu);
} else if(ctx->format_desc || parameter_sets) {
ms_queue_put(&q_nalus2, nalu);
} else {
ms_free(nalu);
}
}
if(parameter_sets) {
CMFormatDescriptionRef last_format = ctx->format_desc ? CFRetain(ctx->format_desc) : NULL;
h264_dec_update_format_description(ctx, parameter_sets);
parameter_sets = ms_list_free_with_data(parameter_sets, (void (*)(void *))freemsg);
if(ctx->format_desc == NULL) goto fail;
if(last_format) {
CMVideoDimensions last_vsize = CMVideoFormatDescriptionGetDimensions(last_format);
CMVideoDimensions vsize = CMVideoFormatDescriptionGetDimensions(ctx->format_desc);
if(last_vsize.width != vsize.width || last_vsize.height != vsize.height) {
ms_message("VideoToolboxDecoder: new encoded video size %dx%d -> %dx%d",
(int)last_vsize.width, (int)last_vsize.height, (int)vsize.width, (int)vsize.height);
ms_message("VideoToolboxDecoder: destroying decoding session");
VTDecompressionSessionInvalidate(ctx->session);
CFRelease(ctx->session);
ctx->session = NULL;
}
CFRelease(last_format);
}
}
/* Stops proccessing if no IDR has been received yet */
if(ctx->format_desc == NULL) {
ms_warning("VideoToolboxDecoder: no IDR packet has been received yet");
goto fail;
}
/* Initializes the decoder if it has not be done yet or reconfigure it when
the size of the encoded video change */
if(ctx->session == NULL) {
if(!h264_dec_init_decoder(ctx)) {
ms_error("VideoToolboxDecoder: failed to initialized decoder");
goto fail;
}
}
// Pack all nalus in a VTBlockBuffer
CMBlockBufferCreateEmpty(NULL, 0, kCMBlockBufferAssureMemoryNowFlag, &stream);
while((nalu = ms_queue_get(&q_nalus2))) {
CMBlockBufferRef nalu_block;
size_t nalu_block_size = msgdsize(nalu) + H264_NALU_HEAD_SIZE;
uint32_t nalu_size = htonl(msgdsize(nalu));
CMBlockBufferCreateWithMemoryBlock(NULL, NULL, nalu_block_size, NULL, NULL, 0, nalu_block_size, kCMBlockBufferAssureMemoryNowFlag, &nalu_block);
CMBlockBufferReplaceDataBytes(&nalu_size, nalu_block, 0, H264_NALU_HEAD_SIZE);
CMBlockBufferReplaceDataBytes(nalu->b_rptr, nalu_block, H264_NALU_HEAD_SIZE, msgdsize(nalu));
CMBlockBufferAppendBufferReference(stream, nalu_block, 0, nalu_block_size, 0);
CFRelease(nalu_block);
freemsg(nalu);
}
if(!CMBlockBufferIsEmpty(stream)) {
timing_info.duration = kCMTimeInvalid;
timing_info.presentationTimeStamp = CMTimeMake(f->ticker->time, 1000);
timing_info.decodeTimeStamp = CMTimeMake(f->ticker->time, 1000);
CMSampleBufferCreate(
NULL, stream, TRUE, NULL, NULL,
ctx->format_desc, 1, 1, &timing_info,
0, NULL, &sample);
status = VTDecompressionSessionDecodeFrame(ctx->session, sample, 0, NULL, NULL);
CFRelease(sample);
if(status != noErr) {
CFRelease(stream);
ms_error("VideoToolboxDecoder: error while passing encoded frames to the decoder: %d", status);
if(status == kVTInvalidSessionErr) {
h264_dec_uninit_decoder(ctx);
}
goto fail;
}
}
CFRelease(stream);
goto put_frames_out;
fail:
ms_filter_notify_no_arg(f, MS_VIDEO_DECODER_DECODING_ERRORS);
ms_filter_lock(f);
if(ctx->enable_avpf) {
ms_message("VideoToolboxDecoder: sending PLI");
ms_filter_notify_no_arg(f, MS_VIDEO_DECODER_SEND_PLI);
}
ms_filter_unlock(f);
put_frames_out:
// Transfer decoded frames in the output queue
ms_mutex_lock(&ctx->mutex);
while((pixbuf = ms_queue_get(&ctx->queue))) {
ms_mutex_unlock(&ctx->mutex);
ms_yuv_buf_init_from_mblk(&pixbuf_desc, pixbuf);
ms_filter_lock(f);
if(pixbuf_desc.w != ctx->vsize.width || pixbuf_desc.h != ctx->vsize.height) {
ctx->vsize = (MSVideoSize){ pixbuf_desc.w , pixbuf_desc.h };
}
ms_average_fps_update(&ctx->fps, (uint32_t)f->ticker->time);
if(ctx->first_image) {
ms_filter_notify_no_arg(f, MS_VIDEO_DECODER_FIRST_IMAGE_DECODED);
ctx->first_image = FALSE;
}
ms_filter_unlock(f);
ms_queue_put(f->outputs[0], pixbuf);
ms_mutex_lock(&ctx->mutex);
}
ms_mutex_unlock(&ctx->mutex);
// Cleaning
ms_queue_flush(&q_nalus);
ms_queue_flush(&q_nalus2);
ms_queue_flush(f->inputs[0]);
return;
}
static GF_Err VTBDec_ProcessData(GF_MediaDecoder *ifcg,
char *inBuffer, u32 inBufferLength,
u16 ES_ID, u32 *CTS,
char *outBuffer, u32 *outBufferLength,
u8 PaddingBits, u32 mmlevel)
{
OSStatus status;
CMSampleBufferRef sample = NULL;
CMBlockBufferRef block_buffer = NULL;
OSType type;
char *in_data;
u32 in_data_size;
GF_Err e;
VTBDec *ctx = (VTBDec *)ifcg->privateStack;
if (ctx->skip_mpeg4_vosh) {
GF_M4VDecSpecInfo dsi;
dsi.width = dsi.height = 0;
e = gf_m4v_get_config(inBuffer, inBufferLength, &dsi);
//found a vosh - remove it from payload, init decoder if needed
if ((e==GF_OK) && dsi.width && dsi.height) {
if (!ctx->vtb_session) {
ctx->vosh = inBuffer;
ctx->vosh_size = dsi.next_object_start;
e = VTBDec_InitDecoder(ctx, GF_FALSE);
if (e) return e;
//enfoce removal for all frames
ctx->skip_mpeg4_vosh = GF_TRUE;
if (ctx->out_size != *outBufferLength) {
*outBufferLength = ctx->out_size;
return GF_BUFFER_TOO_SMALL;
}
}
ctx->vosh_size = dsi.next_object_start;
} else if (!ctx->vtb_session) {
*outBufferLength=0;
return GF_OK;
}
}
if (ctx->init_mpeg12) {
GF_M4VDecSpecInfo dsi;
dsi.width = dsi.height = 0;
e = gf_mpegv12_get_config(inBuffer, inBufferLength, &dsi);
if ((e==GF_OK) && dsi.width && dsi.height) {
ctx->width = dsi.width;
ctx->height = dsi.height;
ctx->pixel_ar = dsi.par_num;
ctx->pixel_ar <<= 16;
ctx->pixel_ar |= dsi.par_den;
e = VTBDec_InitDecoder(ctx, GF_FALSE);
if (e) return e;
if (ctx->out_size != *outBufferLength) {
*outBufferLength = ctx->out_size;
return GF_BUFFER_TOO_SMALL;
}
}
if (!ctx->vtb_session) {
*outBufferLength=0;
return GF_OK;
}
}
if (ctx->is_annex_b || (!ctx->vtb_session && ctx->nalu_size_length) ) {
if (ctx->cached_annex_b) {
in_data = ctx->cached_annex_b;
in_data_size = ctx->cached_annex_b_size;
ctx->cached_annex_b = NULL;
} else {
e = VTB_RewriteNALs(ctx, inBuffer, inBufferLength, &in_data, &in_data_size);
if (e) return e;
}
if (ctx->out_size != *outBufferLength) {
*outBufferLength = ctx->out_size;
ctx->cached_annex_b = in_data;
ctx->cached_annex_b_size = in_data_size;
return GF_BUFFER_TOO_SMALL;
}
} else if (ctx->vosh_size) {
in_data = inBuffer + ctx->vosh_size;
in_data_size = inBufferLength - ctx->vosh_size;
ctx->vosh_size = 0;
} else {
in_data = inBuffer;
in_data_size = inBufferLength;
}
if (!ctx->vtb_session) {
*outBufferLength=0;
return GF_OK;
}
status = CMBlockBufferCreateWithMemoryBlock(kCFAllocatorDefault, in_data, in_data_size, kCFAllocatorNull, NULL, 0, in_data_size, 0, &block_buffer);
if (status) {
return GF_IO_ERR;
}
*outBufferLength=0;
if (block_buffer == NULL)
return GF_OK;
status = CMSampleBufferCreate(kCFAllocatorDefault, block_buffer, TRUE, NULL, NULL, ctx->fmt_desc, 1, 0, NULL, 0, NULL, &sample);
if (status || (sample==NULL)) {
if (block_buffer)
CFRelease(block_buffer);
return GF_IO_ERR;
}
ctx->last_error = GF_OK;
status = VTDecompressionSessionDecodeFrame(ctx->vtb_session, sample, 0, NULL, 0);
if (!status)
status = VTDecompressionSessionWaitForAsynchronousFrames(ctx->vtb_session);
CFRelease(block_buffer);
CFRelease(sample);
if (ctx->cached_annex_b)
gf_free(in_data);
if (ctx->last_error) return ctx->last_error;
if (status) return GF_NON_COMPLIANT_BITSTREAM;
if (!ctx->frame) {
*outBufferLength=0;
return ctx->last_error;
}
*outBufferLength = ctx->out_size;
status = CVPixelBufferLockBaseAddress(ctx->frame, kCVPixelBufferLock_ReadOnly);
if (status != kCVReturnSuccess) {
GF_LOG(GF_LOG_ERROR, GF_LOG_CODEC, ("[VTB] Error locking frame data\n"));
return GF_IO_ERR;
}
type = CVPixelBufferGetPixelFormatType(ctx->frame);
if (CVPixelBufferIsPlanar(ctx->frame)) {
u32 i, j, nb_planes = (u32) CVPixelBufferGetPlaneCount(ctx->frame);
char *dst = outBuffer;
Bool needs_stride=GF_FALSE;
if ((type==kCVPixelFormatType_420YpCbCr8Planar)
|| (type==kCVPixelFormatType_420YpCbCr8PlanarFullRange)
|| (type==kCVPixelFormatType_422YpCbCr8_yuvs)
|| (type==kCVPixelFormatType_444YpCbCr8)
|| (type=='444v')
) {
u32 stride = (u32) CVPixelBufferGetBytesPerRowOfPlane(ctx->frame, 0);
//TOCHECK - for now the 3 planes are consecutive in VideoToolbox
if (stride==ctx->width) {
char *data = CVPixelBufferGetBaseAddressOfPlane(ctx->frame, 0);
memcpy(dst, data, sizeof(char)*ctx->out_size);
} else {
for (i=0; i<nb_planes; i++) {
char *data = CVPixelBufferGetBaseAddressOfPlane(ctx->frame, i);
u32 stride = (u32) CVPixelBufferGetBytesPerRowOfPlane(ctx->frame, i);
u32 w, h = (u32) CVPixelBufferGetHeightOfPlane(ctx->frame, i);
w = ctx->width;
if (i) {
switch (ctx->pix_fmt) {
case GF_PIXEL_YUV444:
break;
case GF_PIXEL_YUV422:
case GF_PIXEL_YV12:
w /= 2;
break;
}
}
if (stride != w) {
needs_stride=GF_TRUE;
for (j=0; j<h; j++) {
memcpy(dst, data, sizeof(char)*w);
dst += w;
data += stride;
}
} else {
memcpy(dst, data, sizeof(char)*h*stride);
dst += sizeof(char)*h*stride;
}
}
}
} else if ((type==kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange) || (type==kCVPixelFormatType_420YpCbCr8BiPlanarFullRange)) {
char *dst_v;
char *data = CVPixelBufferGetBaseAddressOfPlane(ctx->frame, 0);
u32 stride = (u32) CVPixelBufferGetBytesPerRowOfPlane(ctx->frame, 0);
u32 i, h = (u32) CVPixelBufferGetHeightOfPlane(ctx->frame, 0);
if (stride==ctx->width) {
memcpy(dst, data, sizeof(char)*h*stride);
dst += sizeof(char)*h*stride;
} else {
for (i=0; i<h; i++) {
memcpy(dst, data, sizeof(char)*ctx->width);
dst += ctx->width;
data += stride;
}
needs_stride=GF_TRUE;
}
data = CVPixelBufferGetBaseAddressOfPlane(ctx->frame, 1);
stride = (u32) CVPixelBufferGetBytesPerRowOfPlane(ctx->frame, 1);
h = (u32) CVPixelBufferGetHeightOfPlane(ctx->frame, 1);
dst_v = dst+sizeof(char) * h*stride/2;
for (i=0; i<ctx->width * h / 2; i++) {
*dst = data[0];
*dst_v = data[1];
data += 2;
dst_v++;
dst++;
if (!(i%ctx->width)) data += (stride - ctx->width);
}
}
}
CVPixelBufferUnlockBaseAddress(ctx->frame, kCVPixelBufferLock_ReadOnly);
return GF_OK;
}
