nsresult AppleVTDecoder::SubmitFrame(MediaRawData* aSample) { MOZ_ASSERT(mTaskQueue->IsCurrentThreadIn()); mInputIncoming--; // For some reason this gives me a double-free error with stagefright. AutoCFRelease<CMBlockBufferRef> block = nullptr; AutoCFRelease<CMSampleBufferRef> sample = nullptr; VTDecodeInfoFlags infoFlags; OSStatus rv; // FIXME: This copies the sample data. I think we can provide // a custom block source which reuses the aSample buffer. // But note that there may be a problem keeping the samples // alive over multiple frames. rv = CMBlockBufferCreateWithMemoryBlock(kCFAllocatorDefault, // Struct allocator. const_cast<uint8_t*>(aSample->Data()), aSample->Size(), kCFAllocatorNull, // Block allocator. NULL, // Block source. 0, // Data offset. aSample->Size(), false, block.receive()); if (rv != noErr) { NS_ERROR("Couldn't create CMBlockBuffer"); return NS_ERROR_FAILURE; } CMSampleTimingInfo timestamp = TimingInfoFromSample(aSample); rv = CMSampleBufferCreate(kCFAllocatorDefault, block, true, 0, 0, mFormat, 1, 1, ×tamp, 0, NULL, sample.receive()); if (rv != noErr) { NS_ERROR("Couldn't create CMSampleBuffer"); return NS_ERROR_FAILURE; } mQueuedSamples++; VTDecodeFrameFlags decodeFlags = kVTDecodeFrame_EnableAsynchronousDecompression; rv = VTDecompressionSessionDecodeFrame(mSession, sample, decodeFlags, CreateAppleFrameRef(aSample), &infoFlags); if (rv != noErr && !(infoFlags & kVTDecodeInfo_FrameDropped)) { LOG("AppleVTDecoder: Error %d VTDecompressionSessionDecodeFrame", rv); NS_WARNING("Couldn't pass frame to decoder"); mCallback->Error(); return NS_ERROR_FAILURE; } // Ask for more data. if (!mInputIncoming && mQueuedSamples <= mMaxRefFrames) { LOG("AppleVTDecoder task queue empty; requesting more data"); mCallback->InputExhausted(); } return NS_OK; }
nsresult AppleVTDecoder::SubmitFrame(mp4_demuxer::MP4Sample* aSample) { // For some reason this gives me a double-free error with stagefright. AutoCFRelease<CMBlockBufferRef> block = nullptr; AutoCFRelease<CMSampleBufferRef> sample = nullptr; VTDecodeInfoFlags flags; OSStatus rv; // FIXME: This copies the sample data. I think we can provide // a custom block source which reuses the aSample buffer. // But note that there may be a problem keeping the samples // alive over multiple frames. rv = CMBlockBufferCreateWithMemoryBlock(kCFAllocatorDefault // Struct allocator. ,aSample->data ,aSample->size ,kCFAllocatorNull // Block allocator. ,NULL // Block source. ,0 // Data offset. ,aSample->size ,false ,block.receive()); if (rv != noErr) { NS_ERROR("Couldn't create CMBlockBuffer"); return NS_ERROR_FAILURE; } CMSampleTimingInfo timestamp = TimingInfoFromSample(aSample); rv = CMSampleBufferCreate(kCFAllocatorDefault, block, true, 0, 0, mFormat, 1, 1, ×tamp, 0, NULL, sample.receive()); if (rv != noErr) { NS_ERROR("Couldn't create CMSampleBuffer"); return NS_ERROR_FAILURE; } rv = VTDecompressionSessionDecodeFrame(mSession, sample, 0, CreateAppleFrameRef(aSample), &flags); if (rv != noErr) { NS_WARNING("Couldn't pass frame to decoder"); return NS_ERROR_FAILURE; } // Ask for more data. if (mTaskQueue->IsEmpty()) { LOG("AppleVTDecoder task queue empty; requesting more data"); mCallback->InputExhausted(); } return NS_OK; }
static CMSampleBufferRef gst_vtdec_sample_buffer_from (GstVTDec * self, GstBuffer * buf) { OSStatus status; CMBlockBufferRef bbuf = NULL; CMSampleBufferRef sbuf = NULL; GstMapInfo map; CMSampleTimingInfo sample_timing; CMSampleTimingInfo time_array[1]; g_assert (self->fmt_desc != NULL); gst_buffer_map (buf, &map, GST_MAP_READ); status = CMBlockBufferCreateWithMemoryBlock (NULL, map.data, (gint64) map.size, kCFAllocatorNull, NULL, 0, (gint64) map.size, FALSE, &bbuf); gst_buffer_unmap (buf, &map); if (status != noErr) goto error; sample_timing.duration = CMTimeMake (GST_BUFFER_DURATION (buf), 1); sample_timing.presentationTimeStamp = CMTimeMake (GST_BUFFER_PTS (buf), 1); sample_timing.decodeTimeStamp = CMTimeMake (GST_BUFFER_DTS (buf), 1); time_array[0] = sample_timing; status = CMSampleBufferCreate (NULL, bbuf, TRUE, 0, 0, self->fmt_desc, 1, 1, time_array, 0, NULL, &sbuf); if (status != noErr) goto error; beach: CFRelease (bbuf); return sbuf; error: GST_ERROR_OBJECT (self, "err %d", status); goto beach; }
static void vtb_decode(struct media_codec *mc, struct video_decoder *vd, struct media_queue *mq, struct media_buf *mb, int reqsize) { vtb_decoder_t *vtbd = mc->opaque; VTDecodeInfoFlags infoflags; int flags = kVTDecodeFrame_EnableAsynchronousDecompression | kVTDecodeFrame_EnableTemporalProcessing; OSStatus status; CMBlockBufferRef block_buf; CMSampleBufferRef sample_buf; vtbd->vtbd_vd = vd; status = CMBlockBufferCreateWithMemoryBlock(kCFAllocatorDefault, mb->mb_data, mb->mb_size, kCFAllocatorNull, NULL, 0, mb->mb_size, 0, &block_buf); if(status) { TRACE(TRACE_ERROR, "VTB", "Data buffer allocation error %d", status); return; } CMSampleTimingInfo ti; ti.duration = CMTimeMake(mb->mb_duration, 1000000); ti.presentationTimeStamp = CMTimeMake(mb->mb_pts, 1000000); ti.decodeTimeStamp = CMTimeMake(mb->mb_dts, 1000000); status = CMSampleBufferCreate(kCFAllocatorDefault, block_buf, TRUE, 0, 0, vtbd->vtbd_fmt, 1, 1, &ti, 0, NULL, &sample_buf); CFRelease(block_buf); if(status) { TRACE(TRACE_ERROR, "VTB", "Sample buffer allocation error %d", status); return; } void *frame_opaque = &vd->vd_reorder[vd->vd_reorder_ptr]; copy_mbm_from_mb(frame_opaque, mb); vd->vd_reorder_ptr = (vd->vd_reorder_ptr + 1) & VIDEO_DECODER_REORDER_MASK; if(mb->mb_skip) flags |= kVTDecodeFrame_DoNotOutputFrame; status = VTDecompressionSessionDecodeFrame(vtbd->vtbd_session, sample_buf, flags, frame_opaque, &infoflags); CFRelease(sample_buf); if(status) { TRACE(TRACE_ERROR, "VTB", "Decoding error %d", status); } hts_mutex_lock(&vtbd->vtbd_mutex); if(vtbd->vtbd_flush_to != PTS_UNSET) { vtb_frame_t *vf; while((vf = LIST_FIRST(&vtbd->vtbd_frames)) != NULL) { if(vtbd->vtbd_flush_to < vf->vf_mbm.mbm_pts) break; LIST_REMOVE(vf, vf_link); hts_mutex_unlock(&vtbd->vtbd_mutex); emit_frame(vtbd, vf, mq); hts_mutex_lock(&vtbd->vtbd_mutex); CFRelease(vf->vf_buf); free(vf); } } hts_mutex_unlock(&vtbd->vtbd_mutex); }
static void h264_dec_process(MSFilter *f) { VTH264DecCtx *ctx = (VTH264DecCtx *)f->data; mblk_t *pkt; mblk_t *nalu; mblk_t *pixbuf; MSQueue q_nalus; MSQueue q_nalus2; CMBlockBufferRef stream = NULL; CMSampleBufferRef sample = NULL; CMSampleTimingInfo timing_info; MSPicture pixbuf_desc; OSStatus status; MSList *parameter_sets = NULL; bool_t unpacking_failed; ms_queue_init(&q_nalus); ms_queue_init(&q_nalus2); // unpack RTP packet unpacking_failed = FALSE; while((pkt = ms_queue_get(f->inputs[0]))) { unpacking_failed |= (rfc3984_unpack(&ctx->unpacker, pkt, &q_nalus) != 0); } if(unpacking_failed) { ms_error("VideoToolboxDecoder: error while unpacking RTP packets"); goto fail; } // Pull out SPSs and PPSs and put them into the filter context if necessary while((nalu = ms_queue_get(&q_nalus))) { MSH264NaluType nalu_type = ms_h264_nalu_get_type(nalu); if(nalu_type == MSH264NaluTypeSPS || nalu_type == MSH264NaluTypePPS) { parameter_sets = ms_list_append(parameter_sets, nalu); } else if(ctx->format_desc || parameter_sets) { ms_queue_put(&q_nalus2, nalu); } else { ms_free(nalu); } } if(parameter_sets) { CMFormatDescriptionRef last_format = ctx->format_desc ? CFRetain(ctx->format_desc) : NULL; h264_dec_update_format_description(ctx, parameter_sets); parameter_sets = ms_list_free_with_data(parameter_sets, (void (*)(void *))freemsg); if(ctx->format_desc == NULL) goto fail; if(last_format) { CMVideoDimensions last_vsize = CMVideoFormatDescriptionGetDimensions(last_format); CMVideoDimensions vsize = CMVideoFormatDescriptionGetDimensions(ctx->format_desc); if(last_vsize.width != vsize.width || last_vsize.height != vsize.height) { ms_message("VideoToolboxDecoder: new encoded video size %dx%d -> %dx%d", (int)last_vsize.width, (int)last_vsize.height, (int)vsize.width, (int)vsize.height); ms_message("VideoToolboxDecoder: destroying decoding session"); VTDecompressionSessionInvalidate(ctx->session); CFRelease(ctx->session); ctx->session = NULL; } CFRelease(last_format); } } /* Stops proccessing if no IDR has been received yet */ if(ctx->format_desc == NULL) { ms_warning("VideoToolboxDecoder: no IDR packet has been received yet"); goto fail; } /* Initializes the decoder if it has not be done yet or reconfigure it when the size of the encoded video change */ if(ctx->session == NULL) { if(!h264_dec_init_decoder(ctx)) { ms_error("VideoToolboxDecoder: failed to initialized decoder"); goto fail; } } // Pack all nalus in a VTBlockBuffer CMBlockBufferCreateEmpty(NULL, 0, kCMBlockBufferAssureMemoryNowFlag, &stream); while((nalu = ms_queue_get(&q_nalus2))) { CMBlockBufferRef nalu_block; size_t nalu_block_size = msgdsize(nalu) + H264_NALU_HEAD_SIZE; uint32_t nalu_size = htonl(msgdsize(nalu)); CMBlockBufferCreateWithMemoryBlock(NULL, NULL, nalu_block_size, NULL, NULL, 0, nalu_block_size, kCMBlockBufferAssureMemoryNowFlag, &nalu_block); CMBlockBufferReplaceDataBytes(&nalu_size, nalu_block, 0, H264_NALU_HEAD_SIZE); CMBlockBufferReplaceDataBytes(nalu->b_rptr, nalu_block, H264_NALU_HEAD_SIZE, msgdsize(nalu)); CMBlockBufferAppendBufferReference(stream, nalu_block, 0, nalu_block_size, 0); CFRelease(nalu_block); freemsg(nalu); } if(!CMBlockBufferIsEmpty(stream)) { timing_info.duration = kCMTimeInvalid; timing_info.presentationTimeStamp = CMTimeMake(f->ticker->time, 1000); timing_info.decodeTimeStamp = CMTimeMake(f->ticker->time, 1000); CMSampleBufferCreate( NULL, stream, TRUE, NULL, NULL, ctx->format_desc, 1, 1, &timing_info, 0, NULL, &sample); status = VTDecompressionSessionDecodeFrame(ctx->session, sample, 0, NULL, NULL); CFRelease(sample); if(status != noErr) { CFRelease(stream); ms_error("VideoToolboxDecoder: error while passing encoded frames to the decoder: %d", status); if(status == kVTInvalidSessionErr) { h264_dec_uninit_decoder(ctx); } goto fail; } } CFRelease(stream); goto put_frames_out; fail: ms_filter_notify_no_arg(f, MS_VIDEO_DECODER_DECODING_ERRORS); ms_filter_lock(f); if(ctx->enable_avpf) { ms_message("VideoToolboxDecoder: sending PLI"); ms_filter_notify_no_arg(f, MS_VIDEO_DECODER_SEND_PLI); } ms_filter_unlock(f); put_frames_out: // Transfer decoded frames in the output queue ms_mutex_lock(&ctx->mutex); while((pixbuf = ms_queue_get(&ctx->queue))) { ms_mutex_unlock(&ctx->mutex); ms_yuv_buf_init_from_mblk(&pixbuf_desc, pixbuf); ms_filter_lock(f); if(pixbuf_desc.w != ctx->vsize.width || pixbuf_desc.h != ctx->vsize.height) { ctx->vsize = (MSVideoSize){ pixbuf_desc.w , pixbuf_desc.h }; } ms_average_fps_update(&ctx->fps, (uint32_t)f->ticker->time); if(ctx->first_image) { ms_filter_notify_no_arg(f, MS_VIDEO_DECODER_FIRST_IMAGE_DECODED); ctx->first_image = FALSE; } ms_filter_unlock(f); ms_queue_put(f->outputs[0], pixbuf); ms_mutex_lock(&ctx->mutex); } ms_mutex_unlock(&ctx->mutex); // Cleaning ms_queue_flush(&q_nalus); ms_queue_flush(&q_nalus2); ms_queue_flush(f->inputs[0]); return; }
static GF_Err VTBDec_ProcessData(GF_MediaDecoder *ifcg, char *inBuffer, u32 inBufferLength, u16 ES_ID, u32 *CTS, char *outBuffer, u32 *outBufferLength, u8 PaddingBits, u32 mmlevel) { OSStatus status; CMSampleBufferRef sample = NULL; CMBlockBufferRef block_buffer = NULL; OSType type; char *in_data; u32 in_data_size; GF_Err e; VTBDec *ctx = (VTBDec *)ifcg->privateStack; if (ctx->skip_mpeg4_vosh) { GF_M4VDecSpecInfo dsi; dsi.width = dsi.height = 0; e = gf_m4v_get_config(inBuffer, inBufferLength, &dsi); //found a vosh - remove it from payload, init decoder if needed if ((e==GF_OK) && dsi.width && dsi.height) { if (!ctx->vtb_session) { ctx->vosh = inBuffer; ctx->vosh_size = dsi.next_object_start; e = VTBDec_InitDecoder(ctx, GF_FALSE); if (e) return e; //enfoce removal for all frames ctx->skip_mpeg4_vosh = GF_TRUE; if (ctx->out_size != *outBufferLength) { *outBufferLength = ctx->out_size; return GF_BUFFER_TOO_SMALL; } } ctx->vosh_size = dsi.next_object_start; } else if (!ctx->vtb_session) { *outBufferLength=0; return GF_OK; } } if (ctx->init_mpeg12) { GF_M4VDecSpecInfo dsi; dsi.width = dsi.height = 0; e = gf_mpegv12_get_config(inBuffer, inBufferLength, &dsi); if ((e==GF_OK) && dsi.width && dsi.height) { ctx->width = dsi.width; ctx->height = dsi.height; ctx->pixel_ar = dsi.par_num; ctx->pixel_ar <<= 16; ctx->pixel_ar |= dsi.par_den; e = VTBDec_InitDecoder(ctx, GF_FALSE); if (e) return e; if (ctx->out_size != *outBufferLength) { *outBufferLength = ctx->out_size; return GF_BUFFER_TOO_SMALL; } } if (!ctx->vtb_session) { *outBufferLength=0; return GF_OK; } } if (ctx->is_annex_b || (!ctx->vtb_session && ctx->nalu_size_length) ) { if (ctx->cached_annex_b) { in_data = ctx->cached_annex_b; in_data_size = ctx->cached_annex_b_size; ctx->cached_annex_b = NULL; } else { e = VTB_RewriteNALs(ctx, inBuffer, inBufferLength, &in_data, &in_data_size); if (e) return e; } if (ctx->out_size != *outBufferLength) { *outBufferLength = ctx->out_size; ctx->cached_annex_b = in_data; ctx->cached_annex_b_size = in_data_size; return GF_BUFFER_TOO_SMALL; } } else if (ctx->vosh_size) { in_data = inBuffer + ctx->vosh_size; in_data_size = inBufferLength - ctx->vosh_size; ctx->vosh_size = 0; } else { in_data = inBuffer; in_data_size = inBufferLength; } if (!ctx->vtb_session) { *outBufferLength=0; return GF_OK; } status = CMBlockBufferCreateWithMemoryBlock(kCFAllocatorDefault, in_data, in_data_size, kCFAllocatorNull, NULL, 0, in_data_size, 0, &block_buffer); if (status) { return GF_IO_ERR; } *outBufferLength=0; if (block_buffer == NULL) return GF_OK; status = CMSampleBufferCreate(kCFAllocatorDefault, block_buffer, TRUE, NULL, NULL, ctx->fmt_desc, 1, 0, NULL, 0, NULL, &sample); if (status || (sample==NULL)) { if (block_buffer) CFRelease(block_buffer); return GF_IO_ERR; } ctx->last_error = GF_OK; status = VTDecompressionSessionDecodeFrame(ctx->vtb_session, sample, 0, NULL, 0); if (!status) status = VTDecompressionSessionWaitForAsynchronousFrames(ctx->vtb_session); CFRelease(block_buffer); CFRelease(sample); if (ctx->cached_annex_b) gf_free(in_data); if (ctx->last_error) return ctx->last_error; if (status) return GF_NON_COMPLIANT_BITSTREAM; if (!ctx->frame) { *outBufferLength=0; return ctx->last_error; } *outBufferLength = ctx->out_size; status = CVPixelBufferLockBaseAddress(ctx->frame, kCVPixelBufferLock_ReadOnly); if (status != kCVReturnSuccess) { GF_LOG(GF_LOG_ERROR, GF_LOG_CODEC, ("[VTB] Error locking frame data\n")); return GF_IO_ERR; } type = CVPixelBufferGetPixelFormatType(ctx->frame); if (CVPixelBufferIsPlanar(ctx->frame)) { u32 i, j, nb_planes = (u32) CVPixelBufferGetPlaneCount(ctx->frame); char *dst = outBuffer; Bool needs_stride=GF_FALSE; if ((type==kCVPixelFormatType_420YpCbCr8Planar) || (type==kCVPixelFormatType_420YpCbCr8PlanarFullRange) || (type==kCVPixelFormatType_422YpCbCr8_yuvs) || (type==kCVPixelFormatType_444YpCbCr8) || (type=='444v') ) { u32 stride = (u32) CVPixelBufferGetBytesPerRowOfPlane(ctx->frame, 0); //TOCHECK - for now the 3 planes are consecutive in VideoToolbox if (stride==ctx->width) { char *data = CVPixelBufferGetBaseAddressOfPlane(ctx->frame, 0); memcpy(dst, data, sizeof(char)*ctx->out_size); } else { for (i=0; i<nb_planes; i++) { char *data = CVPixelBufferGetBaseAddressOfPlane(ctx->frame, i); u32 stride = (u32) CVPixelBufferGetBytesPerRowOfPlane(ctx->frame, i); u32 w, h = (u32) CVPixelBufferGetHeightOfPlane(ctx->frame, i); w = ctx->width; if (i) { switch (ctx->pix_fmt) { case GF_PIXEL_YUV444: break; case GF_PIXEL_YUV422: case GF_PIXEL_YV12: w /= 2; break; } } if (stride != w) { needs_stride=GF_TRUE; for (j=0; j<h; j++) { memcpy(dst, data, sizeof(char)*w); dst += w; data += stride; } } else { memcpy(dst, data, sizeof(char)*h*stride); dst += sizeof(char)*h*stride; } } } } else if ((type==kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange) || (type==kCVPixelFormatType_420YpCbCr8BiPlanarFullRange)) { char *dst_v; char *data = CVPixelBufferGetBaseAddressOfPlane(ctx->frame, 0); u32 stride = (u32) CVPixelBufferGetBytesPerRowOfPlane(ctx->frame, 0); u32 i, h = (u32) CVPixelBufferGetHeightOfPlane(ctx->frame, 0); if (stride==ctx->width) { memcpy(dst, data, sizeof(char)*h*stride); dst += sizeof(char)*h*stride; } else { for (i=0; i<h; i++) { memcpy(dst, data, sizeof(char)*ctx->width); dst += ctx->width; data += stride; } needs_stride=GF_TRUE; } data = CVPixelBufferGetBaseAddressOfPlane(ctx->frame, 1); stride = (u32) CVPixelBufferGetBytesPerRowOfPlane(ctx->frame, 1); h = (u32) CVPixelBufferGetHeightOfPlane(ctx->frame, 1); dst_v = dst+sizeof(char) * h*stride/2; for (i=0; i<ctx->width * h / 2; i++) { *dst = data[0]; *dst_v = data[1]; data += 2; dst_v++; dst++; if (!(i%ctx->width)) data += (stride - ctx->width); } } } CVPixelBufferUnlockBaseAddress(ctx->frame, kCVPixelBufferLock_ReadOnly); return GF_OK; }