M4Err MediaCodec_Process(GenericCodec *codec, u32 TimeAvailable)
{
LPAUBUFFER AU;
Channel *ch;
char *cu_buf;
u32 cu_buf_size, mmlevel, deltaTS;
u32 first, entryTime, now, obj_time;
MediaDecoder *mdec = (MediaDecoder*)codec->decio;
M4Err e = M4OK;
/*if video codec muted don't decode
if audio codec muted we dispatch to keep sync in place*/
if (codec->Muted && (codec->type==M4ST_VISUAL) ) return M4OK;
entryTime = Term_GetTime(codec->odm->term);
/*fetch next AU in DTS order for this codec*/
Decoder_GetNextAU(codec, &ch, &AU);
/*no active channel return*/
if (!AU || !ch) {
/*if the codec is in EOS state, assume we're done*/
if (codec->Status == CODEC_EOS) {
/*if codec is reordering, try to flush it*/
if (codec->is_reordering) {
if ( LockCompositionUnit(codec, codec->last_unit_cts+1, &cu_buf, &cu_buf_size) == M4OutOfMem)
return M4OK;
e = mdec->ProcessData(mdec, NULL, 0, 0, cu_buf, &cu_buf_size, 0, 0);
if (e==M4OK) e = UnlockCompositionUnit(codec, codec->last_unit_cts+1, cu_buf_size);
}
MM_StopCodec(codec);
if (codec->CB) CB_SetEndOfStream(codec->CB);
}
/*if no data, and channel not buffering, ABORT CB buffer (data timeout or EOS not detectable)*/
else if (ch && !ch->BufferOn)
CB_AbortBuffering(codec->CB);
return M4OK;
}
/*get the object time*/
obj_time = CK_GetTime(codec->ck);
/*Media Time for media codecs is updated in the CB*/
if (!codec->CB) {
Channel_DropAU(ch);
return M4BadParam;
}
/*try to refill the full buffer*/
first = 1;
while (codec->CB->Capacity > codec->CB->UnitCount) {
/*set media processing level*/
mmlevel = MM_LEVEL_NORMAL;
/*SEEK: if the last frame had the same TS, we are seeking. Ask the codec to drop*/
if (!ch->skip_sl && codec->last_unit_cts && (codec->last_unit_cts == AU->CTS) && !ch->esd->dependsOnESID) {
mmlevel = MM_LEVEL_SEEK;
}
/*only perform drop in normal playback*/
else if (codec->CB->Status == CB_PLAY) {
if (!ch->skip_sl && (AU->CTS < obj_time)) {
/*extremely late, even if we decode the renderer will drop the frame
so set the level to drop*/
mmlevel = MM_LEVEL_DROP;
}
/*we are late according to the media manager*/
else if (codec->PriorityBoost) {
mmlevel = MM_LEVEL_VERY_LATE;
}
/*otherwise we must have an idea of the load in order to set the right level
use the composition buffer for that, only on the first frame*/
else if (first) {
//if the CB is almost empty set to very late
if (codec->CB->UnitCount <= codec->CB->Min+1) {
mmlevel = MM_LEVEL_VERY_LATE;
} else if (codec->CB->UnitCount * 2 <= codec->CB->Capacity) {
mmlevel = MM_LEVEL_LATE;
}
first = 0;
}
}
/*when using temporal scalability make sure we can decode*/
if (ch->esd->dependsOnESID && (codec->last_unit_dts > AU->DTS)){
// printf("SCALABLE STREAM DEAD!!\n");
goto drop;
}
if (ch->skip_sl) {
if (codec->bytes_per_sec) {
AU->CTS = codec->last_unit_cts + ch->ts_offset + codec->cur_audio_bytes * 1000 / codec->bytes_per_sec;
} else if (codec->fps) {
AU->CTS = codec->last_unit_cts + ch->ts_offset + (u32) (codec->cur_video_frames * 1000 / codec->fps);
}
}
if ( LockCompositionUnit(codec, AU->CTS, &cu_buf, &cu_buf_size) == M4OutOfMem)
return M4OK;
now = Term_GetTime(codec->odm->term);
e = mdec->ProcessData(mdec, AU->data, AU->dataLength,
ch->esd->ESID, cu_buf, &cu_buf_size, AU->PaddingBits, mmlevel);
now = Term_GetTime(codec->odm->term) - now;
/*input is too small, resize composition memory*/
switch (e) {
case M4BufferTooSmall:
/*release but no dispatch*/
UnlockCompositionUnit(codec, AU->CTS, 0);
if (ResizeCompositionBuffer(codec, cu_buf_size)==M4OK) continue;
break;
case M4OK:
/*in seek don't dispatch any output*/
if (mmlevel == MM_LEVEL_SEEK) cu_buf_size = 0;
e = UnlockCompositionUnit(codec, AU->CTS, cu_buf_size);
codec_update_stats(codec, AU->dataLength, now);
if (ch->skip_sl) {
if (codec->bytes_per_sec) {
codec->cur_audio_bytes += cu_buf_size;
while (codec->cur_audio_bytes>codec->bytes_per_sec) {
codec->cur_audio_bytes -= codec->bytes_per_sec;
codec->last_unit_cts += 1000;
}
} else if (codec->fps && cu_buf_size) {
codec->cur_video_frames += 1;
}
}
break;
/*this happens a lot when using non-MPEG-4 streams (ex: ffmpeg demuxer)*/
case M4PackedFrames:
/*in seek don't dispatch any output*/
if (mmlevel == MM_LEVEL_SEEK) cu_buf_size = 0;
e = UnlockCompositionUnit(codec, AU->CTS, cu_buf_size);
if (ch->skip_sl) {
if (codec->bytes_per_sec) {
codec->cur_audio_bytes += cu_buf_size;
} else if (codec->fps && cu_buf_size) {
codec->cur_video_frames += 1;
}
} else {
if (codec->bytes_per_sec) {
deltaTS = cu_buf_size * 1000 / codec->bytes_per_sec;
} else if (codec->fps && cu_buf_size) {
deltaTS = (u32) (1000.0f / codec->fps);
}
AU->DTS += deltaTS;
AU->CTS += deltaTS;
}
codec_update_stats(codec, 0, now);
continue;
default:
UnlockCompositionUnit(codec, AU->CTS, 0);
/*error - if the object is in intitial buffering resume it!!*/
CB_AbortBuffering(codec->CB);
break;
}
codec->last_unit_dts = AU->DTS;
/*remember base layer timing*/
if (!ch->esd->dependsOnESID && !ch->skip_sl) codec->last_unit_cts = AU->CTS;
drop:
Channel_DropAU(ch);
if (e) return e;
/*escape from decoding loop only if above critical limit - this is to avoid starvation on audio*/
if (codec->CB->UnitCount > codec->CB->Min) {
now = Term_GetTime(codec->odm->term);
if (now - entryTime >= TimeAvailable - TIME_CHECK) {
return M4OK;
}
}
Decoder_GetNextAU(codec, &ch, &AU);
if (!ch || !AU) return M4OK;
}
return M4OK;
}
static GF_Err MediaCodec_Process(GF_Codec *codec, u32 TimeAvailable)
{
GF_CMUnit *CU;
GF_DBUnit *AU;
GF_Channel *ch, *prev_ch;
u32 mmlevel, cts;
u32 first, entryTime, now, obj_time, unit_size;
GF_MediaDecoder *mdec = (GF_MediaDecoder*)codec->decio;
GF_Err e = GF_OK;
CU = NULL;
/*if video codec muted don't decode (try to saves ressources)
if audio codec muted we dispatch to keep sync in place*/
if (codec->Muted && (codec->type==GF_STREAM_VISUAL) ) return GF_OK;
entryTime = gf_term_get_time(codec->odm->term);
/*fetch next AU in DTS order for this codec*/
Decoder_GetNextAU(codec, &ch, &AU);
/*no active channel return*/
if (!AU || !ch) {
/*if the codec is in EOS state, assume we're done*/
if (codec->Status == GF_ESM_CODEC_EOS) {
/*if codec is reordering, try to flush it*/
if (codec->is_reordering) {
if ( LockCompositionUnit(codec, codec->last_unit_cts+1, &CU, &unit_size) == GF_OUT_OF_MEM)
return GF_OK;
assert( CU );
e = mdec->ProcessData(mdec, NULL, 0, 0, CU->data, &unit_size, 0, 0);
if (e==GF_OK) e = UnlockCompositionUnit(codec, CU, unit_size);
}
gf_term_stop_codec(codec);
if (codec->CB) gf_cm_set_eos(codec->CB);
}
/*if no data, and channel not buffering, ABORT CB buffer (data timeout or EOS not detectable)*/
else if (ch && !ch->BufferOn)
gf_cm_abort_buffering(codec->CB);
return GF_OK;
}
/*get the object time*/
obj_time = gf_clock_time(codec->ck);
/*Media Time for media codecs is updated in the CB*/
if (!codec->CB) {
gf_es_drop_au(ch);
return GF_BAD_PARAM;
}
/*image codecs*/
if (codec->CB->Capacity == 1) {
/*usually only one image is tolerated in the stream, but just in case force reset of CB*/
if (codec->CB->UnitCount && (obj_time>=AU->CTS)) {
gf_mx_p(codec->odm->mx);
codec->CB->output->dataLength = 0;
codec->CB->UnitCount = 0;
gf_mx_v(codec->odm->mx);
}
/*CB is already full*/
if (codec->CB->UnitCount)
return GF_OK;
/*a SHA signature is computed for each AU. This avoids decoding/recompositing when identical (for instance streaming a carousel)*/
{
u8 new_unit_signature[20];
gf_sha1_csum(AU->data, AU->dataLength, new_unit_signature);
if (!memcmp(codec->last_unit_signature, new_unit_signature, sizeof(new_unit_signature))) {
codec->nb_repeted_frames++;
gf_es_drop_au(ch);
return GF_OK;
}
codec->nb_repeted_frames = 0;
memcpy(codec->last_unit_signature, new_unit_signature, sizeof(new_unit_signature));
}
}
/*try to refill the full buffer*/
first = 1;
while (codec->CB->Capacity > codec->CB->UnitCount) {
/*set media processing level*/
mmlevel = GF_CODEC_LEVEL_NORMAL;
/*SEEK: if the last frame had the same TS, we are seeking. Ask the codec to drop*/
if (!ch->skip_sl && codec->last_unit_cts && (codec->last_unit_cts == AU->CTS) && !ch->esd->dependsOnESID) {
mmlevel = GF_CODEC_LEVEL_SEEK;
/*object clock is paused by media control or terminal is paused: exact frame seek*/
if (
#ifndef GPAC_DISABLE_VRML
(codec->ck->mc && codec->ck->mc->paused) ||
#endif
(codec->odm->term->play_state)
) {
gf_cm_rewind_input(codec->CB);
mmlevel = GF_CODEC_LEVEL_NORMAL;
/*force staying in step-mode*/
codec->odm->term->compositor->step_mode=1;
}
}
/*only perform drop in normal playback*/
else if (codec->CB->Status == CB_PLAY) {
/*extremely late, set the level to drop
NOTE: the 100 ms safety gard is to avoid discarding audio*/
if (!ch->skip_sl && (AU->CTS + 100 < obj_time) ) {
mmlevel = GF_CODEC_LEVEL_DROP;
GF_LOG(GF_LOG_DEBUG, GF_LOG_CODEC, ("[%s] ODM%d: frame too late (%d vs %d) - using drop level\n", codec->decio->module_name, codec->odm->OD->objectDescriptorID, AU->CTS, obj_time));
if (ch->resync_drift && (AU->CTS + ch->resync_drift < obj_time)) {
ch->clock->StartTime += (obj_time - AU->CTS);
GF_LOG(GF_LOG_WARNING, GF_LOG_CODEC, ("[%s] ODM%d: decoder too slow on OCR stream - rewinding clock of %d ms\n", codec->decio->module_name, codec->odm->OD->objectDescriptorID, obj_time - AU->CTS));
obj_time = gf_clock_time(codec->ck);
}
}
/*we are late according to the media manager*/
else if (codec->PriorityBoost) {
mmlevel = GF_CODEC_LEVEL_VERY_LATE;
}
/*otherwise we must have an idea of the load in order to set the right level
use the composition buffer for that, only on the first frame*/
else if (first) {
//if the CB is almost empty set to very late
if (codec->CB->UnitCount <= codec->CB->Min+1) {
mmlevel = GF_CODEC_LEVEL_VERY_LATE;
} else if (codec->CB->UnitCount * 2 <= codec->CB->Capacity) {
mmlevel = GF_CODEC_LEVEL_LATE;
}
first = 0;
}
}
/*when using temporal scalability make sure we can decode*/
if (ch->esd->dependsOnESID && (codec->last_unit_dts > AU->DTS)){
// printf("SCALABLE STREAM DEAD!!\n");
goto drop;
}
if (ch->skip_sl) {
if (codec->bytes_per_sec) {
AU->CTS = codec->last_unit_cts + ch->ts_offset + codec->cur_audio_bytes * 1000 / codec->bytes_per_sec;
} else if (codec->fps) {
AU->CTS = codec->last_unit_cts + ch->ts_offset + (u32) (codec->cur_video_frames * 1000 / codec->fps);
}
}
if ( LockCompositionUnit(codec, AU->CTS, &CU, &unit_size) == GF_OUT_OF_MEM)
return GF_OK;
scalable_retry:
now = gf_term_get_time(codec->odm->term);
assert( CU );
if (!CU->data && unit_size)
e = GF_OUT_OF_MEM;
else
e = mdec->ProcessData(mdec, AU->data, AU->dataLength, ch->esd->ESID, CU->data, &unit_size, AU->PaddingBits, mmlevel);
now = gf_term_get_time(codec->odm->term) - now;
if (codec->Status == GF_ESM_CODEC_STOP) return GF_OK;
/*input is too small, resize composition memory*/
switch (e) {
case GF_BUFFER_TOO_SMALL:
/*release but no dispatch*/
UnlockCompositionUnit(codec, CU, 0);
ResizeCompositionBuffer(codec, unit_size);
continue;
/*this happens a lot when using non-MPEG-4 streams (ex: ffmpeg demuxer)*/
case GF_PACKED_FRAMES:
/*in seek don't dispatch any output*/
if (mmlevel == GF_CODEC_LEVEL_SEEK)
unit_size = 0;
e = UnlockCompositionUnit(codec, CU, unit_size);
GF_LOG(GF_LOG_DEBUG, GF_LOG_RTI|GF_LOG_CODEC, ("[%s] ODM%d at %d decoded packed frame TS %d in %d ms\n", codec->decio->module_name, codec->odm->OD->objectDescriptorID, gf_clock_real_time(ch->clock), AU->CTS, now));
if (ch->skip_sl) {
if (codec->bytes_per_sec) {
codec->cur_audio_bytes += unit_size;
} else if (codec->fps && unit_size) {
codec->cur_video_frames += 1;
}
} else {
u32 deltaTS = 0;
if (codec->bytes_per_sec) {
deltaTS = unit_size * 1000 / codec->bytes_per_sec;
} /*else if (0 && codec->fps && unit_size) {
deltaTS = (u32) (1000.0f / codec->fps);
} */else {
deltaTS = (AU->DTS - codec->last_unit_dts);
}
AU->CTS += deltaTS;
}
codec_update_stats(codec, 0, now);
continue;
/*for all cases below, don't release the composition buffer until we are sure we are not
processing a scalable stream*/
case GF_OK:
if (unit_size) {
GF_LOG(GF_LOG_DEBUG, GF_LOG_RTI|GF_LOG_CODEC, ("[%s] ODM%d at %d decoded frame TS %d in %d ms (DTS %d) - %d in CB\n", codec->decio->module_name, codec->odm->OD->objectDescriptorID, gf_clock_real_time(ch->clock), AU->CTS, now, AU->DTS, codec->CB->UnitCount + 1));
}
/*if no size the decoder is not using the composition memory - if the object is in intitial buffering resume it!!*/
else if (codec->CB->Status == CB_BUFFER) {
gf_cm_abort_buffering(codec->CB);
}
codec_update_stats(codec, AU->dataLength, now);
if (ch->skip_sl) {
if (codec->bytes_per_sec) {
codec->cur_audio_bytes += unit_size;
while (codec->cur_audio_bytes>codec->bytes_per_sec) {
codec->cur_audio_bytes -= codec->bytes_per_sec;
codec->last_unit_cts += 1000;
}
} else if (codec->fps && unit_size) {
codec->cur_video_frames += 1;
}
}
#ifndef GPAC_DISABLE_LOGS
if (codec->odm->flags & GF_ODM_PREFETCH) {
GF_LOG(GF_LOG_INFO, GF_LOG_CODEC, ("[%s] ODM%d At %d decoding frame TS %d in prefetch mode\n", codec->decio->module_name, codec->odm->OD->objectDescriptorID, gf_clock_real_time(ch->clock) ));
}
#endif
break;
default:
unit_size = 0;
/*error - if the object is in intitial buffering resume it!!*/
gf_cm_abort_buffering(codec->CB);
GF_LOG(GF_LOG_DEBUG, GF_LOG_CODEC, ("[%s] ODM%d At %d (frame TS %d - %d ms ): decoded error %s\n", codec->decio->module_name, codec->odm->OD->objectDescriptorID, gf_clock_real_time(ch->clock), AU->CTS, now, gf_error_to_string(e) ));
e = GF_OK;
break;
}
codec->last_unit_dts = AU->DTS;
/*remember base layer timing*/
if (!ch->esd->dependsOnESID && !ch->skip_sl) codec->last_unit_cts = AU->CTS;
drop:
/*store current CTS*/
cts = AU->CTS;
prev_ch = ch;
gf_es_drop_au(ch);
AU = NULL;
if (e) {
UnlockCompositionUnit(codec, CU, unit_size);
return e;
}
Decoder_GetNextAU(codec, &ch, &AU);
/*same CTS: same output, likely scalable stream so don't release the CB*/
if (AU && (AU->CTS == cts) && (ch !=prev_ch) ) {
unit_size = codec->CB->UnitSize;
goto scalable_retry;
}
/*in seek don't dispatch any output*/
if (mmlevel == GF_CODEC_LEVEL_SEEK)
unit_size = 0;
UnlockCompositionUnit(codec, CU, unit_size);
if (!ch || !AU) return GF_OK;
/*escape from decoding loop only if above critical limit - this is to avoid starvation on audio*/
if (!ch->esd->dependsOnESID && (codec->CB->UnitCount > codec->CB->Min)) {
now = gf_term_get_time(codec->odm->term);
if (now - entryTime >= TimeAvailable) {
return GF_OK;
}
}
Decoder_GetNextAU(codec, &ch, &AU);
if (!ch || !AU) return GF_OK;
}
return GF_OK;
}