static GF_Err SystemCodec_Process(GF_Codec *codec, u32 TimeAvailable)
{
GF_DBUnit *AU;
GF_Channel *ch;
u32 now, obj_time, mm_level, au_time, cts;
GF_Scene *scene_locked;
Bool check_next_unit;
GF_SceneDecoder *sdec = (GF_SceneDecoder *)codec->decio;
GF_Err e = GF_OK;
scene_locked = NULL;
/*for resync, if needed - the logic behind this is that there is no composition memory on sytems codecs so
"frame dropping" is done by preventing the compositor from redrawing after an update and decoding following AU
so that the compositor is always woken up once all late systems AUs are decoded. This flag is overriden when
seeking*/
check_next_unit = (codec->odm->term->flags & GF_TERM_SYSDEC_RESYNC) ? 1 : 0;
check_unit:
/*muting systems codec means we don't decode until mute is off - likely there will be visible however
there is no other way to decode system AUs without modifying the content, which is what mute is about on visual...*/
if (codec->Muted) goto exit;
/*fetch next AU in DTS order for this codec*/
Decoder_GetNextAU(codec, &ch, &AU);
/*no active channel return*/
if (!AU || !ch) {
/*if the codec is in EOS state, move to STOP*/
if (codec->Status == GF_ESM_CODEC_EOS) {
GF_CodecCapability cap;
cap.CapCode = GF_CODEC_MEDIA_NOT_OVER;
cap.cap.valueInt = 0;
sdec->GetCapabilities(codec->decio, &cap);
if (!cap.cap.valueInt) {
gf_term_stop_codec(codec);
if ((codec->type==GF_STREAM_OD) && (codec->nb_dec_frames==1)) {
/*this is just by safety, since seeking is only allowed when a single clock is present
in the scene*/
if (gf_list_count(codec->odm->net_service->Clocks)==1)
codec->odm->subscene->static_media_ressources=1;
}
}
}
goto exit;
}
#ifndef GPAC_DISABLE_VRML
if (ch && ch->odm->media_ctrl && !ch->odm->media_ctrl->media_speed)
goto exit;
#endif
/*get the object time*/
obj_time = gf_clock_time(codec->ck);
/*clock is not started*/
// if (ch->first_au_fetched && !gf_clock_is_started(ch->clock)) goto exit;
/*check timing based on the input channel and main FPS*/
if (AU->DTS > obj_time /*+ codec->odm->term->half_frame_duration*/) goto exit;
cts = AU->CTS;
/*in cases where no CTS was set for the BIFS (which may be interpreted as "now", although not compliant), use the object clock*/
if (AU->flags & GF_DB_AU_NO_TIMESTAMPS) au_time = obj_time;
/*case where CTS is in the past (carousels) */
else if (AU->flags & GF_DB_AU_CTS_IN_PAST) {
au_time = - (s32) AU->CTS;
cts = AU->DTS;
}
/*regular case, SFTime will be from CTS (since DTS == CTS)*/
else au_time = AU->DTS;
/*check seeking and update timing - do NOT use the base layer, since BIFS streams may depend on other
streams not on the same clock*/
if (codec->last_unit_cts == cts) {
/*seeking for systems is done by not releasing the graph until seek is done*/
check_next_unit = 1;
mm_level = GF_CODEC_LEVEL_SEEK;
}
/*set system stream timing*/
else {
codec->last_unit_cts = AU->CTS;
/*we're droping the frame*/
if (scene_locked) codec->nb_droped ++;
mm_level = GF_CODEC_LEVEL_NORMAL;
}
/*lock scene*/
if (!scene_locked) {
scene_locked = codec->odm->subscene ? codec->odm->subscene : codec->odm->parentscene;
if (!gf_mx_try_lock(scene_locked->root_od->term->compositor->mx))
return GF_OK;
/*if terminal is paused, force step-mode: it won't hurt in regular pause/play and ensures proper frame dumping*/
if (codec->odm->term->play_state) codec->odm->term->compositor->step_mode=1;
}
/*current media time for system objects is the clock time, since the media is likely to have random
updates in time*/
codec->odm->current_time = gf_clock_time(codec->ck);
now = gf_term_get_time(codec->odm->term);
e = sdec->ProcessData(sdec, AU->data, AU->dataLength, ch->esd->ESID, au_time, mm_level);
now = gf_term_get_time(codec->odm->term) - now;
GF_LOG(GF_LOG_DEBUG, GF_LOG_CODEC, ("[%s] ODM%d#CH%d at %d decoded AU TS %d in %d ms\n", sdec->module_name, codec->odm->OD->objectDescriptorID, ch->esd->ESID, codec->odm->current_time, AU->CTS, now));
codec_update_stats(codec, AU->dataLength, now);
codec->prev_au_size = AU->dataLength;
/*destroy this AU*/
gf_es_drop_au(ch);
if (e) {
GF_LOG(GF_LOG_ERROR, GF_LOG_CODEC, ("[SysDec] Codec %s AU CTS %d Decode error %s\n", sdec->module_name , cts, gf_error_to_string(e) ));
if (e<0) ch->stream_state = 2;
goto exit;
}
/*in broadcast mode, generate a scene if none is available*/
if (codec->ck->no_time_ctrl) {
GF_Scene *scene = codec->odm->subscene ? codec->odm->subscene : codec->odm->parentscene;
/*static OD resources (embedded in ESD) in broadcast mode, reset time*/
if (codec->flags & GF_ESM_CODEC_IS_STATIC_OD) gf_clock_reset(codec->ck);
/*generate a temp scene if none is in place*/
if (scene->graph_attached != 1) {
Bool prev_dyn = scene->is_dynamic_scene;
scene->is_dynamic_scene = 1;
gf_scene_regenerate(scene);
scene->graph_attached = 2;
scene->is_dynamic_scene = prev_dyn;
GF_LOG(GF_LOG_INFO, GF_LOG_CODEC, ("[Decoder] Got OD resources before scene - generating temporary scene\n"));
}
}
/*if no release restart*/
if (check_next_unit) goto check_unit;
exit:
if (scene_locked) {
gf_mx_v(scene_locked->root_od->term->compositor->mx);
}
return e;
}
static GF_Err MediaCodec_Process(GF_Codec *codec, u32 TimeAvailable)
{
GF_CMUnit *CU;
GF_DBUnit *AU;
GF_Channel *ch, *prev_ch;
u32 mmlevel, cts;
u32 first, entryTime, now, obj_time, unit_size;
GF_MediaDecoder *mdec = (GF_MediaDecoder*)codec->decio;
GF_Err e = GF_OK;
CU = NULL;
/*if video codec muted don't decode (try to saves ressources)
if audio codec muted we dispatch to keep sync in place*/
if (codec->Muted && (codec->type==GF_STREAM_VISUAL) ) return GF_OK;
entryTime = gf_term_get_time(codec->odm->term);
/*fetch next AU in DTS order for this codec*/
Decoder_GetNextAU(codec, &ch, &AU);
/*no active channel return*/
if (!AU || !ch) {
/*if the codec is in EOS state, assume we're done*/
if (codec->Status == GF_ESM_CODEC_EOS) {
/*if codec is reordering, try to flush it*/
if (codec->is_reordering) {
if ( LockCompositionUnit(codec, codec->last_unit_cts+1, &CU, &unit_size) == GF_OUT_OF_MEM)
return GF_OK;
assert( CU );
e = mdec->ProcessData(mdec, NULL, 0, 0, CU->data, &unit_size, 0, 0);
if (e==GF_OK) e = UnlockCompositionUnit(codec, CU, unit_size);
}
gf_term_stop_codec(codec);
if (codec->CB) gf_cm_set_eos(codec->CB);
}
/*if no data, and channel not buffering, ABORT CB buffer (data timeout or EOS not detectable)*/
else if (ch && !ch->BufferOn)
gf_cm_abort_buffering(codec->CB);
return GF_OK;
}
/*get the object time*/
obj_time = gf_clock_time(codec->ck);
/*Media Time for media codecs is updated in the CB*/
if (!codec->CB) {
gf_es_drop_au(ch);
return GF_BAD_PARAM;
}
/*image codecs*/
if (codec->CB->Capacity == 1) {
/*usually only one image is tolerated in the stream, but just in case force reset of CB*/
if (codec->CB->UnitCount && (obj_time>=AU->CTS)) {
gf_mx_p(codec->odm->mx);
codec->CB->output->dataLength = 0;
codec->CB->UnitCount = 0;
gf_mx_v(codec->odm->mx);
}
/*CB is already full*/
if (codec->CB->UnitCount)
return GF_OK;
/*a SHA signature is computed for each AU. This avoids decoding/recompositing when identical (for instance streaming a carousel)*/
{
u8 new_unit_signature[20];
gf_sha1_csum(AU->data, AU->dataLength, new_unit_signature);
if (!memcmp(codec->last_unit_signature, new_unit_signature, sizeof(new_unit_signature))) {
codec->nb_repeted_frames++;
gf_es_drop_au(ch);
return GF_OK;
}
codec->nb_repeted_frames = 0;
memcpy(codec->last_unit_signature, new_unit_signature, sizeof(new_unit_signature));
}
}
/*try to refill the full buffer*/
first = 1;
while (codec->CB->Capacity > codec->CB->UnitCount) {
/*set media processing level*/
mmlevel = GF_CODEC_LEVEL_NORMAL;
/*SEEK: if the last frame had the same TS, we are seeking. Ask the codec to drop*/
if (!ch->skip_sl && codec->last_unit_cts && (codec->last_unit_cts == AU->CTS) && !ch->esd->dependsOnESID) {
mmlevel = GF_CODEC_LEVEL_SEEK;
/*object clock is paused by media control or terminal is paused: exact frame seek*/
if (
#ifndef GPAC_DISABLE_VRML
(codec->ck->mc && codec->ck->mc->paused) ||
#endif
(codec->odm->term->play_state)
) {
gf_cm_rewind_input(codec->CB);
mmlevel = GF_CODEC_LEVEL_NORMAL;
/*force staying in step-mode*/
codec->odm->term->compositor->step_mode=1;
}
}
/*only perform drop in normal playback*/
else if (codec->CB->Status == CB_PLAY) {
/*extremely late, set the level to drop
NOTE: the 100 ms safety gard is to avoid discarding audio*/
if (!ch->skip_sl && (AU->CTS + 100 < obj_time) ) {
mmlevel = GF_CODEC_LEVEL_DROP;
GF_LOG(GF_LOG_DEBUG, GF_LOG_CODEC, ("[%s] ODM%d: frame too late (%d vs %d) - using drop level\n", codec->decio->module_name, codec->odm->OD->objectDescriptorID, AU->CTS, obj_time));
if (ch->resync_drift && (AU->CTS + ch->resync_drift < obj_time)) {
ch->clock->StartTime += (obj_time - AU->CTS);
GF_LOG(GF_LOG_WARNING, GF_LOG_CODEC, ("[%s] ODM%d: decoder too slow on OCR stream - rewinding clock of %d ms\n", codec->decio->module_name, codec->odm->OD->objectDescriptorID, obj_time - AU->CTS));
obj_time = gf_clock_time(codec->ck);
}
}
/*we are late according to the media manager*/
else if (codec->PriorityBoost) {
mmlevel = GF_CODEC_LEVEL_VERY_LATE;
}
/*otherwise we must have an idea of the load in order to set the right level
use the composition buffer for that, only on the first frame*/
else if (first) {
//if the CB is almost empty set to very late
if (codec->CB->UnitCount <= codec->CB->Min+1) {
mmlevel = GF_CODEC_LEVEL_VERY_LATE;
} else if (codec->CB->UnitCount * 2 <= codec->CB->Capacity) {
mmlevel = GF_CODEC_LEVEL_LATE;
}
first = 0;
}
}
/*when using temporal scalability make sure we can decode*/
if (ch->esd->dependsOnESID && (codec->last_unit_dts > AU->DTS)){
// printf("SCALABLE STREAM DEAD!!\n");
goto drop;
}
if (ch->skip_sl) {
if (codec->bytes_per_sec) {
AU->CTS = codec->last_unit_cts + ch->ts_offset + codec->cur_audio_bytes * 1000 / codec->bytes_per_sec;
} else if (codec->fps) {
AU->CTS = codec->last_unit_cts + ch->ts_offset + (u32) (codec->cur_video_frames * 1000 / codec->fps);
}
}
if ( LockCompositionUnit(codec, AU->CTS, &CU, &unit_size) == GF_OUT_OF_MEM)
return GF_OK;
scalable_retry:
now = gf_term_get_time(codec->odm->term);
assert( CU );
if (!CU->data && unit_size)
e = GF_OUT_OF_MEM;
else
e = mdec->ProcessData(mdec, AU->data, AU->dataLength, ch->esd->ESID, CU->data, &unit_size, AU->PaddingBits, mmlevel);
now = gf_term_get_time(codec->odm->term) - now;
if (codec->Status == GF_ESM_CODEC_STOP) return GF_OK;
/*input is too small, resize composition memory*/
switch (e) {
case GF_BUFFER_TOO_SMALL:
/*release but no dispatch*/
UnlockCompositionUnit(codec, CU, 0);
ResizeCompositionBuffer(codec, unit_size);
continue;
/*this happens a lot when using non-MPEG-4 streams (ex: ffmpeg demuxer)*/
case GF_PACKED_FRAMES:
/*in seek don't dispatch any output*/
if (mmlevel == GF_CODEC_LEVEL_SEEK)
unit_size = 0;
e = UnlockCompositionUnit(codec, CU, unit_size);
GF_LOG(GF_LOG_DEBUG, GF_LOG_RTI|GF_LOG_CODEC, ("[%s] ODM%d at %d decoded packed frame TS %d in %d ms\n", codec->decio->module_name, codec->odm->OD->objectDescriptorID, gf_clock_real_time(ch->clock), AU->CTS, now));
if (ch->skip_sl) {
if (codec->bytes_per_sec) {
codec->cur_audio_bytes += unit_size;
} else if (codec->fps && unit_size) {
codec->cur_video_frames += 1;
}
} else {
u32 deltaTS = 0;
if (codec->bytes_per_sec) {
deltaTS = unit_size * 1000 / codec->bytes_per_sec;
} /*else if (0 && codec->fps && unit_size) {
deltaTS = (u32) (1000.0f / codec->fps);
} */else {
deltaTS = (AU->DTS - codec->last_unit_dts);
}
AU->CTS += deltaTS;
}
codec_update_stats(codec, 0, now);
continue;
/*for all cases below, don't release the composition buffer until we are sure we are not
processing a scalable stream*/
case GF_OK:
if (unit_size) {
GF_LOG(GF_LOG_DEBUG, GF_LOG_RTI|GF_LOG_CODEC, ("[%s] ODM%d at %d decoded frame TS %d in %d ms (DTS %d) - %d in CB\n", codec->decio->module_name, codec->odm->OD->objectDescriptorID, gf_clock_real_time(ch->clock), AU->CTS, now, AU->DTS, codec->CB->UnitCount + 1));
}
/*if no size the decoder is not using the composition memory - if the object is in intitial buffering resume it!!*/
else if (codec->CB->Status == CB_BUFFER) {
gf_cm_abort_buffering(codec->CB);
}
codec_update_stats(codec, AU->dataLength, now);
if (ch->skip_sl) {
if (codec->bytes_per_sec) {
codec->cur_audio_bytes += unit_size;
while (codec->cur_audio_bytes>codec->bytes_per_sec) {
codec->cur_audio_bytes -= codec->bytes_per_sec;
codec->last_unit_cts += 1000;
}
} else if (codec->fps && unit_size) {
codec->cur_video_frames += 1;
}
}
#ifndef GPAC_DISABLE_LOGS
if (codec->odm->flags & GF_ODM_PREFETCH) {
GF_LOG(GF_LOG_INFO, GF_LOG_CODEC, ("[%s] ODM%d At %d decoding frame TS %d in prefetch mode\n", codec->decio->module_name, codec->odm->OD->objectDescriptorID, gf_clock_real_time(ch->clock) ));
}
#endif
break;
default:
unit_size = 0;
/*error - if the object is in intitial buffering resume it!!*/
gf_cm_abort_buffering(codec->CB);
GF_LOG(GF_LOG_DEBUG, GF_LOG_CODEC, ("[%s] ODM%d At %d (frame TS %d - %d ms ): decoded error %s\n", codec->decio->module_name, codec->odm->OD->objectDescriptorID, gf_clock_real_time(ch->clock), AU->CTS, now, gf_error_to_string(e) ));
e = GF_OK;
break;
}
codec->last_unit_dts = AU->DTS;
/*remember base layer timing*/
if (!ch->esd->dependsOnESID && !ch->skip_sl) codec->last_unit_cts = AU->CTS;
drop:
/*store current CTS*/
cts = AU->CTS;
prev_ch = ch;
gf_es_drop_au(ch);
AU = NULL;
if (e) {
UnlockCompositionUnit(codec, CU, unit_size);
return e;
}
Decoder_GetNextAU(codec, &ch, &AU);
/*same CTS: same output, likely scalable stream so don't release the CB*/
if (AU && (AU->CTS == cts) && (ch !=prev_ch) ) {
unit_size = codec->CB->UnitSize;
goto scalable_retry;
}
/*in seek don't dispatch any output*/
if (mmlevel == GF_CODEC_LEVEL_SEEK)
unit_size = 0;
UnlockCompositionUnit(codec, CU, unit_size);
if (!ch || !AU) return GF_OK;
/*escape from decoding loop only if above critical limit - this is to avoid starvation on audio*/
if (!ch->esd->dependsOnESID && (codec->CB->UnitCount > codec->CB->Min)) {
now = gf_term_get_time(codec->odm->term);
if (now - entryTime >= TimeAvailable) {
return GF_OK;
}
}
Decoder_GetNextAU(codec, &ch, &AU);
if (!ch || !AU) return GF_OK;
}
return GF_OK;
}
static void MediaDecoder_GetNextAU(GF_Codec *codec, GF_Channel **activeChannel, GF_DBUnit **nextAU)
{
GF_Channel *ch;
GF_DBUnit *AU;
u32 count, minDTS, i;
count = gf_list_count(codec->inChannels);
*nextAU = NULL;
*activeChannel = NULL;
if (!count) return;
minDTS = 0;
/*browse from base to top layer*/
for (i=0;i<count;i++) {
ch = (GF_Channel*)gf_list_get(codec->inChannels, i);
if ((codec->type==GF_STREAM_OCR) && ch->IsClockInit) {
/*check duration - we assume that scalable OCR streams are just pure nonsense...*/
if (ch->is_pulling && codec->odm->duration) {
if (gf_clock_time(codec->ck) > codec->odm->duration)
gf_es_on_eos(ch);
}
return;
}
AU = gf_es_get_au(ch);
if (!AU) {
if (! (*activeChannel)) *activeChannel = ch;
continue;
}
/*aggregate all AUs with the same timestamp on the base AU and delete the upper layers)*/
if (! *nextAU) {
GF_LOG(GF_LOG_DEBUG, GF_LOG_CODEC, ("[%s] ODM%d#CH%d AU CTS %d selected as first layer\n", codec->decio->module_name, codec->odm->OD->objectDescriptorID, ch->esd->ESID, AU->CTS));
*nextAU = AU;
*activeChannel = ch;
minDTS = AU->DTS;
} else if (AU->DTS == minDTS) {
GF_DBUnit *baseAU = *nextAU;
assert(baseAU);
baseAU->data = gf_realloc(baseAU->data, baseAU->dataLength + AU->dataLength);
memcpy(baseAU->data + baseAU->dataLength , AU->data, AU->dataLength);
baseAU->dataLength += AU->dataLength;
GF_LOG(GF_LOG_DEBUG, GF_LOG_CODEC, ("[%s] ODM%d#CH%d AU CTS %d reaggregated on base layer %d\n", codec->decio->module_name, codec->odm->OD->objectDescriptorID, ch->esd->ESID, AU->CTS, (*activeChannel)->esd->ESID));
gf_es_drop_au(ch);
} else {
break;
}
}
if (codec->is_reordering && *nextAU && codec->first_frame_dispatched) {
if ((*activeChannel)->esd->slConfig->no_dts_signaling) {
u32 CTS = (*nextAU)->CTS;
/*reordering !!*/
u32 prev_ts_diff;
u32 diff = 0;
if (codec->recomputed_cts && (codec->recomputed_cts > (*nextAU)->CTS)) {
diff = codec->recomputed_cts - CTS;
}
prev_ts_diff = (CTS > codec->last_unit_cts) ? (CTS - codec->last_unit_cts) : (codec->last_unit_cts - CTS);
if (!diff) diff = prev_ts_diff;
else if (prev_ts_diff && (prev_ts_diff < diff) ) diff = prev_ts_diff;
if (!codec->min_au_duration || (diff < codec->min_au_duration))
codec->min_au_duration = diff;
} else {
codec->min_au_duration = 0;
/*FIXME - we're breaking sync (couple of frames delay)*/
(*nextAU)->CTS = (*nextAU)->DTS;
}
}
}