mlt_frame mlt_consumer_rt_frame( mlt_consumer self )
{
// Frame to return
mlt_frame frame = NULL;
// Get the properties
mlt_properties properties = MLT_CONSUMER_PROPERTIES( self );
// Check if the user has requested real time or not
if ( self->real_time > 1 || self->real_time < -1 )
{
// see above
return worker_get_frame( self, properties );
}
else if ( self->real_time == 1 || self->real_time == -1 )
{
int size = 1;
// Is the read ahead running?
if ( self->ahead == 0 )
{
int buffer = mlt_properties_get_int( properties, "buffer" );
int prefill = mlt_properties_get_int( properties, "prefill" );
consumer_read_ahead_start( self );
if ( buffer > 1 )
size = prefill > 0 && prefill < buffer ? prefill : buffer;
}
// Get frame from queue
pthread_mutex_lock( &self->queue_mutex );
while( self->ahead && mlt_deque_count( self->queue ) < size )
pthread_cond_wait( &self->queue_cond, &self->queue_mutex );
frame = mlt_deque_pop_front( self->queue );
pthread_cond_broadcast( &self->queue_cond );
pthread_mutex_unlock( &self->queue_mutex );
}
else // real_time == 0
{
// Get the frame in non real time
frame = mlt_consumer_get_frame( self );
// This isn't true, but from the consumers perspective it is
if ( frame != NULL )
mlt_properties_set_int( MLT_FRAME_PROPERTIES( frame ), "rendered", 1 );
}
return frame;
}
static void *consumer_thread( void *arg )
{
// Identify the arg
consumer_sdl self = arg;
// Get the consumer
mlt_consumer consumer = &self->parent;
// Convenience functionality
int terminate_on_pause = mlt_properties_get_int( MLT_CONSUMER_PROPERTIES( consumer ), "terminate_on_pause" );
int terminated = 0;
// Video thread
pthread_t thread;
// internal intialization
int init_audio = 1;
int init_video = 1;
mlt_frame frame = NULL;
int duration = 0;
int64_t playtime = 0;
struct timespec tm = { 0, 100000 };
// Loop until told not to
while( self->running )
{
// Get a frame from the attached producer
frame = !terminated? mlt_consumer_rt_frame( consumer ) : NULL;
// Check for termination
if ( terminate_on_pause && frame )
terminated = mlt_properties_get_double( MLT_FRAME_PROPERTIES( frame ), "_speed" ) == 0.0;
// Ensure that we have a frame
if ( frame )
{
// Play audio
init_audio = consumer_play_audio( self, frame, init_audio, &duration );
// Determine the start time now
if ( self->playing && init_video )
{
// Create the video thread
pthread_create( &thread, NULL, video_thread, self );
// Video doesn't need to be initialised any more
init_video = 0;
}
// Set playtime for this frame
mlt_properties_set_int( MLT_FRAME_PROPERTIES( frame ), "playtime", playtime );
while ( self->running && mlt_deque_count( self->queue ) > 15 )
nanosleep( &tm, NULL );
// Push this frame to the back of the queue
pthread_mutex_lock( &self->video_mutex );
if ( self->is_purge )
{
mlt_frame_close( frame );
frame = NULL;
self->is_purge = 0;
}
else
{
mlt_deque_push_back( self->queue, frame );
pthread_cond_broadcast( &self->video_cond );
}
pthread_mutex_unlock( &self->video_mutex );
// Calculate the next playtime
playtime += ( duration * 1000 );
}
else if ( terminated )
{
if ( init_video || mlt_deque_count( self->queue ) == 0 )
break;
else
nanosleep( &tm, NULL );
}
}
self->running = 0;
// Unblock sdl_preview
if ( mlt_properties_get_int( MLT_CONSUMER_PROPERTIES( consumer ), "put_mode" ) &&
mlt_properties_get_int( MLT_CONSUMER_PROPERTIES( consumer ), "put_pending" ) )
{
frame = mlt_consumer_get_frame( consumer );
if ( frame )
mlt_frame_close( frame );
frame = NULL;
}
// Kill the video thread
if ( init_video == 0 )
{
pthread_mutex_lock( &self->video_mutex );
pthread_cond_broadcast( &self->video_cond );
pthread_mutex_unlock( &self->video_mutex );
pthread_join( thread, NULL );
}
while( mlt_deque_count( self->queue ) )
mlt_frame_close( mlt_deque_pop_back( self->queue ) );
self->audio_avail = 0;
return NULL;
}
static void *consumer_thread( void *arg )
{
// Identify the arg
consumer_sdl self = arg;
// Get the consumer
mlt_consumer consumer = &self->parent;
// Get the properties
mlt_properties properties = MLT_CONSUMER_PROPERTIES( consumer );
// internal intialization
mlt_frame frame = NULL;
int last_position = -1;
int eos = 0;
int eos_threshold = 20;
if ( self->play )
eos_threshold = eos_threshold + mlt_properties_get_int( MLT_CONSUMER_PROPERTIES( self->play ), "buffer" );
// Determine if the application is dealing with the preview
int preview_off = mlt_properties_get_int( properties, "preview_off" );
pthread_mutex_lock( &self->refresh_mutex );
self->refresh_count = 0;
pthread_mutex_unlock( &self->refresh_mutex );
// Loop until told not to
while( self->running )
{
// Get a frame from the attached producer
frame = mlt_consumer_get_frame( consumer );
// Ensure that we have a frame
if ( self->running && frame != NULL )
{
// Get the speed of the frame
double speed = mlt_properties_get_double( MLT_FRAME_PROPERTIES( frame ), "_speed" );
// Lock during the operation
mlt_service_lock( MLT_CONSUMER_SERVICE( consumer ) );
// Get refresh request for the current frame
int refresh = mlt_properties_get_int( properties, "refresh" );
// Decrement refresh and clear changed
mlt_events_block( properties, properties );
mlt_properties_set_int( properties, "refresh", 0 );
mlt_events_unblock( properties, properties );
// Unlock after the operation
mlt_service_unlock( MLT_CONSUMER_SERVICE( consumer ) );
// Set the changed property on this frame for the benefit of still
mlt_properties_set_int( MLT_FRAME_PROPERTIES( frame ), "refresh", refresh );
// Make sure the recipient knows that this frame isn't really rendered
mlt_properties_set_int( MLT_FRAME_PROPERTIES( frame ), "rendered", 0 );
// Optimisation to reduce latency
if ( speed == 1.0 )
{
if ( last_position != -1 && last_position + 1 != mlt_frame_get_position( frame ) )
mlt_consumer_purge( self->play );
last_position = mlt_frame_get_position( frame );
}
else
{
//mlt_consumer_purge( self->play );
last_position = -1;
}
// If we aren't playing normally, then use the still
if ( speed != 1 )
{
mlt_producer producer = MLT_PRODUCER( mlt_service_get_producer( MLT_CONSUMER_SERVICE( consumer ) ) );
mlt_position duration = producer? mlt_producer_get_playtime( producer ) : -1;
int pause = 0;
#ifndef SKIP_WAIT_EOS
if ( self->active == self->play )
{
// Do not interrupt the play consumer near the end
if ( duration - self->last_position > eos_threshold )
{
// Get a new frame at the sought position
mlt_frame_close( frame );
if ( producer )
mlt_producer_seek( producer, self->last_position );
frame = mlt_consumer_get_frame( consumer );
pause = 1;
}
else
{
// Send frame with speed 0 to stop it
if ( frame && !mlt_consumer_is_stopped( self->play ) )
{
mlt_consumer_put_frame( self->play, frame );
frame = NULL;
eos = 1;
}
// Check for end of stream
if ( mlt_consumer_is_stopped( self->play ) )
{
// Stream has ended
mlt_log_verbose( MLT_CONSUMER_SERVICE( consumer ), "END OF STREAM\n" );
pause = 1;
eos = 0; // reset eos indicator
}
else
{
// Prevent a tight busy loop
struct timespec tm = { 0, 100000L }; // 100 usec
nanosleep( &tm, NULL );
}
}
}
#else
pause = self->active == self->play;
#endif
if ( pause )
{
// Start the still consumer
if ( !mlt_consumer_is_stopped( self->play ) )
mlt_consumer_stop( self->play );
self->last_speed = speed;
self->active = self->still;
self->ignore_change = 0;
mlt_consumer_start( self->still );
}
// Send the frame to the active child
if ( frame && !eos )
{
mlt_properties_set_int( MLT_FRAME_PROPERTIES( frame ), "refresh", 1 );
if ( self->active )
mlt_consumer_put_frame( self->active, frame );
}
if ( pause && speed == 0.0 )
{
mlt_events_fire( properties, "consumer-sdl-paused", NULL );
}
}
// Allow a little grace time before switching consumers on speed changes
else if ( self->ignore_change -- > 0 && self->active != NULL && !mlt_consumer_is_stopped( self->active ) )
{
mlt_consumer_put_frame( self->active, frame );
}
// Otherwise use the normal player
else
{
if ( !mlt_consumer_is_stopped( self->still ) )
mlt_consumer_stop( self->still );
if ( mlt_consumer_is_stopped( self->play ) )
{
self->last_speed = speed;
self->active = self->play;
self->ignore_change = 0;
mlt_consumer_start( self->play );
}
if ( self->play )
mlt_consumer_put_frame( self->play, frame );
}
// Copy the rectangle info from the active consumer
if ( self->running && preview_off == 0 && self->active )
{
mlt_properties active = MLT_CONSUMER_PROPERTIES( self->active );
mlt_service_lock( MLT_CONSUMER_SERVICE( consumer ) );
mlt_properties_set_int( properties, "rect_x", mlt_properties_get_int( active, "rect_x" ) );
mlt_properties_set_int( properties, "rect_y", mlt_properties_get_int( active, "rect_y" ) );
mlt_properties_set_int( properties, "rect_w", mlt_properties_get_int( active, "rect_w" ) );
mlt_properties_set_int( properties, "rect_h", mlt_properties_get_int( active, "rect_h" ) );
mlt_service_unlock( MLT_CONSUMER_SERVICE( consumer ) );
}
if ( self->active == self->still )
{
pthread_mutex_lock( &self->refresh_mutex );
if ( self->running && speed == 0 && self->refresh_count <= 0 )
{
mlt_events_fire( properties, "consumer-sdl-paused", NULL );
pthread_cond_wait( &self->refresh_cond, &self->refresh_mutex );
}
self->refresh_count --;
pthread_mutex_unlock( &self->refresh_mutex );
}
}
else
{
if ( frame ) mlt_frame_close( frame );
mlt_consumer_put_frame( self->active, NULL );
self->running = 0;
}
}
if ( self->play ) mlt_consumer_stop( self->play );
if ( self->still ) mlt_consumer_stop( self->still );
return NULL;
}
static void *consumer_read_ahead_thread( void *arg )
{
// The argument is the consumer
mlt_consumer self = arg;
// Get the properties of the consumer
mlt_properties properties = MLT_CONSUMER_PROPERTIES( self );
// Get the width and height
int width = mlt_properties_get_int( properties, "width" );
int height = mlt_properties_get_int( properties, "height" );
// See if video is turned off
int video_off = mlt_properties_get_int( properties, "video_off" );
int preview_off = mlt_properties_get_int( properties, "preview_off" );
int preview_format = mlt_properties_get_int( properties, "preview_format" );
// Get the audio settings
mlt_audio_format afmt = mlt_audio_s16;
const char *format = mlt_properties_get( properties, "mlt_audio_format" );
if ( format )
{
if ( !strcmp( format, "none" ) )
afmt = mlt_audio_none;
else if ( !strcmp( format, "s32" ) )
afmt = mlt_audio_s32;
else if ( !strcmp( format, "s32le" ) )
afmt = mlt_audio_s32le;
else if ( !strcmp( format, "float" ) )
afmt = mlt_audio_float;
else if ( !strcmp( format, "f32le" ) )
afmt = mlt_audio_f32le;
else if ( !strcmp( format, "u8" ) )
afmt = mlt_audio_u8;
}
int counter = 0;
double fps = mlt_properties_get_double( properties, "fps" );
int channels = mlt_properties_get_int( properties, "channels" );
int frequency = mlt_properties_get_int( properties, "frequency" );
int samples = 0;
void *audio = NULL;
// See if audio is turned off
int audio_off = mlt_properties_get_int( properties, "audio_off" );
// Get the maximum size of the buffer
int buffer = mlt_properties_get_int( properties, "buffer" ) + 1;
// General frame variable
mlt_frame frame = NULL;
uint8_t *image = NULL;
// Time structures
struct timeval ante;
// Average time for get_frame and get_image
int count = 0;
int skipped = 0;
int64_t time_process = 0;
int skip_next = 0;
mlt_position pos = 0;
mlt_position start_pos = 0;
mlt_position last_pos = 0;
int frame_duration = mlt_properties_get_int( properties, "frame_duration" );
int drop_max = mlt_properties_get_int( properties, "drop_max" );
if ( preview_off && preview_format != 0 )
self->format = preview_format;
// Get the first frame
frame = mlt_consumer_get_frame( self );
if ( frame )
{
// Get the image of the first frame
if ( !video_off )
{
mlt_events_fire( MLT_CONSUMER_PROPERTIES( self ), "consumer-frame-render", frame, NULL );
mlt_frame_get_image( frame, &image, &self->format, &width, &height, 0 );
}
if ( !audio_off )
{
samples = mlt_sample_calculator( fps, frequency, counter++ );
mlt_frame_get_audio( frame, &audio, &afmt, &frequency, &channels, &samples );
}
// Mark as rendered
mlt_properties_set_int( MLT_FRAME_PROPERTIES( frame ), "rendered", 1 );
last_pos = start_pos = pos = mlt_frame_get_position( frame );
}
// Get the starting time (can ignore the times above)
gettimeofday( &ante, NULL );
// Continue to read ahead
while ( self->ahead )
{
// Put the current frame into the queue
pthread_mutex_lock( &self->queue_mutex );
while( self->ahead && mlt_deque_count( self->queue ) >= buffer )
pthread_cond_wait( &self->queue_cond, &self->queue_mutex );
mlt_deque_push_back( self->queue, frame );
pthread_cond_broadcast( &self->queue_cond );
pthread_mutex_unlock( &self->queue_mutex );
// Get the next frame
frame = mlt_consumer_get_frame( self );
// If there's no frame, we're probably stopped...
if ( frame == NULL )
continue;
pos = mlt_frame_get_position( frame );
// Increment the counter used for averaging processing cost
count ++;
// All non-normal playback frames should be shown
if ( mlt_properties_get_int( MLT_FRAME_PROPERTIES( frame ), "_speed" ) != 1 )
{
#ifdef DEINTERLACE_ON_NOT_NORMAL_SPEED
mlt_properties_set_int( MLT_FRAME_PROPERTIES( frame ), "consumer_deinterlace", 1 );
#endif
// Indicate seeking or trick-play
start_pos = pos;
}
// If skip flag not set or frame-dropping disabled
if ( !skip_next || self->real_time == -1 )
{
if ( !video_off )
{
// Reset width/height - could have been changed by previous mlt_frame_get_image
width = mlt_properties_get_int( properties, "width" );
height = mlt_properties_get_int( properties, "height" );
// Get the image
mlt_events_fire( MLT_CONSUMER_PROPERTIES( self ), "consumer-frame-render", frame, NULL );
mlt_frame_get_image( frame, &image, &self->format, &width, &height, 0 );
}
// Indicate the rendered image is available.
mlt_properties_set_int( MLT_FRAME_PROPERTIES( frame ), "rendered", 1 );
// Reset consecutively-skipped counter
skipped = 0;
}
else // Skip image processing
{
// Increment the number of consecutively-skipped frames
skipped++;
// If too many (1 sec) consecutively-skipped frames
if ( skipped > drop_max )
{
// Reset cost tracker
time_process = 0;
count = 1;
mlt_log_verbose( self, "too many frames dropped - forcing next frame\n" );
}
}
// Always process audio
if ( !audio_off )
{
samples = mlt_sample_calculator( fps, frequency, counter++ );
mlt_frame_get_audio( frame, &audio, &afmt, &frequency, &channels, &samples );
}
// Get the time to process this frame
int64_t time_current = time_difference( &ante );
// If the current time is not suddenly some large amount
if ( time_current < time_process / count * 20 || !time_process || count < 5 )
{
// Accumulate the cost for processing this frame
time_process += time_current;
}
else
{
mlt_log_debug( self, "current %"PRId64" threshold %"PRId64" count %d\n",
time_current, (int64_t) (time_process / count * 20), count );
// Ignore the cost of this frame's time
count--;
}
// Determine if we started, resumed, or seeked
if ( pos != last_pos + 1 )
start_pos = pos;
last_pos = pos;
// Do not skip the first 20% of buffer at start, resume, or seek
if ( pos - start_pos <= buffer / 5 + 1 )
{
// Reset cost tracker
time_process = 0;
count = 1;
}
// Reset skip flag
skip_next = 0;
// Only consider skipping if the buffer level is low (or really small)
if ( mlt_deque_count( self->queue ) <= buffer / 5 + 1 )
{
// Skip next frame if average cost exceeds frame duration.
if ( time_process / count > frame_duration )
skip_next = 1;
if ( skip_next )
mlt_log_debug( self, "avg usec %"PRId64" (%"PRId64"/%d) duration %d\n",
time_process/count, time_process, count, frame_duration);
}
}
// Remove the last frame
mlt_frame_close( frame );
return NULL;
}
static mlt_frame worker_get_frame( mlt_consumer self, mlt_properties properties )
{
// Frame to return
mlt_frame frame = NULL;
double fps = mlt_properties_get_double( properties, "fps" );
int threads = abs( self->real_time );
int buffer = mlt_properties_get_int( properties, "_buffer" );
buffer = buffer > 0 ? buffer : mlt_properties_get_int( properties, "buffer" );
// This is a heuristic to determine a suitable minimum buffer size for the number of threads.
int headroom = 2 + threads * threads;
buffer = buffer < headroom ? headroom : buffer;
// Start worker threads if not already started.
if ( ! self->ahead )
{
int prefill = mlt_properties_get_int( properties, "prefill" );
prefill = prefill > 0 && prefill < buffer ? prefill : buffer;
consumer_work_start( self );
// Fill the work queue.
int i = buffer;
while ( self->ahead && i-- )
{
frame = mlt_consumer_get_frame( self );
if ( frame )
{
pthread_mutex_lock( &self->queue_mutex );
mlt_deque_push_back( self->queue, frame );
pthread_cond_signal( &self->queue_cond );
pthread_mutex_unlock( &self->queue_mutex );
}
}
// Wait for prefill
while ( self->ahead && first_unprocessed_frame( self ) < prefill )
{
pthread_mutex_lock( &self->done_mutex );
pthread_cond_wait( &self->done_cond, &self->done_mutex );
pthread_mutex_unlock( &self->done_mutex );
}
self->process_head = threads;
}
// mlt_log_verbose( MLT_CONSUMER_SERVICE(self), "size %d done count %d work count %d process_head %d\n",
// threads, first_unprocessed_frame( self ), mlt_deque_count( self->queue ), self->process_head );
// Feed the work queue
while ( self->ahead && mlt_deque_count( self->queue ) < buffer )
{
frame = mlt_consumer_get_frame( self );
if ( ! frame )
return frame;
pthread_mutex_lock( &self->queue_mutex );
mlt_deque_push_back( self->queue, frame );
pthread_cond_signal( &self->queue_cond );
pthread_mutex_unlock( &self->queue_mutex );
}
// Wait if not realtime.
mlt_frame head_frame = MLT_FRAME( mlt_deque_peek_front( self->queue ) );
while ( self->ahead && self->real_time < 0 &&
!( head_frame && mlt_properties_get_int( MLT_FRAME_PROPERTIES( head_frame ), "rendered" ) ) )
{
pthread_mutex_lock( &self->done_mutex );
pthread_cond_wait( &self->done_cond, &self->done_mutex );
pthread_mutex_unlock( &self->done_mutex );
}
// Get the frame from the queue.
pthread_mutex_lock( &self->queue_mutex );
frame = mlt_deque_pop_front( self->queue );
pthread_mutex_unlock( &self->queue_mutex );
// Adapt the worker process head to the runtime conditions.
if ( self->real_time > 0 )
{
if ( frame && mlt_properties_get_int( MLT_FRAME_PROPERTIES( frame ), "rendered" ) )
{
self->consecutive_dropped = 0;
if ( self->process_head > threads && self->consecutive_rendered >= self->process_head )
self->process_head--;
else
self->consecutive_rendered++;
}
else
{
self->consecutive_rendered = 0;
if ( self->process_head < buffer - threads && self->consecutive_dropped > threads )
self->process_head++;
else
self->consecutive_dropped++;
}
// mlt_log_verbose( MLT_CONSUMER_SERVICE(self), "dropped %d rendered %d process_head %d\n",
// self->consecutive_dropped, self->consecutive_rendered, self->process_head );
// Check for too many consecutively dropped frames
if ( self->consecutive_dropped > mlt_properties_get_int( properties, "drop_max" ) )
{
int orig_buffer = mlt_properties_get_int( properties, "buffer" );
int prefill = mlt_properties_get_int( properties, "prefill" );
mlt_log_verbose( self, "too many frames dropped - " );
// If using a default low-latency buffer level (SDL) and below the limit
if ( ( orig_buffer == 1 || prefill == 1 ) && buffer < (threads + 1) * 10 )
{
// Auto-scale the buffer to compensate
mlt_log_verbose( self, "increasing buffer to %d\n", buffer + threads );
mlt_properties_set_int( properties, "_buffer", buffer + threads );
self->consecutive_dropped = fps / 2;
}
else
{
// Tell the consumer to render it
mlt_log_verbose( self, "forcing next frame\n" );
mlt_properties_set_int( MLT_FRAME_PROPERTIES( frame ), "rendered", 1 );
self->consecutive_dropped = 0;
}
}
}
return frame;
}