void createFrame()
{
m_videoFrame = 0;
// Generate a DeckLink video frame
if ( S_OK != m_deckLinkOutput->CreateVideoFrame( m_width, m_height,
m_width * 2, bmdFormat8BitYUV, bmdFrameFlagDefault, &m_videoFrame ) )
{
mlt_log_verbose( &m_consumer, "Failed to create video frame\n" );
stop();
return;
}
// Make the first line black for field order correction.
uint8_t *buffer = 0;
if ( S_OK == m_videoFrame->GetBytes( (void**) &buffer ) && buffer )
{
for ( int i = 0; i < m_width; i++ )
{
*buffer++ = 128;
*buffer++ = 16;
}
}
mlt_log_debug( &m_consumer, "created video frame\n" );
mlt_deque_push_back( m_videoFrameQ, m_videoFrame );
}
static int push( mlt_parser self, int multitrack, int track, int position )
{
mlt_properties properties = mlt_parser_properties( self );
mlt_deque stack = mlt_properties_get_data( properties, "stack", NULL );
track_info *info = malloc( sizeof( track_info ) );
info->multitrack = multitrack;
info->track = track;
info->position = position;
info->length = 0;
info->offset = 0;
return mlt_deque_push_back( stack, info );
}
void process_queue( mlt_deque data_queue, mlt_frame frame, mlt_filter filter )
{
if ( data_queue != NULL )
{
// Create a new queue for those that we can't handle
mlt_deque temp_queue = mlt_deque_init( );
// Iterate through each entry on the queue
while ( mlt_deque_peek_front( data_queue ) != NULL )
{
// Get the data feed
mlt_properties feed = mlt_deque_pop_front( data_queue );
if ( mlt_properties_get( MLT_FILTER_PROPERTIES( filter ), "debug" ) != NULL )
mlt_properties_debug( feed, mlt_properties_get( MLT_FILTER_PROPERTIES( filter ), "debug" ), stderr );
// Process the data feed...
if ( process_feed( feed, filter, frame ) == 0 )
mlt_properties_close( feed );
else
mlt_deque_push_back( temp_queue, feed );
}
// Now put the unprocessed feeds back on the stack
while ( mlt_deque_peek_front( temp_queue ) )
{
// Get the data feed
mlt_properties feed = mlt_deque_pop_front( temp_queue );
// Put it back on the data queue
mlt_deque_push_back( data_queue, feed );
}
// Close the temporary queue
mlt_deque_close( temp_queue );
}
}
static void vdpau_release_buffer( AVCodecContext *codec_context, AVFrame *frame )
{
producer_avformat self = codec_context->opaque;
if ( self->vdpau )
{
struct vdpau_render_state *render = (struct vdpau_render_state*) frame->data[0];
mlt_log_debug( MLT_PRODUCER_SERVICE(self->parent), "vdpau_release_buffer (%x)\n", render->surface );
int i;
render->state &= ~FF_VDPAU_STATE_USED_FOR_REFERENCE;
for ( i = 0; i < 4; i++ )
frame->data[i] = NULL;
mlt_deque_push_back( self->vdpau->deque, render );
}
}
static int vdpau_decoder_init( producer_avformat self )
{
mlt_log_debug( MLT_PRODUCER_SERVICE(self->parent), "vdpau_decoder_init\n" );
int success = 1;
self->video_codec->opaque = self;
self->video_codec->get_format = vdpau_get_format;
self->video_codec->get_buffer = vdpau_get_buffer;
self->video_codec->release_buffer = vdpau_release_buffer;
self->video_codec->draw_horiz_band = vdpau_draw_horiz;
self->video_codec->slice_flags = SLICE_FLAG_CODED_ORDER | SLICE_FLAG_ALLOW_FIELD;
self->video_codec->pix_fmt = PIX_FMT_VDPAU_H264;
VdpDecoderProfile profile = VDP_DECODER_PROFILE_H264_HIGH;
uint32_t max_references = self->video_codec->refs;
pthread_mutex_lock( &mlt_sdl_mutex );
VdpStatus status = vdp_decoder_create( self->vdpau->device,
profile, self->video_codec->width, self->video_codec->height, max_references, &self->vdpau->decoder );
pthread_mutex_unlock( &mlt_sdl_mutex );
if ( status == VDP_STATUS_OK )
{
int i, n = FFMIN( self->video_codec->refs + 2, MAX_VDPAU_SURFACES );
self->vdpau->deque = mlt_deque_init();
for ( i = 0; i < n; i++ )
{
if ( VDP_STATUS_OK == vdp_surface_create( self->vdpau->device, VDP_CHROMA_TYPE_420,
self->video_codec->width, self->video_codec->height, &self->vdpau->render_states[i].surface ) )
{
mlt_log_debug( MLT_PRODUCER_SERVICE(self->parent), "successfully created VDPAU surface %x\n",
self->vdpau->render_states[i].surface );
mlt_deque_push_back( self->vdpau->deque, &self->vdpau->render_states[i] );
}
else
{
mlt_log_info( MLT_PRODUCER_SERVICE(self->parent), "failed to create VDPAU surface %dx%d\n",
self->video_codec->width, self->video_codec->height );
while ( mlt_deque_count( self->vdpau->deque ) )
{
struct vdpau_render_state *render = mlt_deque_pop_front( self->vdpau->deque );
vdp_surface_destroy( render->surface );
}
mlt_deque_close( self->vdpau->deque );
success = 0;
break;
}
}
if ( self->vdpau )
self->vdpau->b_age = self->vdpau->ip_age[0] = self->vdpau->ip_age[1] = 256*256*256*64; // magic from Avidemux
}
else
{
success = 0;
self->vdpau->decoder = VDP_INVALID_HANDLE;
mlt_log_error( MLT_PRODUCER_SERVICE(self->parent), "VDPAU failed to initialize decoder (%s)\n",
vdp_get_error_string( status ) );
}
return success;
}
static void *consumer_read_ahead_thread( void *arg )
{
// The argument is the consumer
mlt_consumer self = arg;
// Get the properties of the consumer
mlt_properties properties = MLT_CONSUMER_PROPERTIES( self );
// Get the width and height
int width = mlt_properties_get_int( properties, "width" );
int height = mlt_properties_get_int( properties, "height" );
// See if video is turned off
int video_off = mlt_properties_get_int( properties, "video_off" );
int preview_off = mlt_properties_get_int( properties, "preview_off" );
int preview_format = mlt_properties_get_int( properties, "preview_format" );
// Get the audio settings
mlt_audio_format afmt = mlt_audio_s16;
const char *format = mlt_properties_get( properties, "mlt_audio_format" );
if ( format )
{
if ( !strcmp( format, "none" ) )
afmt = mlt_audio_none;
else if ( !strcmp( format, "s32" ) )
afmt = mlt_audio_s32;
else if ( !strcmp( format, "s32le" ) )
afmt = mlt_audio_s32le;
else if ( !strcmp( format, "float" ) )
afmt = mlt_audio_float;
else if ( !strcmp( format, "f32le" ) )
afmt = mlt_audio_f32le;
else if ( !strcmp( format, "u8" ) )
afmt = mlt_audio_u8;
}
int counter = 0;
double fps = mlt_properties_get_double( properties, "fps" );
int channels = mlt_properties_get_int( properties, "channels" );
int frequency = mlt_properties_get_int( properties, "frequency" );
int samples = 0;
void *audio = NULL;
// See if audio is turned off
int audio_off = mlt_properties_get_int( properties, "audio_off" );
// Get the maximum size of the buffer
int buffer = mlt_properties_get_int( properties, "buffer" ) + 1;
// General frame variable
mlt_frame frame = NULL;
uint8_t *image = NULL;
// Time structures
struct timeval ante;
// Average time for get_frame and get_image
int count = 0;
int skipped = 0;
int64_t time_process = 0;
int skip_next = 0;
mlt_position pos = 0;
mlt_position start_pos = 0;
mlt_position last_pos = 0;
int frame_duration = mlt_properties_get_int( properties, "frame_duration" );
int drop_max = mlt_properties_get_int( properties, "drop_max" );
if ( preview_off && preview_format != 0 )
self->format = preview_format;
// Get the first frame
frame = mlt_consumer_get_frame( self );
if ( frame )
{
// Get the image of the first frame
if ( !video_off )
{
mlt_events_fire( MLT_CONSUMER_PROPERTIES( self ), "consumer-frame-render", frame, NULL );
mlt_frame_get_image( frame, &image, &self->format, &width, &height, 0 );
}
if ( !audio_off )
{
samples = mlt_sample_calculator( fps, frequency, counter++ );
mlt_frame_get_audio( frame, &audio, &afmt, &frequency, &channels, &samples );
}
// Mark as rendered
mlt_properties_set_int( MLT_FRAME_PROPERTIES( frame ), "rendered", 1 );
last_pos = start_pos = pos = mlt_frame_get_position( frame );
}
// Get the starting time (can ignore the times above)
gettimeofday( &ante, NULL );
// Continue to read ahead
while ( self->ahead )
{
// Put the current frame into the queue
pthread_mutex_lock( &self->queue_mutex );
while( self->ahead && mlt_deque_count( self->queue ) >= buffer )
pthread_cond_wait( &self->queue_cond, &self->queue_mutex );
mlt_deque_push_back( self->queue, frame );
pthread_cond_broadcast( &self->queue_cond );
pthread_mutex_unlock( &self->queue_mutex );
// Get the next frame
frame = mlt_consumer_get_frame( self );
// If there's no frame, we're probably stopped...
if ( frame == NULL )
continue;
pos = mlt_frame_get_position( frame );
// Increment the counter used for averaging processing cost
count ++;
// All non-normal playback frames should be shown
if ( mlt_properties_get_int( MLT_FRAME_PROPERTIES( frame ), "_speed" ) != 1 )
{
#ifdef DEINTERLACE_ON_NOT_NORMAL_SPEED
mlt_properties_set_int( MLT_FRAME_PROPERTIES( frame ), "consumer_deinterlace", 1 );
#endif
// Indicate seeking or trick-play
start_pos = pos;
}
// If skip flag not set or frame-dropping disabled
if ( !skip_next || self->real_time == -1 )
{
if ( !video_off )
{
// Reset width/height - could have been changed by previous mlt_frame_get_image
width = mlt_properties_get_int( properties, "width" );
height = mlt_properties_get_int( properties, "height" );
// Get the image
mlt_events_fire( MLT_CONSUMER_PROPERTIES( self ), "consumer-frame-render", frame, NULL );
mlt_frame_get_image( frame, &image, &self->format, &width, &height, 0 );
}
// Indicate the rendered image is available.
mlt_properties_set_int( MLT_FRAME_PROPERTIES( frame ), "rendered", 1 );
// Reset consecutively-skipped counter
skipped = 0;
}
else // Skip image processing
{
// Increment the number of consecutively-skipped frames
skipped++;
// If too many (1 sec) consecutively-skipped frames
if ( skipped > drop_max )
{
// Reset cost tracker
time_process = 0;
count = 1;
mlt_log_verbose( self, "too many frames dropped - forcing next frame\n" );
}
}
// Always process audio
if ( !audio_off )
{
samples = mlt_sample_calculator( fps, frequency, counter++ );
mlt_frame_get_audio( frame, &audio, &afmt, &frequency, &channels, &samples );
}
// Get the time to process this frame
int64_t time_current = time_difference( &ante );
// If the current time is not suddenly some large amount
if ( time_current < time_process / count * 20 || !time_process || count < 5 )
{
// Accumulate the cost for processing this frame
time_process += time_current;
}
else
{
mlt_log_debug( self, "current %"PRId64" threshold %"PRId64" count %d\n",
time_current, (int64_t) (time_process / count * 20), count );
// Ignore the cost of this frame's time
count--;
}
// Determine if we started, resumed, or seeked
if ( pos != last_pos + 1 )
start_pos = pos;
last_pos = pos;
// Do not skip the first 20% of buffer at start, resume, or seek
if ( pos - start_pos <= buffer / 5 + 1 )
{
// Reset cost tracker
time_process = 0;
count = 1;
}
// Reset skip flag
skip_next = 0;
// Only consider skipping if the buffer level is low (or really small)
if ( mlt_deque_count( self->queue ) <= buffer / 5 + 1 )
{
// Skip next frame if average cost exceeds frame duration.
if ( time_process / count > frame_duration )
skip_next = 1;
if ( skip_next )
mlt_log_debug( self, "avg usec %"PRId64" (%"PRId64"/%d) duration %d\n",
time_process/count, time_process, count, frame_duration);
}
}
// Remove the last frame
mlt_frame_close( frame );
return NULL;
}
static mlt_frame worker_get_frame( mlt_consumer self, mlt_properties properties )
{
// Frame to return
mlt_frame frame = NULL;
double fps = mlt_properties_get_double( properties, "fps" );
int threads = abs( self->real_time );
int buffer = mlt_properties_get_int( properties, "_buffer" );
buffer = buffer > 0 ? buffer : mlt_properties_get_int( properties, "buffer" );
// This is a heuristic to determine a suitable minimum buffer size for the number of threads.
int headroom = 2 + threads * threads;
buffer = buffer < headroom ? headroom : buffer;
// Start worker threads if not already started.
if ( ! self->ahead )
{
int prefill = mlt_properties_get_int( properties, "prefill" );
prefill = prefill > 0 && prefill < buffer ? prefill : buffer;
consumer_work_start( self );
// Fill the work queue.
int i = buffer;
while ( self->ahead && i-- )
{
frame = mlt_consumer_get_frame( self );
if ( frame )
{
pthread_mutex_lock( &self->queue_mutex );
mlt_deque_push_back( self->queue, frame );
pthread_cond_signal( &self->queue_cond );
pthread_mutex_unlock( &self->queue_mutex );
}
}
// Wait for prefill
while ( self->ahead && first_unprocessed_frame( self ) < prefill )
{
pthread_mutex_lock( &self->done_mutex );
pthread_cond_wait( &self->done_cond, &self->done_mutex );
pthread_mutex_unlock( &self->done_mutex );
}
self->process_head = threads;
}
// mlt_log_verbose( MLT_CONSUMER_SERVICE(self), "size %d done count %d work count %d process_head %d\n",
// threads, first_unprocessed_frame( self ), mlt_deque_count( self->queue ), self->process_head );
// Feed the work queue
while ( self->ahead && mlt_deque_count( self->queue ) < buffer )
{
frame = mlt_consumer_get_frame( self );
if ( ! frame )
return frame;
pthread_mutex_lock( &self->queue_mutex );
mlt_deque_push_back( self->queue, frame );
pthread_cond_signal( &self->queue_cond );
pthread_mutex_unlock( &self->queue_mutex );
}
// Wait if not realtime.
mlt_frame head_frame = MLT_FRAME( mlt_deque_peek_front( self->queue ) );
while ( self->ahead && self->real_time < 0 &&
!( head_frame && mlt_properties_get_int( MLT_FRAME_PROPERTIES( head_frame ), "rendered" ) ) )
{
pthread_mutex_lock( &self->done_mutex );
pthread_cond_wait( &self->done_cond, &self->done_mutex );
pthread_mutex_unlock( &self->done_mutex );
}
// Get the frame from the queue.
pthread_mutex_lock( &self->queue_mutex );
frame = mlt_deque_pop_front( self->queue );
pthread_mutex_unlock( &self->queue_mutex );
// Adapt the worker process head to the runtime conditions.
if ( self->real_time > 0 )
{
if ( frame && mlt_properties_get_int( MLT_FRAME_PROPERTIES( frame ), "rendered" ) )
{
self->consecutive_dropped = 0;
if ( self->process_head > threads && self->consecutive_rendered >= self->process_head )
self->process_head--;
else
self->consecutive_rendered++;
}
else
{
self->consecutive_rendered = 0;
if ( self->process_head < buffer - threads && self->consecutive_dropped > threads )
self->process_head++;
else
self->consecutive_dropped++;
}
// mlt_log_verbose( MLT_CONSUMER_SERVICE(self), "dropped %d rendered %d process_head %d\n",
// self->consecutive_dropped, self->consecutive_rendered, self->process_head );
// Check for too many consecutively dropped frames
if ( self->consecutive_dropped > mlt_properties_get_int( properties, "drop_max" ) )
{
int orig_buffer = mlt_properties_get_int( properties, "buffer" );
int prefill = mlt_properties_get_int( properties, "prefill" );
mlt_log_verbose( self, "too many frames dropped - " );
// If using a default low-latency buffer level (SDL) and below the limit
if ( ( orig_buffer == 1 || prefill == 1 ) && buffer < (threads + 1) * 10 )
{
// Auto-scale the buffer to compensate
mlt_log_verbose( self, "increasing buffer to %d\n", buffer + threads );
mlt_properties_set_int( properties, "_buffer", buffer + threads );
self->consecutive_dropped = fps / 2;
}
else
{
// Tell the consumer to render it
mlt_log_verbose( self, "forcing next frame\n" );
mlt_properties_set_int( MLT_FRAME_PROPERTIES( frame ), "rendered", 1 );
self->consecutive_dropped = 0;
}
}
}
return frame;
}
static void consumer_work_start( mlt_consumer self )
{
int n = abs( self->real_time );
pthread_t *thread = calloc( 1, sizeof( pthread_t ) * n );
// We're running now
self->ahead = 1;
self->threads = thread;
// These keep track of the accelleration of frame dropping or recovery.
self->consecutive_dropped = 0;
self->consecutive_rendered = 0;
// This is the position in the queue from which to look for a frame to process.
// If we always start from the head, then we may likely not complete processing
// before the frame is played out.
self->process_head = 0;
// Create the queues
self->queue = mlt_deque_init();
self->worker_threads = mlt_deque_init();
// Create the mutexes
pthread_mutex_init( &self->queue_mutex, NULL );
pthread_mutex_init( &self->done_mutex, NULL );
// Create the conditions
pthread_cond_init( &self->queue_cond, NULL );
pthread_cond_init( &self->done_cond, NULL );
// Create the read ahead
if ( mlt_properties_get( MLT_CONSUMER_PROPERTIES( self ), "priority" ) )
{
struct sched_param priority;
pthread_attr_t thread_attributes;
priority.sched_priority = mlt_properties_get_int( MLT_CONSUMER_PROPERTIES( self ), "priority" );
pthread_attr_init( &thread_attributes );
pthread_attr_setschedpolicy( &thread_attributes, SCHED_OTHER );
pthread_attr_setschedparam( &thread_attributes, &priority );
pthread_attr_setinheritsched( &thread_attributes, PTHREAD_EXPLICIT_SCHED );
pthread_attr_setscope( &thread_attributes, PTHREAD_SCOPE_SYSTEM );
while ( n-- )
{
if ( pthread_create( thread, &thread_attributes, consumer_worker_thread, self ) < 0 )
if ( pthread_create( thread, NULL, consumer_worker_thread, self ) == 0 )
mlt_deque_push_back( self->worker_threads, thread );
thread++;
}
pthread_attr_destroy( &thread_attributes );
}
else
{
while ( n-- )
{
if ( pthread_create( thread, NULL, consumer_worker_thread, self ) == 0 )
mlt_deque_push_back( self->worker_threads, thread );
thread++;
}
}
self->started = 1;
}
static void *consumer_thread( void *arg )
{
// Identify the arg
consumer_sdl self = arg;
// Get the consumer
mlt_consumer consumer = &self->parent;
// Get the properties
mlt_properties consumer_props = MLT_CONSUMER_PROPERTIES( consumer );
// Video thread
pthread_t thread;
// internal intialization
int init_audio = 1;
int init_video = 1;
mlt_frame frame = NULL;
mlt_properties properties = NULL;
int duration = 0;
int64_t playtime = 0;
struct timespec tm = { 0, 100000 };
// int last_position = -1;
pthread_mutex_lock( &self->refresh_mutex );
self->refresh_count = 0;
pthread_mutex_unlock( &self->refresh_mutex );
// Loop until told not to
while( self->running )
{
// Get a frame from the attached producer
frame = mlt_consumer_rt_frame( consumer );
// Ensure that we have a frame
if ( frame )
{
// Get the frame properties
properties = MLT_FRAME_PROPERTIES( frame );
// Get the speed of the frame
double speed = mlt_properties_get_double( properties, "_speed" );
// Clear refresh
mlt_events_block( consumer_props, consumer_props );
mlt_properties_set_int( consumer_props, "refresh", 0 );
mlt_events_unblock( consumer_props, consumer_props );
// Play audio
init_audio = consumer_play_audio( self, frame, init_audio, &duration );
// Determine the start time now
if ( self->playing && init_video )
{
// Create the video thread
pthread_create( &thread, NULL, video_thread, self );
// Video doesn't need to be initialised any more
init_video = 0;
}
// Set playtime for this frame
mlt_properties_set_int( properties, "playtime", playtime );
while ( self->running && speed != 0 && mlt_deque_count( self->queue ) > 15 )
nanosleep( &tm, NULL );
// Push this frame to the back of the queue
if ( self->running && speed )
{
pthread_mutex_lock( &self->video_mutex );
if ( self->is_purge && speed == 1.0 )
{
mlt_frame_close( frame );
frame = NULL;
self->is_purge = 0;
}
else
{
mlt_deque_push_back( self->queue, frame );
pthread_cond_broadcast( &self->video_cond );
}
pthread_mutex_unlock( &self->video_mutex );
// Calculate the next playtime
playtime += ( duration * 1000 );
}
else if ( self->running )
{
pthread_mutex_lock( &self->refresh_mutex );
consumer_play_video( self, frame );
mlt_frame_close( frame );
frame = NULL;
self->refresh_count --;
if ( self->refresh_count <= 0 )
{
pthread_cond_wait( &self->refresh_cond, &self->refresh_mutex );
}
pthread_mutex_unlock( &self->refresh_mutex );
}
// Optimisation to reduce latency
if ( speed == 1.0 )
{
// TODO: disabled due to misbehavior on parallel-consumer
// if ( last_position != -1 && last_position + 1 != mlt_frame_get_position( frame ) )
// mlt_consumer_purge( consumer );
// last_position = mlt_frame_get_position( frame );
}
else
{
mlt_consumer_purge( consumer );
// last_position = -1;
}
}
}
// Kill the video thread
if ( init_video == 0 )
{
pthread_mutex_lock( &self->video_mutex );
pthread_cond_broadcast( &self->video_cond );
pthread_mutex_unlock( &self->video_mutex );
pthread_join( thread, NULL );
}
if ( frame )
{
// The video thread has cleared out the queue. But the audio was played
// for this frame. So play the video before stopping so the display has
// the option to catch up with the audio.
consumer_play_video( self, frame );
mlt_frame_close( frame );
frame = NULL;
}
self->audio_avail = 0;
return NULL;
}
int mlt_frame_push_audio( mlt_frame self, void *that )
{
return mlt_deque_push_back( self->stack_audio, that );
}
int mlt_frame_push_service( mlt_frame self, void *that )
{
return mlt_deque_push_back( self->stack_image, that );
}
int mlt_frame_push_frame( mlt_frame self, mlt_frame that )
{
return mlt_deque_push_back( self->stack_image, that );
}
int mlt_frame_push_get_image( mlt_frame self, mlt_get_image get_image )
{
return mlt_deque_push_back( self->stack_image, get_image );
}
virtual HRESULT STDMETHODCALLTYPE VideoInputFrameArrived(
IDeckLinkVideoInputFrame* video,
IDeckLinkAudioInputPacket* audio )
{
if ( mlt_properties_get_int( MLT_PRODUCER_PROPERTIES( getProducer() ), "preview" ) &&
mlt_producer_get_speed( getProducer() ) == 0.0 && !mlt_deque_count( m_queue ))
{
pthread_cond_broadcast( &m_condition );
return S_OK;
}
// Create mlt_frame
mlt_frame frame = mlt_frame_init( MLT_PRODUCER_SERVICE( getProducer() ) );
// Copy video
if ( video )
{
if ( !( video->GetFlags() & bmdFrameHasNoInputSource ) )
{
int size = video->GetRowBytes() * ( video->GetHeight() + m_vancLines );
void* image = mlt_pool_alloc( size );
void* buffer = 0;
unsigned char* p = (unsigned char*) image;
int n = size / 2;
\
// Initialize VANC lines to nominal black
while ( --n )
{
*p ++ = 16;
*p ++ = 128;
}
// Capture VANC
if ( m_vancLines > 0 )
{
IDeckLinkVideoFrameAncillary* vanc = 0;
if ( video->GetAncillaryData( &vanc ) == S_OK && vanc )
{
for ( int i = 1; i < m_vancLines + 1; i++ )
{
if ( vanc->GetBufferForVerticalBlankingLine( i, &buffer ) == S_OK )
swab( (char*) buffer, (char*) image + ( i - 1 ) * video->GetRowBytes(), video->GetRowBytes() );
else
mlt_log_debug( getProducer(), "failed capture vanc line %d\n", i );
}
SAFE_RELEASE(vanc);
}
}
// Capture image
video->GetBytes( &buffer );
if ( image && buffer )
{
size = video->GetRowBytes() * video->GetHeight();
swab( (char*) buffer, (char*) image + m_vancLines * video->GetRowBytes(), size );
mlt_frame_set_image( frame, (uint8_t*) image, size, mlt_pool_release );
}
else if ( image )
{
mlt_log_verbose( getProducer(), "no video\n" );
mlt_pool_release( image );
}
}
else
{
mlt_log_verbose( getProducer(), "no signal\n" );
mlt_frame_close( frame );
frame = 0;
}
// Get timecode
IDeckLinkTimecode* timecode = 0;
if ( video->GetTimecode( bmdTimecodeVITC, &timecode ) == S_OK && timecode )
{
DLString timecodeString = 0;
if ( timecode->GetString( &timecodeString ) == S_OK )
{
char* s = getCString( timecodeString );
mlt_properties_set( MLT_FRAME_PROPERTIES( frame ), "meta.attr.vitc.markup", s );
mlt_log_debug( getProducer(), "timecode %s\n", s );
freeCString( s );
}
freeDLString( timecodeString );
SAFE_RELEASE( timecode );
}
}
else
{
mlt_log_verbose( getProducer(), "no video\n" );
mlt_frame_close( frame );
frame = 0;
}
// Copy audio
if ( frame && audio )
{
int channels = mlt_properties_get_int( MLT_PRODUCER_PROPERTIES( getProducer() ), "channels" );
int size = audio->GetSampleFrameCount() * channels * sizeof(int16_t);
mlt_audio_format format = mlt_audio_s16;
void* pcm = mlt_pool_alloc( size );
void* buffer = 0;
audio->GetBytes( &buffer );
if ( buffer )
{
memcpy( pcm, buffer, size );
mlt_frame_set_audio( frame, pcm, format, size, mlt_pool_release );
mlt_properties_set_int( MLT_FRAME_PROPERTIES(frame), "audio_samples", audio->GetSampleFrameCount() );
}
else
{
mlt_log_verbose( getProducer(), "no audio\n" );
mlt_pool_release( pcm );
}
}
else
{
mlt_log_verbose( getProducer(), "no audio\n" );
}
// Put frame in queue
if ( frame )
{
int queueMax = mlt_properties_get_int( MLT_PRODUCER_PROPERTIES( getProducer() ), "buffer" );
pthread_mutex_lock( &m_mutex );
if ( mlt_deque_count( m_queue ) < queueMax )
{
mlt_deque_push_back( m_queue, frame );
pthread_cond_broadcast( &m_condition );
}
else
{
mlt_frame_close( frame );
mlt_properties_set_int( MLT_PRODUCER_PROPERTIES( getProducer() ), "dropped", ++m_dropped );
mlt_log_warning( getProducer(), "frame dropped %d\n", m_dropped );
}
pthread_mutex_unlock( &m_mutex );
}
return S_OK;
}
int mlt_service_get_frame( mlt_service self, mlt_frame_ptr frame, int index )
{
int result = 0;
// Lock the service
mlt_service_lock( self );
// Ensure that the frame is NULL
*frame = NULL;
// Only process if we have a valid service
if ( self != NULL && self->get_frame != NULL )
{
mlt_properties properties = MLT_SERVICE_PROPERTIES( self );
mlt_position in = mlt_properties_get_position( properties, "in" );
mlt_position out = mlt_properties_get_position( properties, "out" );
mlt_position position = mlt_service_identify( self ) == producer_type ? mlt_producer_position( MLT_PRODUCER( self ) ) : -1;
result = self->get_frame( self, frame, index );
if ( result == 0 )
{
mlt_properties_inc_ref( properties );
properties = MLT_FRAME_PROPERTIES( *frame );
if ( in >=0 && out > 0 )
{
mlt_properties_set_position( properties, "in", in );
mlt_properties_set_position( properties, "out", out );
}
mlt_service_apply_filters( self, *frame, 1 );
mlt_deque_push_back( MLT_FRAME_SERVICE_STACK( *frame ), self );
if ( mlt_service_identify( self ) == producer_type &&
mlt_properties_get_int( MLT_SERVICE_PROPERTIES( self ), "_need_previous_next" ) )
{
// Save the new position from self->get_frame
mlt_position new_position = mlt_producer_position( MLT_PRODUCER( self ) );
// Get the preceding frame, unfiltered
mlt_frame previous_frame;
mlt_producer_seek( MLT_PRODUCER(self), position - 1 );
result = self->get_frame( self, &previous_frame, index );
if ( !result )
mlt_properties_set_data( properties, "previous frame",
previous_frame, 0, ( mlt_destructor ) mlt_frame_close, NULL );
// Get the following frame, unfiltered
mlt_frame next_frame;
mlt_producer_seek( MLT_PRODUCER(self), position + 1 );
result = self->get_frame( self, &next_frame, index );
if ( !result )
{
mlt_properties_set_data( properties, "next frame",
next_frame, 0, ( mlt_destructor ) mlt_frame_close, NULL );
}
// Restore the new position
mlt_producer_seek( MLT_PRODUCER(self), new_position );
}
}
}
// Make sure we return a frame
if ( *frame == NULL )
*frame = mlt_frame_init( self );
// Unlock the service
mlt_service_unlock( self );
return result;
}
static void *consumer_thread( void *arg )
{
// Identify the arg
consumer_sdl self = arg;
// Get the consumer
mlt_consumer consumer = &self->parent;
// Convenience functionality
int terminate_on_pause = mlt_properties_get_int( MLT_CONSUMER_PROPERTIES( consumer ), "terminate_on_pause" );
int terminated = 0;
// Video thread
pthread_t thread;
// internal intialization
int init_audio = 1;
int init_video = 1;
mlt_frame frame = NULL;
int duration = 0;
int64_t playtime = 0;
struct timespec tm = { 0, 100000 };
// Loop until told not to
while( self->running )
{
// Get a frame from the attached producer
frame = !terminated? mlt_consumer_rt_frame( consumer ) : NULL;
// Check for termination
if ( terminate_on_pause && frame )
terminated = mlt_properties_get_double( MLT_FRAME_PROPERTIES( frame ), "_speed" ) == 0.0;
// Ensure that we have a frame
if ( frame )
{
// Play audio
init_audio = consumer_play_audio( self, frame, init_audio, &duration );
// Determine the start time now
if ( self->playing && init_video )
{
// Create the video thread
pthread_create( &thread, NULL, video_thread, self );
// Video doesn't need to be initialised any more
init_video = 0;
}
// Set playtime for this frame
mlt_properties_set_int( MLT_FRAME_PROPERTIES( frame ), "playtime", playtime );
while ( self->running && mlt_deque_count( self->queue ) > 15 )
nanosleep( &tm, NULL );
// Push this frame to the back of the queue
pthread_mutex_lock( &self->video_mutex );
if ( self->is_purge )
{
mlt_frame_close( frame );
frame = NULL;
self->is_purge = 0;
}
else
{
mlt_deque_push_back( self->queue, frame );
pthread_cond_broadcast( &self->video_cond );
}
pthread_mutex_unlock( &self->video_mutex );
// Calculate the next playtime
playtime += ( duration * 1000 );
}
else if ( terminated )
{
if ( init_video || mlt_deque_count( self->queue ) == 0 )
break;
else
nanosleep( &tm, NULL );
}
}
self->running = 0;
// Unblock sdl_preview
if ( mlt_properties_get_int( MLT_CONSUMER_PROPERTIES( consumer ), "put_mode" ) &&
mlt_properties_get_int( MLT_CONSUMER_PROPERTIES( consumer ), "put_pending" ) )
{
frame = mlt_consumer_get_frame( consumer );
if ( frame )
mlt_frame_close( frame );
frame = NULL;
}
// Kill the video thread
if ( init_video == 0 )
{
pthread_mutex_lock( &self->video_mutex );
pthread_cond_broadcast( &self->video_cond );
pthread_mutex_unlock( &self->video_mutex );
pthread_join( thread, NULL );
}
while( mlt_deque_count( self->queue ) )
mlt_frame_close( mlt_deque_pop_back( self->queue ) );
self->audio_avail = 0;
return NULL;
}
static int producer_get_frame( mlt_producer parent, mlt_frame_ptr frame, int track )
{
mlt_tractor self = parent->child;
// We only respond to the first track requests
if ( track == 0 && self->producer != NULL )
{
int i = 0;
int done = 0;
mlt_frame temp = NULL;
int count = 0;
int image_count = 0;
// Get the properties of the parent producer
mlt_properties properties = MLT_PRODUCER_PROPERTIES( parent );
// Try to obtain the multitrack associated to the tractor
mlt_multitrack multitrack = mlt_properties_get_data( properties, "multitrack", NULL );
// Or a specific producer
mlt_producer producer = mlt_properties_get_data( properties, "producer", NULL );
// Determine whether this tractor feeds to the consumer or stops here
int global_feed = mlt_properties_get_int( properties, "global_feed" );
// If we don't have one, we're in trouble...
if ( multitrack != NULL )
{
// The output frame will hold the 'global' data feeds (ie: those which are targetted for the final frame)
mlt_deque data_queue = mlt_deque_init( );
// Used to garbage collect all frames
char label[ 30 ];
// Get the id of the tractor
char *id = mlt_properties_get( properties, "_unique_id" );
// Will be used to store the frame properties object
mlt_properties frame_properties = NULL;
// We'll store audio and video frames to use here
mlt_frame audio = NULL;
mlt_frame video = NULL;
mlt_frame first_video = NULL;
// Temporary properties
mlt_properties temp_properties = NULL;
// Get the multitrack's producer
mlt_producer target = MLT_MULTITRACK_PRODUCER( multitrack );
mlt_producer_seek( target, mlt_producer_frame( parent ) );
mlt_producer_set_speed( target, mlt_producer_get_speed( parent ) );
// We will create one frame and attach everything to it
*frame = mlt_frame_init( MLT_PRODUCER_SERVICE( parent ) );
// Get the properties of the frame
frame_properties = MLT_FRAME_PROPERTIES( *frame );
// Loop through each of the tracks we're harvesting
for ( i = 0; !done; i ++ )
{
// Get a frame from the producer
mlt_service_get_frame( self->producer, &temp, i );
// Get the temporary properties
temp_properties = MLT_FRAME_PROPERTIES( temp );
// Pass all unique meta properties from the producer's frame to the new frame
mlt_properties_lock( temp_properties );
int props_count = mlt_properties_count( temp_properties );
int j;
for ( j = 0; j < props_count; j ++ )
{
char *name = mlt_properties_get_name( temp_properties, j );
if ( !strncmp( name, "meta.", 5 ) && !mlt_properties_get( frame_properties, name ) )
mlt_properties_set( frame_properties, name, mlt_properties_get_value( temp_properties, j ) );
}
mlt_properties_unlock( temp_properties );
// Copy the format conversion virtual functions
if ( ! (*frame)->convert_image && temp->convert_image )
(*frame)->convert_image = temp->convert_image;
if ( ! (*frame)->convert_audio && temp->convert_audio )
(*frame)->convert_audio = temp->convert_audio;
// Check for last track
done = mlt_properties_get_int( temp_properties, "last_track" );
// Handle fx only tracks
if ( mlt_properties_get_int( temp_properties, "fx_cut" ) )
{
int hide = ( video == NULL ? 1 : 0 ) | ( audio == NULL ? 2 : 0 );
mlt_properties_set_int( temp_properties, "hide", hide );
}
// We store all frames with a destructor on the output frame
sprintf( label, "_%s_%d", id, count ++ );
mlt_properties_set_data( frame_properties, label, temp, 0, ( mlt_destructor )mlt_frame_close, NULL );
// We want to append all 'final' feeds to the global queue
if ( !done && mlt_properties_get_data( temp_properties, "data_queue", NULL ) != NULL )
{
// Move the contents of this queue on to the output frames data queue
mlt_deque sub_queue = mlt_properties_get_data( MLT_FRAME_PROPERTIES( temp ), "data_queue", NULL );
mlt_deque temp = mlt_deque_init( );
while ( global_feed && mlt_deque_count( sub_queue ) )
{
mlt_properties p = mlt_deque_pop_back( sub_queue );
if ( mlt_properties_get_int( p, "final" ) )
mlt_deque_push_back( data_queue, p );
else
mlt_deque_push_back( temp, p );
}
while( mlt_deque_count( temp ) )
mlt_deque_push_front( sub_queue, mlt_deque_pop_back( temp ) );
mlt_deque_close( temp );
}
// Now do the same with the global queue but without the conditional behaviour
if ( mlt_properties_get_data( temp_properties, "global_queue", NULL ) != NULL )
{
mlt_deque sub_queue = mlt_properties_get_data( MLT_FRAME_PROPERTIES( temp ), "global_queue", NULL );
while ( mlt_deque_count( sub_queue ) )
{
mlt_properties p = mlt_deque_pop_back( sub_queue );
mlt_deque_push_back( data_queue, p );
}
}
// Pick up first video and audio frames
if ( !done && !mlt_frame_is_test_audio( temp ) && !( mlt_properties_get_int( temp_properties, "hide" ) & 2 ) )
{
// Order of frame creation is starting to get problematic
if ( audio != NULL )
{
mlt_deque_push_front( MLT_FRAME_AUDIO_STACK( temp ), producer_get_audio );
mlt_deque_push_front( MLT_FRAME_AUDIO_STACK( temp ), audio );
}
audio = temp;
}
if ( !done && !mlt_frame_is_test_card( temp ) && !( mlt_properties_get_int( temp_properties, "hide" ) & 1 ) )
{
if ( video != NULL )
{
mlt_deque_push_front( MLT_FRAME_IMAGE_STACK( temp ), producer_get_image );
mlt_deque_push_front( MLT_FRAME_IMAGE_STACK( temp ), video );
}
video = temp;
if ( first_video == NULL )
first_video = temp;
mlt_properties_set_int( MLT_FRAME_PROPERTIES( temp ), "image_count", ++ image_count );
image_count = 1;
}
}
// Now stack callbacks
if ( audio != NULL )
{
mlt_frame_push_audio( *frame, audio );
mlt_frame_push_audio( *frame, producer_get_audio );
}
if ( video != NULL )
{
mlt_properties video_properties = MLT_FRAME_PROPERTIES( first_video );
mlt_frame_push_service( *frame, video );
mlt_frame_push_service( *frame, producer_get_image );
if ( global_feed )
mlt_properties_set_data( frame_properties, "data_queue", data_queue, 0, NULL, NULL );
mlt_properties_set_data( video_properties, "global_queue", data_queue, 0, destroy_data_queue, NULL );
mlt_properties_set_int( frame_properties, "width", mlt_properties_get_int( video_properties, "width" ) );
mlt_properties_set_int( frame_properties, "height", mlt_properties_get_int( video_properties, "height" ) );
mlt_properties_pass_list( frame_properties, video_properties, "meta.media.width, meta.media.height" );
mlt_properties_set_int( frame_properties, "progressive", mlt_properties_get_int( video_properties, "progressive" ) );
mlt_properties_set_double( frame_properties, "aspect_ratio", mlt_properties_get_double( video_properties, "aspect_ratio" ) );
mlt_properties_set_int( frame_properties, "image_count", image_count );
mlt_properties_set_data( frame_properties, "_producer", mlt_frame_get_original_producer( first_video ), 0, NULL, NULL );
}
else
{
destroy_data_queue( data_queue );
}
mlt_frame_set_position( *frame, mlt_producer_frame( parent ) );
mlt_properties_set_int( MLT_FRAME_PROPERTIES( *frame ), "test_audio", audio == NULL );
mlt_properties_set_int( MLT_FRAME_PROPERTIES( *frame ), "test_image", video == NULL );
}
else if ( producer != NULL )
{
mlt_producer_seek( producer, mlt_producer_frame( parent ) );
mlt_producer_set_speed( producer, mlt_producer_get_speed( parent ) );
mlt_service_get_frame( self->producer, frame, track );
}
else
{
mlt_log( MLT_PRODUCER_SERVICE( parent ), MLT_LOG_ERROR, "tractor without a multitrack!!\n" );
mlt_service_get_frame( self->producer, frame, track );
}
// Prepare the next frame
mlt_producer_prepare_next( parent );
// Indicate our found status
return 0;
}
else
{
// Generate a test card
*frame = mlt_frame_init( MLT_PRODUCER_SERVICE( parent ) );
return 0;
}
}