static int get_audio( mlt_frame frame, void **buffer, mlt_audio_format *format, int *frequency, int *channels, int *samples )
{
context cx = mlt_frame_pop_audio( frame );
mlt_frame nested_frame = mlt_frame_pop_audio( frame );
int result = 0;
// if not repeating last frame
if ( mlt_frame_get_position( nested_frame ) != cx->audio_position )
{
double fps = mlt_profile_fps( cx->profile );
if ( mlt_producer_get_fps( cx->self ) < fps )
fps = mlt_producer_get_fps( cx->self );
*samples = mlt_sample_calculator( fps, *frequency, cx->audio_counter++ );
result = mlt_frame_get_audio( nested_frame, buffer, format, frequency, channels, samples );
int size = mlt_audio_format_size( *format, *samples, *channels );
int16_t *new_buffer = mlt_pool_alloc( size );
mlt_frame_set_audio( frame, new_buffer, *format, size, mlt_pool_release );
memcpy( new_buffer, *buffer, size );
*buffer = new_buffer;
cx->audio_position = mlt_frame_get_position( nested_frame );
}
else
{
// otherwise return no samples
*samples = 0;
*buffer = NULL;
}
return result;
}
void AudioEnvelope::loadEnvelope()
{
Q_ASSERT(m_envelope == NULL);
std::cout << "Loading envelope ..." << std::endl;
int samplingRate = m_info->info(0)->samplingRate();
mlt_audio_format format_s16 = mlt_audio_s16;
int channels = 1;
Mlt::Frame *frame;
int64_t position;
int samples;
m_envelope = new int64_t[m_envelopeSize];
m_envelopeMax = 0;
m_envelopeMean = 0;
QTime t;
t.start();
int count = 0;
m_producer->seek(m_offset);
m_producer->set_speed(1.0); // This is necessary, otherwise we don't get any new frames in the 2nd run.
for (int i = 0; i < m_envelopeSize; i++) {
frame = m_producer->get_frame(i);
position = mlt_frame_get_position(frame->get_frame());
samples = mlt_sample_calculator(m_producer->get_fps(), samplingRate, position);
int16_t *data = static_cast<int16_t*>(frame->get_audio(format_s16, samplingRate, channels, samples));
int64_t sum = 0;
for (int k = 0; k < samples; k++) {
sum += fabs(data[k]);
}
m_envelope[i] = sum;
m_envelopeMean += sum;
if (sum > m_envelopeMax) {
m_envelopeMax = sum;
}
// std::cout << position << "|" << m_producer->get_playtime()
// << "-" << m_producer->get_in() << "+" << m_producer->get_out() << " ";
delete frame;
count++;
if (m_length > 0 && count > m_length) {
break;
}
}
m_envelopeMean /= m_envelopeSize;
std::cout << "Calculating the envelope (" << m_envelopeSize << " frames) took "
<< t.elapsed() << " ms." << std::endl;
}
static int filter_get_image( mlt_frame frame, uint8_t **image, mlt_image_format *image_format, int *width, int *height, int writable )
{
int error = 0;
mlt_properties frame_properties = MLT_FRAME_PROPERTIES( frame );
mlt_filter filter = (mlt_filter)mlt_frame_pop_service( frame );
int samples = 0;
int channels = 0;
int frequency = 0;
mlt_audio_format audio_format = mlt_audio_s16;
int16_t* audio = (int16_t*)mlt_properties_get_data( frame_properties, "audio", NULL );
if ( !audio && !preprocess_warned ) {
// This filter depends on the consumer processing the audio before the
// video. If the audio is not preprocessed, this filter will process it.
// If this filter processes the audio, it could cause confusion for the
// consumer if it needs different audio properties.
mlt_log_warning( MLT_FILTER_SERVICE(filter), "Audio not preprocessed. Potential audio distortion.\n" );
preprocess_warned = true;
}
*image_format = mlt_image_rgb24a;
// Get the current image
error = mlt_frame_get_image( frame, image, image_format, width, height, writable );
// Get the audio
if( !error ) {
frequency = mlt_properties_get_int( frame_properties, "audio_frequency" );
if (!frequency) {
frequency = 48000;
}
channels = mlt_properties_get_int( frame_properties, "audio_channels" );
if (!channels) {
channels = 2;
}
samples = mlt_properties_get_int( frame_properties, "audio_samples" );
if (!samples) {
mlt_producer producer = mlt_frame_get_original_producer( frame );
double fps = mlt_producer_get_fps( mlt_producer_cut_parent( producer ) );
samples = mlt_sample_calculator( fps, frequency, mlt_frame_get_position( frame ) );
}
error = mlt_frame_get_audio( frame, (void**)&audio, &audio_format, &frequency, &channels, &samples );
}
// Draw the waveforms
if( !error ) {
QImage qimg( *width, *height, QImage::Format_ARGB32 );
convert_mlt_to_qimage_rgba( *image, &qimg, *width, *height );
draw_waveforms( filter, frame, &qimg, audio, channels, samples );
convert_qimage_to_mlt_rgba( &qimg, *image, *width, *height );
}
return error;
}
static void detect_blip( mlt_frame frame, mlt_position pos, double fps, avsync_stats* stats )
{
int frequency = SAMPLE_FREQ;
int channels = 1;
int samples = mlt_sample_calculator( fps, frequency, pos );
mlt_audio_format format = mlt_audio_float;
float* buffer = NULL;
int error = mlt_frame_get_audio( frame, (void**) &buffer, &format, &frequency, &channels, &samples );
if ( !error && format == mlt_audio_float && buffer != NULL )
{
int i = 0;
for( i = 0; i < samples; i++ )
{
if( !stats->blip_in_progress )
{
if( buffer[i] > BLIP_THRESHOLD || buffer[i] < -BLIP_THRESHOLD )
{
// This sample must start a blip
stats->blip_in_progress = 1;
stats->samples_since_blip = 0;
stats->blip_history[1] = stats->blip_history[0];
stats->blip_history[0] = mlt_sample_calculator_to_now( fps, SAMPLE_FREQ, pos );
stats->blip_history[0] += i;
if( stats->blip_history_count < 2 )
{
stats->blip_history_count++;
}
stats->blip = 1;
}
}
else
{
if( buffer[i] > -BLIP_THRESHOLD && buffer[i] < BLIP_THRESHOLD )
{
if( ++stats->samples_since_blip > frequency / 1000 )
{
// One ms of silence means the blip is over
stats->blip_in_progress = 0;
stats->samples_since_blip = 0;
}
}
else
{
stats->samples_since_blip = 0;
}
}
}
}
}
static int consumer_play_audio( consumer_sdl self, mlt_frame frame, int init_audio, int *duration )
{
// Get the properties of self consumer
mlt_properties properties = self->properties;
mlt_audio_format afmt = mlt_audio_s16;
// Set the preferred params of the test card signal
int channels = mlt_properties_get_int( properties, "channels" );
int dest_channels = channels;
int frequency = mlt_properties_get_int( properties, "frequency" );
static int counter = 0;
int samples = mlt_sample_calculator( mlt_properties_get_double( self->properties, "fps" ), frequency, counter++ );
int16_t *pcm;
int bytes;
mlt_frame_get_audio( frame, (void**) &pcm, &afmt, &frequency, &channels, &samples );
*duration = ( ( samples * 1000 ) / frequency );
pcm += mlt_properties_get_int( properties, "audio_offset" );
if ( mlt_properties_get_int( properties, "audio_off" ) )
{
self->playing = 1;
init_audio = 1;
return init_audio;
}
if ( init_audio == 1 )
{
SDL_AudioSpec request;
SDL_AudioSpec got;
int audio_buffer = mlt_properties_get_int( properties, "audio_buffer" );
// specify audio format
memset( &request, 0, sizeof( SDL_AudioSpec ) );
self->playing = 0;
request.freq = frequency;
request.format = AUDIO_S16SYS;
request.channels = dest_channels;
request.samples = audio_buffer;
request.callback = sdl_fill_audio;
request.userdata = (void *)self;
if ( SDL_OpenAudio( &request, &got ) != 0 )
{
mlt_log_error( MLT_CONSUMER_SERVICE( self ), "SDL failed to open audio: %s\n", SDL_GetError() );
init_audio = 2;
}
else if ( got.size != 0 )
{
SDL_PauseAudio( 0 );
init_audio = 0;
}
}
if ( init_audio == 0 )
{
mlt_properties properties = MLT_FRAME_PROPERTIES( frame );
bytes = samples * dest_channels * sizeof(*pcm);
pthread_mutex_lock( &self->audio_mutex );
while ( self->running && bytes > ( sizeof( self->audio_buffer) - self->audio_avail ) )
pthread_cond_wait( &self->audio_cond, &self->audio_mutex );
if ( self->running )
{
if ( mlt_properties_get_double( properties, "_speed" ) == 1 )
{
if ( channels == dest_channels )
{
memcpy( &self->audio_buffer[ self->audio_avail ], pcm, bytes );
}
else
{
int16_t *dest = (int16_t*) &self->audio_buffer[ self->audio_avail ];
int i = samples + 1;
while ( --i )
{
memcpy( dest, pcm, dest_channels * sizeof(*pcm) );
pcm += channels;
dest += dest_channels;
}
}
}
else
{
memset( &self->audio_buffer[ self->audio_avail ], 0, bytes );
}
self->audio_avail += bytes;
}
pthread_cond_broadcast( &self->audio_cond );
pthread_mutex_unlock( &self->audio_mutex );
}
else
{
self->playing = 1;
}
return init_audio;
}
static void foreach_consumer_put( mlt_consumer consumer, mlt_frame frame )
{
mlt_properties properties = MLT_CONSUMER_PROPERTIES( consumer );
mlt_consumer nested = NULL;
char key[30];
int index = 0;
do {
snprintf( key, sizeof(key), "%d.consumer", index++ );
nested = mlt_properties_get_data( properties, key, NULL );
if ( nested )
{
mlt_properties nested_props = MLT_CONSUMER_PROPERTIES(nested);
double self_fps = mlt_properties_get_double( properties, "fps" );
double nested_fps = mlt_properties_get_double( nested_props, "fps" );
mlt_position nested_pos = mlt_properties_get_position( nested_props, "_multi_position" );
mlt_position self_pos = mlt_frame_get_position( frame );
double self_time = self_pos / self_fps;
double nested_time = nested_pos / nested_fps;
// get the audio for the current frame
uint8_t *buffer = NULL;
mlt_audio_format format = mlt_audio_s16;
int channels = mlt_properties_get_int( properties, "channels" );
int frequency = mlt_properties_get_int( properties, "frequency" );
int current_samples = mlt_sample_calculator( self_fps, frequency, self_pos );
mlt_frame_get_audio( frame, (void**) &buffer, &format, &frequency, &channels, ¤t_samples );
int current_size = mlt_audio_format_size( format, current_samples, channels );
// get any leftover audio
int prev_size = 0;
uint8_t *prev_buffer = mlt_properties_get_data( nested_props, "_multi_audio", &prev_size );
uint8_t *new_buffer = NULL;
if ( prev_size > 0 )
{
new_buffer = mlt_pool_alloc( prev_size + current_size );
memcpy( new_buffer, prev_buffer, prev_size );
memcpy( new_buffer + prev_size, buffer, current_size );
buffer = new_buffer;
}
current_size += prev_size;
current_samples += mlt_properties_get_int( nested_props, "_multi_samples" );
while ( nested_time <= self_time )
{
// put ideal number of samples into cloned frame
int deeply = index > 1 ? 1 : 0;
mlt_frame clone_frame = mlt_frame_clone( frame, deeply );
int nested_samples = mlt_sample_calculator( nested_fps, frequency, nested_pos );
// -10 is an optimization to avoid tiny amounts of leftover samples
nested_samples = nested_samples > current_samples - 10 ? current_samples : nested_samples;
int nested_size = mlt_audio_format_size( format, nested_samples, channels );
if ( nested_size > 0 )
{
prev_buffer = mlt_pool_alloc( nested_size );
memcpy( prev_buffer, buffer, nested_size );
}
else
{
prev_buffer = NULL;
nested_size = 0;
}
mlt_frame_set_audio( clone_frame, prev_buffer, format, nested_size, mlt_pool_release );
mlt_properties_set_int( MLT_FRAME_PROPERTIES(clone_frame), "audio_samples", nested_samples );
mlt_properties_set_int( MLT_FRAME_PROPERTIES(clone_frame), "audio_frequency", frequency );
mlt_properties_set_int( MLT_FRAME_PROPERTIES(clone_frame), "audio_channels", channels );
// chomp the audio
current_samples -= nested_samples;
current_size -= nested_size;
buffer += nested_size;
// send frame to nested consumer
mlt_consumer_put_frame( nested, clone_frame );
mlt_properties_set_position( nested_props, "_multi_position", ++nested_pos );
nested_time = nested_pos / nested_fps;
}
// save any remaining audio
if ( current_size > 0 )
{
prev_buffer = mlt_pool_alloc( current_size );
memcpy( prev_buffer, buffer, current_size );
}
else
{
prev_buffer = NULL;
current_size = 0;
}
mlt_pool_release( new_buffer );
mlt_properties_set_data( nested_props, "_multi_audio", prev_buffer, current_size, mlt_pool_release, NULL );
mlt_properties_set_int( nested_props, "_multi_samples", current_samples );
}
} while ( nested );
}
static void *consumer_read_ahead_thread( void *arg )
{
// The argument is the consumer
mlt_consumer self = arg;
// Get the properties of the consumer
mlt_properties properties = MLT_CONSUMER_PROPERTIES( self );
// Get the width and height
int width = mlt_properties_get_int( properties, "width" );
int height = mlt_properties_get_int( properties, "height" );
// See if video is turned off
int video_off = mlt_properties_get_int( properties, "video_off" );
int preview_off = mlt_properties_get_int( properties, "preview_off" );
int preview_format = mlt_properties_get_int( properties, "preview_format" );
// Get the audio settings
mlt_audio_format afmt = mlt_audio_s16;
const char *format = mlt_properties_get( properties, "mlt_audio_format" );
if ( format )
{
if ( !strcmp( format, "none" ) )
afmt = mlt_audio_none;
else if ( !strcmp( format, "s32" ) )
afmt = mlt_audio_s32;
else if ( !strcmp( format, "s32le" ) )
afmt = mlt_audio_s32le;
else if ( !strcmp( format, "float" ) )
afmt = mlt_audio_float;
else if ( !strcmp( format, "f32le" ) )
afmt = mlt_audio_f32le;
else if ( !strcmp( format, "u8" ) )
afmt = mlt_audio_u8;
}
int counter = 0;
double fps = mlt_properties_get_double( properties, "fps" );
int channels = mlt_properties_get_int( properties, "channels" );
int frequency = mlt_properties_get_int( properties, "frequency" );
int samples = 0;
void *audio = NULL;
// See if audio is turned off
int audio_off = mlt_properties_get_int( properties, "audio_off" );
// Get the maximum size of the buffer
int buffer = mlt_properties_get_int( properties, "buffer" ) + 1;
// General frame variable
mlt_frame frame = NULL;
uint8_t *image = NULL;
// Time structures
struct timeval ante;
// Average time for get_frame and get_image
int count = 0;
int skipped = 0;
int64_t time_process = 0;
int skip_next = 0;
mlt_position pos = 0;
mlt_position start_pos = 0;
mlt_position last_pos = 0;
int frame_duration = mlt_properties_get_int( properties, "frame_duration" );
int drop_max = mlt_properties_get_int( properties, "drop_max" );
if ( preview_off && preview_format != 0 )
self->format = preview_format;
// Get the first frame
frame = mlt_consumer_get_frame( self );
if ( frame )
{
// Get the image of the first frame
if ( !video_off )
{
mlt_events_fire( MLT_CONSUMER_PROPERTIES( self ), "consumer-frame-render", frame, NULL );
mlt_frame_get_image( frame, &image, &self->format, &width, &height, 0 );
}
if ( !audio_off )
{
samples = mlt_sample_calculator( fps, frequency, counter++ );
mlt_frame_get_audio( frame, &audio, &afmt, &frequency, &channels, &samples );
}
// Mark as rendered
mlt_properties_set_int( MLT_FRAME_PROPERTIES( frame ), "rendered", 1 );
last_pos = start_pos = pos = mlt_frame_get_position( frame );
}
// Get the starting time (can ignore the times above)
gettimeofday( &ante, NULL );
// Continue to read ahead
while ( self->ahead )
{
// Put the current frame into the queue
pthread_mutex_lock( &self->queue_mutex );
while( self->ahead && mlt_deque_count( self->queue ) >= buffer )
pthread_cond_wait( &self->queue_cond, &self->queue_mutex );
mlt_deque_push_back( self->queue, frame );
pthread_cond_broadcast( &self->queue_cond );
pthread_mutex_unlock( &self->queue_mutex );
// Get the next frame
frame = mlt_consumer_get_frame( self );
// If there's no frame, we're probably stopped...
if ( frame == NULL )
continue;
pos = mlt_frame_get_position( frame );
// Increment the counter used for averaging processing cost
count ++;
// All non-normal playback frames should be shown
if ( mlt_properties_get_int( MLT_FRAME_PROPERTIES( frame ), "_speed" ) != 1 )
{
#ifdef DEINTERLACE_ON_NOT_NORMAL_SPEED
mlt_properties_set_int( MLT_FRAME_PROPERTIES( frame ), "consumer_deinterlace", 1 );
#endif
// Indicate seeking or trick-play
start_pos = pos;
}
// If skip flag not set or frame-dropping disabled
if ( !skip_next || self->real_time == -1 )
{
if ( !video_off )
{
// Reset width/height - could have been changed by previous mlt_frame_get_image
width = mlt_properties_get_int( properties, "width" );
height = mlt_properties_get_int( properties, "height" );
// Get the image
mlt_events_fire( MLT_CONSUMER_PROPERTIES( self ), "consumer-frame-render", frame, NULL );
mlt_frame_get_image( frame, &image, &self->format, &width, &height, 0 );
}
// Indicate the rendered image is available.
mlt_properties_set_int( MLT_FRAME_PROPERTIES( frame ), "rendered", 1 );
// Reset consecutively-skipped counter
skipped = 0;
}
else // Skip image processing
{
// Increment the number of consecutively-skipped frames
skipped++;
// If too many (1 sec) consecutively-skipped frames
if ( skipped > drop_max )
{
// Reset cost tracker
time_process = 0;
count = 1;
mlt_log_verbose( self, "too many frames dropped - forcing next frame\n" );
}
}
// Always process audio
if ( !audio_off )
{
samples = mlt_sample_calculator( fps, frequency, counter++ );
mlt_frame_get_audio( frame, &audio, &afmt, &frequency, &channels, &samples );
}
// Get the time to process this frame
int64_t time_current = time_difference( &ante );
// If the current time is not suddenly some large amount
if ( time_current < time_process / count * 20 || !time_process || count < 5 )
{
// Accumulate the cost for processing this frame
time_process += time_current;
}
else
{
mlt_log_debug( self, "current %"PRId64" threshold %"PRId64" count %d\n",
time_current, (int64_t) (time_process / count * 20), count );
// Ignore the cost of this frame's time
count--;
}
// Determine if we started, resumed, or seeked
if ( pos != last_pos + 1 )
start_pos = pos;
last_pos = pos;
// Do not skip the first 20% of buffer at start, resume, or seek
if ( pos - start_pos <= buffer / 5 + 1 )
{
// Reset cost tracker
time_process = 0;
count = 1;
}
// Reset skip flag
skip_next = 0;
// Only consider skipping if the buffer level is low (or really small)
if ( mlt_deque_count( self->queue ) <= buffer / 5 + 1 )
{
// Skip next frame if average cost exceeds frame duration.
if ( time_process / count > frame_duration )
skip_next = 1;
if ( skip_next )
mlt_log_debug( self, "avg usec %"PRId64" (%"PRId64"/%d) duration %d\n",
time_process/count, time_process, count, frame_duration);
}
}
// Remove the last frame
mlt_frame_close( frame );
return NULL;
}
unsigned char *mlt_frame_get_waveform( mlt_frame self, int w, int h )
{
int16_t *pcm = NULL;
mlt_properties properties = MLT_FRAME_PROPERTIES( self );
mlt_audio_format format = mlt_audio_s16;
int frequency = 16000;
int channels = 2;
mlt_producer producer = mlt_frame_get_original_producer( self );
double fps = mlt_producer_get_fps( mlt_producer_cut_parent( producer ) );
int samples = mlt_sample_calculator( fps, frequency, mlt_frame_get_position( self ) );
// Increase audio resolution proportional to requested image size
while ( samples < w )
{
frequency += 16000;
samples = mlt_sample_calculator( fps, frequency, mlt_frame_get_position( self ) );
}
// Get the pcm data
mlt_frame_get_audio( self, (void**)&pcm, &format, &frequency, &channels, &samples );
// Make an 8-bit buffer large enough to hold rendering
int size = w * h;
if ( size <= 0 )
return NULL;
unsigned char *bitmap = ( unsigned char* )mlt_pool_alloc( size );
if ( bitmap != NULL )
memset( bitmap, 0, size );
else
return NULL;
mlt_properties_set_data( properties, "waveform", bitmap, size, ( mlt_destructor )mlt_pool_release, NULL );
// Render vertical lines
int16_t *ubound = pcm + samples * channels;
int skip = samples / w;
skip = !skip ? 1 : skip;
unsigned char gray = 0xFF / skip;
int i, j, k;
// Iterate sample stream and along x coordinate
for ( i = 0; pcm < ubound; i++ )
{
// pcm data has channels interleaved
for ( j = 0; j < channels; j++, pcm++ )
{
// Determine sample's magnitude from 2s complement;
int pcm_magnitude = *pcm < 0 ? ~(*pcm) + 1 : *pcm;
// The height of a line is the ratio of the magnitude multiplied by
// the vertical resolution of a single channel
int height = h * pcm_magnitude / channels / 2 / 32768;
// Determine the starting y coordinate - left top, right bottom
int displacement = h * (j * 2 + 1) / channels / 2 - ( *pcm < 0 ? 0 : height );
// Position buffer pointer using y coordinate, stride, and x coordinate
unsigned char *p = bitmap + i / skip + displacement * w;
// Draw vertical line
for ( k = 0; k < height + 1; k++ )
if ( *pcm < 0 )
p[ w * k ] = ( k == 0 ) ? 0xFF : p[ w * k ] + gray;
else
p[ w * k ] = ( k == height ) ? 0xFF : p[ w * k ] + gray;
}
}
return bitmap;
}
static int consumer_play_audio( consumer_sdl self, mlt_frame frame, int init_audio, int *duration )
{
// Get the properties of self consumer
mlt_properties properties = self->properties;
mlt_audio_format afmt = mlt_audio_s16;
// Set the preferred params of the test card signal
int channels = mlt_properties_get_int( properties, "channels" );
int frequency = mlt_properties_get_int( properties, "frequency" );
int scrub = mlt_properties_get_int( properties, "scrub_audio" );
static int counter = 0;
int samples = mlt_sample_calculator( mlt_properties_get_double( self->properties, "fps" ), frequency, counter++ );
int16_t *pcm;
mlt_frame_get_audio( frame, (void**) &pcm, &afmt, &frequency, &channels, &samples );
*duration = ( ( samples * 1000 ) / frequency );
pcm += mlt_properties_get_int( properties, "audio_offset" );
if ( mlt_properties_get_int( properties, "audio_off" ) )
{
self->playing = 1;
init_audio = 1;
return init_audio;
}
if ( init_audio == 1 )
{
SDL_AudioSpec request;
SDL_AudioSpec got;
SDL_AudioDeviceID dev;
int audio_buffer = mlt_properties_get_int( properties, "audio_buffer" );
// specify audio format
memset( &request, 0, sizeof( SDL_AudioSpec ) );
self->playing = 0;
request.freq = frequency;
request.format = AUDIO_S16SYS;
request.channels = mlt_properties_get_int( properties, "channels" );
request.samples = audio_buffer;
request.callback = sdl_fill_audio;
request.userdata = (void *)self;
dev = sdl2_open_audio( &request, &got );
if( dev == 0 )
{
mlt_log_error( MLT_CONSUMER_SERVICE( self ), "SDL failed to open audio\n" );
init_audio = 2;
}
else
{
if( got.channels != request.channels )
{
mlt_log_info( MLT_CONSUMER_SERVICE( self ), "Unable to output %d channels. Change to %d\n", request.channels, got.channels );
}
mlt_log_info( MLT_CONSUMER_SERVICE( self ), "Audio Opened: driver=%s channels=%d frequency=%d\n", SDL_GetCurrentAudioDriver(), got.channels, got.freq );
SDL_PauseAudioDevice( dev, 0 );
init_audio = 0;
self->out_channels = got.channels;
}
}
if ( init_audio == 0 )
{
mlt_properties properties = MLT_FRAME_PROPERTIES( frame );
int samples_copied = 0;
int dst_stride = self->out_channels * sizeof( *pcm );
pthread_mutex_lock( &self->audio_mutex );
while ( self->running && samples_copied < samples )
{
int sample_space = ( sizeof( self->audio_buffer ) - self->audio_avail ) / dst_stride;
while ( self->running && sample_space == 0 )
{
pthread_cond_wait( &self->audio_cond, &self->audio_mutex );
sample_space = ( sizeof( self->audio_buffer ) - self->audio_avail ) / dst_stride;
}
if ( self->running )
{
int samples_to_copy = samples - samples_copied;
if ( samples_to_copy > sample_space )
{
samples_to_copy = sample_space;
}
int dst_bytes = samples_to_copy * dst_stride;
if ( scrub || mlt_properties_get_double( properties, "_speed" ) == 1 )
{
if ( channels == self->out_channels )
{
memcpy( &self->audio_buffer[ self->audio_avail ], pcm, dst_bytes );
pcm += samples_to_copy * channels;
}
else
{
int16_t *dest = (int16_t*) &self->audio_buffer[ self->audio_avail ];
int i = samples_to_copy + 1;
while ( --i )
{
memcpy( dest, pcm, dst_stride );
pcm += channels;
dest += self->out_channels;
}
}
}
else
{
memset( &self->audio_buffer[ self->audio_avail ], 0, dst_bytes );
pcm += samples_to_copy * channels;
}
self->audio_avail += dst_bytes;
samples_copied += samples_to_copy;
}
pthread_cond_broadcast( &self->audio_cond );
}
pthread_mutex_unlock( &self->audio_mutex );
}
else
{
self->playing = 1;
}
return init_audio;
}
HRESULT render( mlt_frame frame )
{
HRESULT result = S_OK;
// Get the audio
double speed = mlt_properties_get_double( MLT_FRAME_PROPERTIES(frame), "_speed" );
if ( speed == 1.0 )
{
mlt_audio_format format = mlt_audio_s16;
int frequency = bmdAudioSampleRate48kHz;
int samples = mlt_sample_calculator( m_fps, frequency, m_count );
int16_t *pcm = 0;
if ( !mlt_frame_get_audio( frame, (void**) &pcm, &format, &frequency, &m_channels, &samples ) )
{
int count = samples;
if ( !m_isPrerolling )
{
uint32_t audioCount = 0;
uint32_t videoCount = 0;
// Check for resync
m_deckLinkOutput->GetBufferedAudioSampleFrameCount( &audioCount );
m_deckLinkOutput->GetBufferedVideoFrameCount( &videoCount );
// Underflow typically occurs during non-normal speed playback.
if ( audioCount < 1 || videoCount < 1 )
{
// Upon switching to normal playback, buffer some frames faster than realtime.
mlt_log_info( &m_consumer, "buffer underrun: audio buf %u video buf %u frames\n", audioCount, videoCount );
m_prerollCounter = 0;
}
// While rebuffering
if ( isBuffering() )
{
// Only append audio to reach the ideal level and not overbuffer.
int ideal = ( m_preroll - 1 ) * bmdAudioSampleRate48kHz / m_fps;
int actual = m_fifo->used / m_channels + audioCount;
int diff = ideal / 2 - actual;
count = diff < 0 ? 0 : diff < count ? diff : count;
}
}
if ( count > 0 )
sample_fifo_append( m_fifo, pcm, count * m_channels );
}
}
// Create video frames while pre-rolling
if ( m_isPrerolling )
{
createFrame();
if ( !m_videoFrame )
{
mlt_log_error( &m_consumer, "failed to create video frame\n" );
return S_FALSE;
}
}
// Get the video
if ( mlt_properties_get_int( MLT_FRAME_PROPERTIES( frame ), "rendered") )
{
mlt_image_format format = mlt_image_yuv422;
uint8_t* image = 0;
uint8_t* buffer = 0;
if ( !mlt_frame_get_image( frame, &image, &format, &m_width, &m_height, 0 ) )
{
m_videoFrame = (IDeckLinkMutableVideoFrame*) mlt_deque_pop_back( m_videoFrameQ );
m_videoFrame->GetBytes( (void**) &buffer );
if ( m_displayMode->GetFieldDominance() == bmdUpperFieldFirst )
// convert lower field first to top field first
swab( image, buffer + m_width * 2, m_width * ( m_height - 1 ) * 2 );
else
swab( image, buffer, m_width * m_height * 2 );
m_deckLinkOutput->ScheduleVideoFrame( m_videoFrame, m_count * m_duration, m_duration, m_timescale );
mlt_deque_push_front( m_videoFrameQ, m_videoFrame );
}
}
else
{
mlt_log_verbose( &m_consumer, "dropped video frame\n" );
}
++m_count;
// Check for end of pre-roll
if ( ++m_prerollCounter > m_preroll && m_isPrerolling )
{
// Start audio and video output
m_deckLinkOutput->EndAudioPreroll();
m_deckLinkOutput->StartScheduledPlayback( 0, m_timescale, 1.0 );
m_isPrerolling = false;
}
return result;
}
static int producer_get_audio( mlt_frame frame, int16_t** buffer, mlt_audio_format* format, int* frequency, int* channels, int* samples )
{
mlt_producer producer = (mlt_producer)mlt_frame_pop_audio( frame );
mlt_properties producer_properties = MLT_PRODUCER_PROPERTIES( producer );
char* sound = mlt_properties_get( producer_properties, "sound" );
double fps = mlt_producer_get_fps( producer );
mlt_position position = mlt_frame_original_position( frame );
int size = 0;
int do_beep = 0;
time_info info;
if( fps == 0 ) fps = 25;
// Correct the returns if necessary
*format = mlt_audio_float;
*frequency = *frequency <= 0 ? 48000 : *frequency;
*channels = *channels <= 0 ? 2 : *channels;
*samples = *samples <= 0 ? mlt_sample_calculator( fps, *frequency, position ) : *samples;
// Allocate the buffer
size = *samples * *channels * sizeof( float );
*buffer = mlt_pool_alloc( size );
mlt_service_lock( MLT_PRODUCER_SERVICE( producer ) );
get_time_info( producer, frame, &info );
// Determine if this should be a tone or silence.
if( strcmp( sound, "none") )
{
if( !strcmp( sound, "2pop" ) )
{
mlt_position out = mlt_properties_get_int( producer_properties, "out" );
mlt_position frames = out - position;
if( frames == ( info.fps * 2 ) )
{
do_beep = 1;
}
}
else if( !strcmp( sound, "frame0" ) )
{
if( info.frames == 0 )
{
do_beep = 1;
}
}
}
if( do_beep )
{
fill_beep( producer_properties, (float*)*buffer, *frequency, *channels, *samples );
}
else
{
// Fill silence.
memset( *buffer, 0, size );
}
mlt_service_unlock( MLT_PRODUCER_SERVICE( producer ) );
// Set the buffer for destruction
mlt_frame_set_audio( frame, *buffer, *format, size, mlt_pool_release );
return 0;
}
