void Synchronizer::NewSegment() {
if(m_output_format->m_audio_enabled) {
AudioLock audiolock(&m_audio_data);
InitAudioSegment(audiolock.get());
}
SharedLock lock(&m_shared_data);
NewSegment(lock.get());
}
void Synchronizer::NewSegment() {
{
AudioLock audiolock(&m_audio_data);
InitAudioSegment(audiolock.get());
}
SharedLock lock(&m_shared_data);
NewSegment(lock.get());
}
void Synchronizer::Init() {
// initialize video
if(m_output_format->m_video_enabled) {
m_max_frames_skipped = (m_output_settings->video_allow_frame_skipping)? (MAX_FRAME_DELAY * m_output_format->m_video_frame_rate + 500000) / 1000000 : 0;
VideoLock videolock(&m_video_data);
videolock->m_last_timestamp = std::numeric_limits<int64_t>::min();
videolock->m_next_timestamp = SINK_TIMESTAMP_ASAP;
}
// initialize audio
if(m_output_format->m_audio_enabled) {
AudioLock audiolock(&m_audio_data);
audiolock->m_fast_resampler.reset(new FastResampler(m_output_format->m_audio_channels, 0.9f));
InitAudioSegment(audiolock.get());
audiolock->m_warn_desync = true;
}
// create sync diagram
if(g_option_syncdiagram) {
m_sync_diagram.reset(new SyncDiagram(4));
m_sync_diagram->SetChannelName(0, SyncDiagram::tr("Video in"));
m_sync_diagram->SetChannelName(1, SyncDiagram::tr("Audio in"));
m_sync_diagram->SetChannelName(2, SyncDiagram::tr("Video out"));
m_sync_diagram->SetChannelName(3, SyncDiagram::tr("Audio out"));
m_sync_diagram->show();
}
// initialize shared data
{
SharedLock lock(&m_shared_data);
if(m_output_format->m_audio_enabled) {
lock->m_partial_audio_frame.Alloc(m_output_format->m_audio_frame_size * m_output_format->m_audio_channels);
lock->m_partial_audio_frame_samples = 0;
}
lock->m_video_pts = 0;
lock->m_audio_samples = 0;
lock->m_time_offset = 0;
InitSegment(lock.get());
lock->m_warn_drop_video = true;
}
// start synchronizer thread
m_should_stop = false;
m_error_occurred = false;
m_thread = std::thread(&Synchronizer::SynchronizerThread, this);
}
void Synchronizer::ReadAudioHole() {
assert(m_audio_encoder != NULL);
AudioLock audiolock(&m_audio_data);
if(audiolock->m_first_timestamp != AV_NOPTS_VALUE) {
audiolock->m_average_drift = 0.0;
if(!audiolock->m_insert_zeros) {
Logger::LogWarning("[Synchronizer::ReadAudioHole] " + Logger::tr("Warning: Received hole in audio stream, inserting silence to keep the audio in sync with the video."));
audiolock->m_insert_zeros = true;
}
}
}
void Synchronizer::ReadAudioHole() {
assert(m_output_format->m_audio_enabled);
AudioLock audiolock(&m_audio_data);
if(audiolock->m_first_timestamp != (int64_t) AV_NOPTS_VALUE) {
audiolock->m_average_drift = 0.0;
if(!audiolock->m_drop_samples || !audiolock->m_insert_samples) {
Logger::LogWarning("[Synchronizer::ReadAudioHole] " + Logger::tr("Warning: Received hole in audio stream, inserting silence to keep the audio in sync with the video."));
audiolock->m_drop_samples = true; // because PulseAudio is weird
audiolock->m_insert_samples = true;
}
}
}
void Synchronizer::ReadAudioSamples(unsigned int channels, unsigned int sample_rate, AVSampleFormat format, unsigned int sample_count, const uint8_t* data, int64_t timestamp) {
assert(m_output_format->m_audio_enabled);
// sanity check
if(sample_count == 0)
return;
// add new block to sync diagram
if(m_sync_diagram != NULL)
m_sync_diagram->AddBlock(1, (double) timestamp * 1.0e-6, (double) timestamp * 1.0e-6 + (double) sample_count / (double) sample_rate, QColor(0, 255, 0));
AudioLock audiolock(&m_audio_data);
// check the timestamp
if(timestamp < audiolock->m_last_timestamp) {
if(timestamp < audiolock->m_last_timestamp - 10000)
Logger::LogWarning("[Synchronizer::ReadAudioSamples] " + Logger::tr("Warning: Received audio samples with non-monotonic timestamp."));
timestamp = audiolock->m_last_timestamp;
}
// update the timestamps
int64_t previous_timestamp;
if(audiolock->m_first_timestamp == (int64_t) AV_NOPTS_VALUE) {
audiolock->m_filtered_timestamp = timestamp;
audiolock->m_first_timestamp = timestamp;
previous_timestamp = timestamp;
} else {
previous_timestamp = audiolock->m_last_timestamp;
}
audiolock->m_last_timestamp = timestamp;
// filter the timestamp
int64_t timestamp_delta = (int64_t) sample_count * (int64_t) 1000000 / (int64_t) sample_rate;
audiolock->m_filtered_timestamp += (timestamp - audiolock->m_filtered_timestamp) / AUDIO_TIMESTAMP_FILTER;
// calculate drift
double current_drift = GetAudioDrift(audiolock.get());
// if there are too many audio samples, drop some of them (unlikely unless you use PulseAudio)
if(current_drift > DRIFT_ERROR_THRESHOLD && !audiolock->m_drop_samples) {
audiolock->m_drop_samples = true;
Logger::LogWarning("[Synchronizer::ReadAudioSamples] " + Logger::tr("Warning: Too many audio samples, dropping samples to keep the audio in sync with the video."));
}
// if there are not enough audio samples, insert zeros
if(current_drift < -DRIFT_ERROR_THRESHOLD && !audiolock->m_insert_samples) {
audiolock->m_insert_samples = true;
Logger::LogWarning("[Synchronizer::ReadAudioSamples] " + Logger::tr("Warning: Not enough audio samples, inserting silence to keep the audio in sync with the video."));
}
// reset filter and recalculate drift if necessary
if(audiolock->m_drop_samples || audiolock->m_insert_samples) {
audiolock->m_filtered_timestamp = timestamp;
current_drift = GetAudioDrift(audiolock.get());
}
// drop samples
if(audiolock->m_drop_samples) {
audiolock->m_drop_samples = false;
// drop samples
int n = (int) round(current_drift * (double) sample_rate);
if(n > 0) {
if(n >= (int) sample_count) {
audiolock->m_drop_samples = true;
return; // drop all samples
}
if(format == AV_SAMPLE_FMT_FLT) {
data += n * channels * sizeof(float);
} else if(format == AV_SAMPLE_FMT_S16) {
data += n * channels * sizeof(int16_t);
} else {
assert(false);
}
sample_count -= n;
}
}
// insert zeros
unsigned int sample_count_out = 0;
if(audiolock->m_insert_samples) {
audiolock->m_insert_samples = false;
// how many samples should be inserted?
int n = (int) round(-current_drift * (double) sample_rate);
if(n > 0) {
// insert zeros
audiolock->m_temp_input_buffer.Alloc(n * m_output_format->m_audio_channels);
std::fill_n(audiolock->m_temp_input_buffer.GetData(), n * m_output_format->m_audio_channels, 0.0f);
sample_count_out = audiolock->m_fast_resampler->Resample((double) sample_rate / (double) m_output_format->m_audio_sample_rate, 1.0,
audiolock->m_temp_input_buffer.GetData(), n, &audiolock->m_temp_output_buffer, sample_count_out);
// recalculate drift
current_drift = GetAudioDrift(audiolock.get(), sample_count_out);
}
}
// increase filtered timestamp
audiolock->m_filtered_timestamp += timestamp_delta;
// do drift correction
// The point of drift correction is to keep video and audio in sync even when the clocks are not running at exactly the same speed.
// This can happen because the sample rate of the sound card is not always 100% accurate. Even a 0.1% error will result in audio that is
// seconds too early or too late at the end of a one hour video. This problem doesn't occur on all computers though (I'm not sure why).
// Another cause of desynchronization is problems/glitches with PulseAudio (e.g. jumps in time when switching between sources).
double drift_correction_dt = fmin((double) (timestamp - previous_timestamp) * 1.0e-6, DRIFT_MAX_BLOCK);
audiolock->m_average_drift = clamp(audiolock->m_average_drift + DRIFT_CORRECTION_I * current_drift * drift_correction_dt, -0.5, 0.5);
if(audiolock->m_average_drift < -0.02 && audiolock->m_warn_desync) {
audiolock->m_warn_desync = false;
Logger::LogWarning("[Synchronizer::ReadAudioSamples] " + Logger::tr("Warning: Audio input is more than 2% too slow!"));
}
if(audiolock->m_average_drift > 0.02 && audiolock->m_warn_desync) {
audiolock->m_warn_desync = false;
Logger::LogWarning("[Synchronizer::ReadAudioSamples] " + Logger::tr("Warning: Audio input is more than 2% too fast!"));
}
double length = (double) sample_count / (double) sample_rate;
double drift_correction = clamp(DRIFT_CORRECTION_P * current_drift + audiolock->m_average_drift, -0.5, 0.5) * fmin(1.0, DRIFT_MAX_BLOCK / length);
//qDebug() << "current_drift" << current_drift << "average_drift" << audiolock->m_average_drift << "drift_correction" << drift_correction;
// convert the samples
const float *data_float = NULL; // to keep GCC happy
if(format == AV_SAMPLE_FMT_FLT) {
if(channels == m_output_format->m_audio_channels) {
data_float = (const float*) data;
} else {
audiolock->m_temp_input_buffer.Alloc(sample_count * m_output_format->m_audio_channels);
data_float = audiolock->m_temp_input_buffer.GetData();
SampleChannelRemap(sample_count, (const float*) data, channels, audiolock->m_temp_input_buffer.GetData(), m_output_format->m_audio_channels);
}
} else if(format == AV_SAMPLE_FMT_S16) {
audiolock->m_temp_input_buffer.Alloc(sample_count * m_output_format->m_audio_channels);
data_float = audiolock->m_temp_input_buffer.GetData();
SampleChannelRemap(sample_count, (const int16_t*) data, channels, audiolock->m_temp_input_buffer.GetData(), m_output_format->m_audio_channels);
} else {
assert(false);
}
// resample
sample_count_out = audiolock->m_fast_resampler->Resample((double) sample_rate / (double) m_output_format->m_audio_sample_rate, 1.0 / (1.0 - drift_correction),
data_float, sample_count, &audiolock->m_temp_output_buffer, sample_count_out);
audiolock->m_samples_written += sample_count_out;
SharedLock lock(&m_shared_data);
// avoid memory problems by limiting the audio buffer size
if(lock->m_audio_buffer.GetSize() / m_output_format->m_audio_channels >= MAX_AUDIO_SAMPLES_BUFFERED) {
if(lock->m_segment_video_started) {
Logger::LogWarning("[Synchronizer::ReadAudioSamples] " + Logger::tr("Warning: Audio buffer overflow, starting new segment to keep the audio in sync with the video "
"(some video and/or audio may be lost). The video input seems to be too slow."));
NewSegment(lock.get());
} else {
// If the video hasn't started yet, it makes more sense to drop the oldest samples.
// Shifting the start time like this isn't completely accurate, but this shouldn't happen often anyway.
// The number of samples dropped is calculated so that the buffer will be 90% full after this.
size_t n = lock->m_audio_buffer.GetSize() / m_output_format->m_audio_channels - MAX_AUDIO_SAMPLES_BUFFERED * 9 / 10;
lock->m_audio_buffer.Pop(n * m_output_format->m_audio_channels);
lock->m_segment_audio_start_time += (int64_t) round((double) n / (double) m_output_format->m_audio_sample_rate * 1.0e6);
}
}
// start audio
if(!lock->m_segment_audio_started) {
lock->m_segment_audio_started = true;
lock->m_segment_audio_start_time = timestamp;
lock->m_segment_audio_stop_time = timestamp;
}
// store the samples
lock->m_audio_buffer.Push(audiolock->m_temp_output_buffer.GetData(), sample_count_out * m_output_format->m_audio_channels);
// increase segment stop time
double new_sample_length = (double) (lock->m_segment_audio_samples_read + lock->m_audio_buffer.GetSize() / m_output_format->m_audio_channels) / (double) m_output_format->m_audio_sample_rate;
lock->m_segment_audio_stop_time = lock->m_segment_audio_start_time + (int64_t) round(new_sample_length * 1.0e6);
}
void Synchronizer::Init() {
// initialize video
if(m_video_encoder != NULL) {
m_video_width = m_video_encoder->GetWidth();
m_video_height = m_video_encoder->GetHeight();
m_video_frame_rate = m_video_encoder->GetFrameRate();
m_video_max_frames_skipped = (m_allow_frame_skipping)? (MAX_FRAME_DELAY * m_video_frame_rate + 500000) / 1000000 : 0;
}
// initialize audio
if(m_audio_encoder != NULL) {
m_audio_sample_rate = m_audio_encoder->GetSampleRate();
m_audio_channels = 2; //TODO// never larger than AV_NUM_DATA_POINTERS
m_audio_required_frame_samples = m_audio_encoder->GetRequiredFrameSamples();
m_audio_required_sample_format = m_audio_encoder->GetRequiredSampleFormat();
switch(m_audio_required_sample_format) {
case AV_SAMPLE_FMT_S16:
#if SSR_USE_AVUTIL_PLANAR_SAMPLE_FMT
case AV_SAMPLE_FMT_S16P:
#endif
m_audio_required_sample_size = m_audio_channels * 2; break;
case AV_SAMPLE_FMT_FLT:
#if SSR_USE_AVUTIL_PLANAR_SAMPLE_FMT
case AV_SAMPLE_FMT_FLTP:
#endif
m_audio_required_sample_size = m_audio_channels * 4; break;
default: assert(false); break;
}
}
// create sync diagram
if(g_option_syncdiagram) {
m_sync_diagram.reset(new SyncDiagram(4));
m_sync_diagram->SetChannelName(0, SyncDiagram::tr("Video in"));
m_sync_diagram->SetChannelName(1, SyncDiagram::tr("Audio in"));
m_sync_diagram->SetChannelName(2, SyncDiagram::tr("Video out"));
m_sync_diagram->SetChannelName(3, SyncDiagram::tr("Audio out"));
m_sync_diagram->show();
}
// initialize video data
{
VideoLock videolock(&m_video_data);
videolock->m_last_timestamp = std::numeric_limits<int64_t>::min();
videolock->m_next_timestamp = SINK_TIMESTAMP_ASAP;
}
// initialize audio data
{
AudioLock audiolock(&m_audio_data);
audiolock->m_fast_resampler.reset(new FastResampler(m_audio_channels, 0.9f));
InitAudioSegment(audiolock.get());
audiolock->m_warn_desync = true;
}
// initialize shared data
{
SharedLock lock(&m_shared_data);
if(m_audio_encoder != NULL) {
lock->m_partial_audio_frame.Alloc(m_audio_required_frame_samples * m_audio_channels);
lock->m_partial_audio_frame_samples = 0;
}
lock->m_video_pts = 0;
lock->m_audio_samples = 0;
lock->m_time_offset = 0;
InitSegment(lock.get());
lock->m_warn_drop_video = true;
}
// start synchronizer thread
m_should_stop = false;
m_error_occurred = false;
m_thread = std::thread(&Synchronizer::SynchronizerThread, this);
}
