unsigned int FastResampler::Resample(double resample_ratio, double drift_ratio, const float* samples_in, unsigned int sample_count_in, TempBuffer<float>* samples_out, unsigned int sample_offset_out) {
// check the resampling ratio
if(resample_ratio < 1.0e-3 || resample_ratio > 1.0e3) {
Logger::LogError("[FastResampler::Resample] " + Logger::tr("Error: Resample ratio is out of range!"));
throw ResamplerException();
}
if(drift_ratio < 1.0e-1 || drift_ratio > 1.0e1) {
Logger::LogError("[FastResampler::Resample] " + Logger::tr("Error: Drift ratio is out of range!"));
throw ResamplerException();
}
// should we flush old samples first?
if((m_resample_ratio != resample_ratio || samples_in == NULL) && m_filter_length != 0) {
// pad with zero samples
unsigned int pad = m_filter_length / 2 * m_channels;
std::fill_n(m_samples_memory.Reserve(pad), pad, 0.0f);
m_samples_memory.Push(pad);
// reserve memory (with some margin since floating-point isn't 100% accurate)
unsigned int available = m_samples_memory.GetSize() / m_channels;
samples_out->Alloc((sample_offset_out + (unsigned int) lrint((double) available / (m_resample_ratio * m_drift_ratio) * 1.001) + 4) * m_channels, (sample_offset_out != 0));
// resample
std::pair<unsigned int, unsigned int> done = ResampleBatch(m_samples_memory.GetData(), available, samples_out->GetData() + sample_offset_out * m_channels);
sample_offset_out += done.second;
}
// is there new input data?
if(samples_in == NULL) {
ResetResamplerState();
return sample_offset_out;
}
// update filter if the resample ratio changes
if(m_resample_ratio != resample_ratio) {
Logger::LogInfo("[FastResampler::Resample] " + Logger::tr("Resample ratio is %1 (was %2).").arg(resample_ratio, 0, 'f', 4).arg(m_resample_ratio, 0, 'f', 4));
m_resample_ratio = resample_ratio;
UpdateFilterCoefficients();
ResetResamplerState();
}
m_drift_ratio = drift_ratio;
// save input samples
m_samples_memory.Push(samples_in, sample_count_in * m_channels);
// resample one batch at a time
for( ; ; ) {
// reserve memory (with some margin since floating-point isn't 100% accurate)
unsigned int available = m_samples_memory.GetSize() / m_channels;
unsigned int batch = std::min(available, m_filter_length * 256); // needs to be limited to avoid some numerical problems
samples_out->Alloc((sample_offset_out + (unsigned int) lrint((double) batch / (m_resample_ratio * m_drift_ratio) * 1.001) + 4) * m_channels, (sample_offset_out != 0));
// resample
std::pair<unsigned int, unsigned int> done = ResampleBatch(m_samples_memory.GetData(), batch, samples_out->GetData() + sample_offset_out * m_channels);
m_samples_memory.Pop(done.first * m_channels);
sample_offset_out += done.second;
// is this the last batch?
if(batch == available)
break;
}
return sample_offset_out;
}
void Synchronizer::ReadAudioSamples(unsigned int channels, unsigned int sample_rate, AVSampleFormat format, unsigned int sample_count, const uint8_t* data, int64_t timestamp) {
assert(m_audio_encoder != NULL);
assert(channels == m_audio_channels); // remixing isn't supported yet
// sanity check
if(sample_count == 0)
return;
// add new block to sync diagram
if(m_sync_diagram != NULL)
m_sync_diagram->AddBlock(1, (double) timestamp * 1.0e-6, (double) timestamp * 1.0e-6 + (double) sample_count / (double) sample_rate, QColor(0, 255, 0));
AudioLock audiolock(&m_audio_data);
// check the timestamp
if(timestamp < audiolock->m_last_timestamp) {
if(timestamp < audiolock->m_last_timestamp - 10000)
Logger::LogWarning("[Synchronizer::ReadAudioSamples] " + Logger::tr("Warning: Received audio samples with non-monotonic timestamp."));
timestamp = audiolock->m_last_timestamp;
}
// update the timestamps
int64_t previous_timestamp;
if(audiolock->m_first_timestamp == AV_NOPTS_VALUE) {
audiolock->m_first_timestamp = timestamp;
previous_timestamp = timestamp;
} else {
previous_timestamp = audiolock->m_last_timestamp;
}
audiolock->m_last_timestamp = timestamp;
// calculate drift
double current_drift = ((double) audiolock->m_samples_written + audiolock->m_fast_resampler->GetOutputLatency()) / (double) m_audio_sample_rate
- (double) (timestamp - audiolock->m_first_timestamp) * 1.0e-6;
// if there are too many audio samples, drop the frame (unlikely)
if(current_drift > DRIFT_ERROR_THRESHOLD) {
Logger::LogWarning("[Synchronizer::ReadAudioSamples] " + Logger::tr("Warning: Too many audio samples, dropping samples to keep the audio in sync with the video."));
return;
}
// if there are not enough audio samples, insert zeros
unsigned int sample_count_out = 0;
if(current_drift < -DRIFT_ERROR_THRESHOLD || audiolock->m_insert_zeros) {
if(!audiolock->m_insert_zeros)
Logger::LogWarning("[Synchronizer::ReadAudioSamples] " + Logger::tr("Warning: Not enough audio samples, inserting silence to keep the audio in sync with the video."));
audiolock->m_insert_zeros = false;
// insert zeros
unsigned int n = std::max(0, (int) round(-current_drift * (double) sample_rate));
audiolock->m_temp_input_buffer.Alloc(n * m_audio_channels);
std::fill_n(audiolock->m_temp_input_buffer.GetData(), n * m_audio_channels, 0.0f);
sample_count_out = audiolock->m_fast_resampler->Resample((double) sample_rate / (double) m_audio_sample_rate, 1.0,
audiolock->m_temp_input_buffer.GetData(), n, &audiolock->m_temp_output_buffer, sample_count_out);
// recalculate drift
current_drift = ((double) (audiolock->m_samples_written + sample_count_out) + audiolock->m_fast_resampler->GetOutputLatency()) / (double) m_audio_sample_rate
- (double) (timestamp - audiolock->m_first_timestamp) * 1.0e-6;
}
// do drift correction
// The point of drift correction is to keep video and audio in sync even when the clocks are not running at exactly the same speed.
// This can happen because the sample rate of the sound card is not always 100% accurate. Even a 0.1% error will result in audio that is
// seconds too early or too late at the end of a one hour video. This problem doesn't occur on all computers though (I'm not sure why).
// Another cause of desynchronization is problems/glitches with PulseAudio (e.g. jumps in time when switching between sources).
double dt = fmin((double) (timestamp - previous_timestamp) * 1.0e-6, DRIFT_MAX_BLOCK);
audiolock->m_average_drift = clamp(audiolock->m_average_drift + DRIFT_CORRECTION_I * current_drift * dt, -0.5, 0.5);
if(audiolock->m_average_drift < -0.02 && audiolock->m_warn_desync) {
audiolock->m_warn_desync = false;
Logger::LogWarning("[Synchronizer::ReadAudioSamples] " + Logger::tr("Warning: Audio input is more than 2% too slow!"));
}
if(audiolock->m_average_drift > 0.02 && audiolock->m_warn_desync) {
audiolock->m_warn_desync = false;
Logger::LogWarning("[Synchronizer::ReadAudioSamples] " + Logger::tr("Warning: Audio input is more than 2% too fast!"));
}
double length = (double) sample_count / (double) sample_rate;
double drift_correction = clamp(DRIFT_CORRECTION_P * current_drift + audiolock->m_average_drift, -0.5, 0.5) * fmin(1.0, DRIFT_MAX_BLOCK / length);
//qDebug() << "current_drift" << current_drift << "average_drift" << audiolock->m_average_drift << "drift_correction" << drift_correction;
// convert the samples
const float *data_float;
if(format == AV_SAMPLE_FMT_FLT) {
data_float = (const float*) data;
} else if(format == AV_SAMPLE_FMT_S16) {
audiolock->m_temp_input_buffer.Alloc(sample_count * m_audio_channels);
data_float = audiolock->m_temp_input_buffer.GetData();
SampleCopy(sample_count * m_audio_channels, (const int16_t*) data, 1, audiolock->m_temp_input_buffer.GetData(), 1);
} else {
Logger::LogError("[Synchronizer::ReadAudioSamples] " + Logger::tr("Error: Audio sample format is not supported!"));
throw ResamplerException();
}
// resample
sample_count_out = audiolock->m_fast_resampler->Resample((double) sample_rate / (double) m_audio_sample_rate, 1.0 / (1.0 - drift_correction),
data_float, sample_count, &audiolock->m_temp_output_buffer, sample_count_out);
audiolock->m_samples_written += sample_count_out;
SharedLock lock(&m_shared_data);
// avoid memory problems by limiting the audio buffer size
if(lock->m_audio_buffer.GetSize() / m_audio_channels >= MAX_AUDIO_SAMPLES_BUFFERED) {
if(lock->m_segment_video_started) {
Logger::LogWarning("[Synchronizer::ReadAudioSamples] " + Logger::tr("Warning: Audio buffer overflow, starting new segment to keep the audio in sync with the video "
"(some video and/or audio may be lost). The video input seems to be too slow."));
NewSegment(lock.get());
} else {
// If the video hasn't started yet, it makes more sense to drop the oldest samples.
// Shifting the start time like this isn't completely accurate, but this shouldn't happen often anyway.
// The number of samples dropped is calculated so that the buffer will be 90% full after this.
size_t n = lock->m_audio_buffer.GetSize() / m_audio_channels - MAX_AUDIO_SAMPLES_BUFFERED * 9 / 10;
lock->m_audio_buffer.Pop(n * m_audio_channels);
lock->m_segment_audio_start_time += (int64_t) round((double) n / (double) m_audio_sample_rate * 1.0e6);
}
}
// start audio
if(!lock->m_segment_audio_started) {
lock->m_segment_audio_started = true;
lock->m_segment_audio_start_time = timestamp;
lock->m_segment_audio_stop_time = timestamp;
}
// store the samples
lock->m_audio_buffer.Push(audiolock->m_temp_output_buffer.GetData(), sample_count_out * m_audio_channels);
// increase segment stop time
double new_sample_length = (double) (lock->m_segment_audio_samples_read + lock->m_audio_buffer.GetSize() / m_audio_channels) / (double) m_audio_sample_rate;
lock->m_segment_audio_stop_time = lock->m_segment_audio_start_time + (int64_t) round(new_sample_length * 1.0e6);
}
