void Synchronizer::FlushBuffers(SharedData* lock) {
if(!lock->m_segment_video_started || !lock->m_segment_audio_started)
return;
// Sometimes long delays between video frames can occur, e.g. when a game is showing a loading screen.
// Not all codecs/players can handle that. It's also a problem for streaming. To fix this, long delays should be avoided by
// duplicating the previous frame a few times when needed. Whenever a video frame is sent to the encoder, it is also copied,
// with reference counting for the actual image to minimize overhead. When there is a gap, duplicate frames are inserted.
// Duplicate frames are always inserted with a timestamp in the past, because we don't want to drop a real frame if it is captured
// right after the duplicate was inserted. MAX_INPUT_LATENCY simulates the latency from the capturing of a frame to the synchronizer,
// i.e. any new frame is assumed to have a timestamp higher than the current time minus MAX_INPUT_LATENCY. The duplicate
// frame will have a timestamp that's one frame earlier than that time, so it will never interfere with the real frame.
// There are two situations where duplicate frames can be inserted:
// (1) The queue is not empty, but there is a gap between frames that is too large.
// (2) The queue is empty and the last timestamp is too long ago (relative to the end of the video segment).
// It is perfectly possible that *both* happen, each possibly multiple times, in just one function call.
int64_t segment_start_time, segment_stop_time;
GetSegmentStartStop(lock, &segment_start_time, &segment_stop_time);
// flush video
//int64_t segment_stop_video_pts = (int64_t) round((double) (lock->m_time_offset + (segment_stop_time - segment_start_time)) * 1.0e-6 * (double) m_video_frame_rate);
int64_t segment_stop_video_pts = (lock->m_time_offset + (segment_stop_time - segment_start_time)) * (int64_t) m_video_frame_rate / (int64_t) 1000000;
int64_t delay_time_per_frame = 1000000 / m_video_frame_rate + 1; // add one to avoid endless accumulation
for( ; ; ) {
// get/predict the timestamp of the next frame
int64_t next_timestamp = (lock->m_video_buffer.empty())? lock->m_segment_video_stop_time - (int64_t) (1000000 / m_video_frame_rate) : lock->m_video_buffer.front()->GetFrame()->pts;
int64_t next_pts = (lock->m_time_offset + (next_timestamp - segment_start_time)) * (int64_t) m_video_frame_rate / (int64_t) 1000000;
// insert delays if needed, up to either the next frame or the segment end
// It doesn't really matter where the delays end up, they will never cause real frames to be dropped, only duplicates.
while(lock->m_segment_video_accumulated_delay >= delay_time_per_frame && lock->m_video_pts < std::min(next_pts, segment_stop_video_pts)) {
lock->m_segment_video_accumulated_delay -= delay_time_per_frame;
lock->m_video_pts += 1;
//Logger::LogInfo("[Synchronizer::FlushBuffers] Delay [" + QString::number(lock->m_video_pts - 1) + "] acc " + QString::number(lock->m_segment_video_accumulated_delay) + ".");
}
// insert duplicate frames if needed, up to either the next frame or the segment end
if(lock->m_last_video_frame_data != NULL) {
while(lock->m_video_pts + m_video_max_frames_skipped < std::min(next_pts, segment_stop_video_pts)) {
// create duplicate frame
std::unique_ptr<AVFrameWrapper> duplicate_frame = CreateVideoFrameYUV(m_video_width, m_video_height, &lock->m_last_video_frame_data);
duplicate_frame->GetFrame()->pts = lock->m_video_pts + m_video_max_frames_skipped;
if(lock->m_sync_diagram != NULL) {
double t = (double) duplicate_frame->GetFrame()->pts / (double) m_video_frame_rate;
lock->m_sync_diagram->AddBlock(2, t, t + 1.0 / (double) m_video_frame_rate, QColor(255, 196, 0));
}
lock->m_segment_video_accumulated_delay = std::max((int64_t) 0, lock->m_segment_video_accumulated_delay - (duplicate_frame->GetFrame()->pts - lock->m_video_pts) * delay_time_per_frame);
lock->m_video_pts = duplicate_frame->GetFrame()->pts + 1;
//Logger::LogInfo("[Synchronizer::FlushBuffers] Encoded video frame [" + QString::number(duplicate_frame->GetFrame()->pts) + "] (duplicate) acc " + QString::number(lock->m_segment_video_accumulated_delay) + ".");
m_video_encoder->AddFrame(std::move(duplicate_frame));
}
}
// if there are no frames, or they are beyond the segment end, stop
if(lock->m_video_buffer.empty() || next_pts >= segment_stop_video_pts)
break;
// get the frame
std::unique_ptr<AVFrameWrapper> frame = std::move(lock->m_video_buffer.front());
lock->m_video_buffer.pop_front();
frame->GetFrame()->pts = next_pts;
lock->m_last_video_frame_data = frame->GetFrameData();
// if the frame is way too early, drop it
if(frame->GetFrame()->pts < lock->m_video_pts - 1) {
//Logger::LogInfo("[Synchronizer::FlushBuffers] Dropped video frame [" + QString::number(frame->GetFrame()->pts) + "] acc " + QString::number(lock->m_segment_video_accumulated_delay) + ".");
continue;
}
// if the frame is just a little too early, move it
if(frame->GetFrame()->pts < lock->m_video_pts)
frame->GetFrame()->pts = lock->m_video_pts;
// if this is the first video frame, always set the pts to zero
if(lock->m_video_pts == 0)
frame->GetFrame()->pts = 0;
if(lock->m_sync_diagram != NULL) {
double t = (double) frame->GetFrame()->pts / (double) m_video_frame_rate;
lock->m_sync_diagram->AddBlock(2, t, t + 1.0 / (double) m_video_frame_rate, QColor(255, 0, 0));
}
// send the frame to the encoder
lock->m_segment_video_accumulated_delay = std::max((int64_t) 0, lock->m_segment_video_accumulated_delay - (frame->GetFrame()->pts - lock->m_video_pts) * delay_time_per_frame);
lock->m_video_pts = frame->GetFrame()->pts + 1;
//Logger::LogInfo("[Synchronizer::FlushBuffers] Encoded video frame [" + QString::number(frame->GetFrame()->pts) + "].");
m_video_encoder->AddFrame(std::move(frame));
}
// flush audio
double sample_length = (double) (segment_stop_time - lock->m_segment_audio_start_time) * 1.0e-6;
int64_t samples_max = (int64_t) ceil(sample_length * (double) m_audio_sample_rate) - lock->m_segment_audio_samples_read;
if(lock->m_audio_buffer.GetSize() > 0) {
// Normally, the correct way to calculate the position of the first sample would be:
// int64_t timestamp = lock->m_segment_audio_start_time + (int64_t) round((double) lock->m_segment_audio_samples_read / (double) m_audio_sample_rate * 1.0e6);
// int64_t pos = (int64_t) round((double) (lock->m_time_offset + (timestamp - segment_start_time)) * 1.0e-6 * (double) m_audio_sample_rate);
// Simplified:
// int64_t pos = (int64_t) round((double) (lock->m_time_offset + (lock->m_segment_audio_start_time - segment_start_time)) * 1.0e-6 * (double) m_audio_sample_rate)
// + lock->m_segment_audio_samples_read;
// The first part of the expression is constant, so it only has to be calculated at the start of the segment. After that the increase in position is always
// equal to the number of samples written. Samples are only dropped at the start of the segment, so actually
// the position doesn't have to be calculated anymore after that, since it is assumed to be equal to lock->m_audio_samples.
if(lock->m_segment_audio_can_drop) {
// calculate the offset of the first sample
int64_t pos = (int64_t) round((double) (lock->m_time_offset + (lock->m_segment_audio_start_time - segment_start_time)) * 1.0e-6 * (double) m_audio_sample_rate)
+ lock->m_segment_audio_samples_read;
// drop samples that are too early
if(pos < lock->m_audio_samples) {
int64_t n = std::min(lock->m_audio_samples - pos, (int64_t) lock->m_audio_buffer.GetSize() / m_audio_sample_size);
lock->m_audio_buffer.Drop(n * m_audio_sample_size);
lock->m_segment_audio_samples_read += n;
}
}
int64_t samples_left = std::min(samples_max, (int64_t) lock->m_audio_buffer.GetSize() / m_audio_sample_size);
if(lock->m_sync_diagram != NULL && samples_left > 0) {
double t = (double) lock->m_audio_samples / (double) m_audio_sample_rate;
lock->m_sync_diagram->AddBlock(3, t, t + (double) samples_left / (double) m_audio_sample_rate, QColor(0, 255, 0));
}
// send the samples to the encoder
while(samples_left > 0) {
lock->m_segment_audio_can_drop = false;
// copy samples until either the partial frame is full or there are no samples left
int64_t n = std::min((int64_t) (m_audio_required_frame_size - lock->m_partial_audio_frame_samples), samples_left);
lock->m_audio_buffer.Read(lock->m_partial_audio_frame.data() + lock->m_partial_audio_frame_samples * m_audio_sample_size, n * m_audio_sample_size);
lock->m_segment_audio_samples_read += n;
lock->m_partial_audio_frame_samples += n;
lock->m_audio_samples += n;
samples_left -= n;
// is the partial frame full?
if(lock->m_partial_audio_frame_samples == m_audio_required_frame_size) {
// allocate a frame
#if SSR_USE_AVUTIL_PLANAR_SAMPLE_FMT
unsigned int planes = (m_audio_required_sample_format == AV_SAMPLE_FMT_S16P ||
m_audio_required_sample_format == AV_SAMPLE_FMT_FLTP)? m_audio_channels : 1;
#else
unsigned int planes = 1;
#endif
std::unique_ptr<AVFrameWrapper> audio_frame = CreateAudioFrame(planes, m_audio_required_frame_size, m_audio_required_sample_size, m_audio_required_sample_format);
audio_frame->GetFrame()->pts = lock->m_audio_samples;
// copy/convert the samples
switch(m_audio_required_sample_format) {
case AV_SAMPLE_FMT_S16: {
memcpy(audio_frame->GetFrame()->data[0], lock->m_partial_audio_frame.data(), m_audio_required_frame_size * m_audio_required_sample_size);
break;
}
case AV_SAMPLE_FMT_FLT: {
int16_t *data_in = (int16_t*) lock->m_partial_audio_frame.data();
float *data_out = (float*) audio_frame->GetFrame()->data[0];
for(unsigned int i = 0; i < m_audio_required_frame_size * m_audio_channels; ++i) {
*(data_out++) = (float) *(data_in++) / 32768.0f;
}
break;
}
#if SSR_USE_AVUTIL_PLANAR_SAMPLE_FMT
case AV_SAMPLE_FMT_S16P: {
for(unsigned int p = 0; p < planes; ++p) {
int16_t *data_in = (int16_t*) lock->m_partial_audio_frame.data() + p;
int16_t *data_out = (int16_t*) audio_frame->GetFrame()->data[p];
for(unsigned int i = 0; i < m_audio_required_frame_size; ++i) {
*data_out = *data_in;
data_in += planes; data_out++;
}
}
break;
}
case AV_SAMPLE_FMT_FLTP: {
for(unsigned int p = 0; p < planes; ++p) {
int16_t *data_in = (int16_t*) lock->m_partial_audio_frame.data() + p;
float *data_out = (float*) audio_frame->GetFrame()->data[p];
for(unsigned int i = 0; i < m_audio_required_frame_size; ++i) {
*data_out = (float) *data_in / 32768.0f;
data_in += planes; data_out++;
}
}
break;
}
#endif
default: {
Q_ASSERT(false);
break;
}
}
lock->m_partial_audio_frame_samples = 0;
//Logger::LogInfo("[Synchronizer::FlushBuffers] Encoded audio frame [" + QString::number(lock->m_partial_audio_frame->pts) + "].");
m_audio_encoder->AddFrame(std::move(audio_frame));
}
}
}
}
void Synchronizer::FlushAudioBuffer(Synchronizer::SharedData* lock, int64_t segment_start_time, int64_t segment_stop_time) {
double sample_length = (double) (segment_stop_time - lock->m_segment_audio_start_time) * 1.0e-6;
int64_t samples_max = (int64_t) ceil(sample_length * (double) m_output_format->m_audio_sample_rate) - lock->m_segment_audio_samples_read;
if(lock->m_audio_buffer.GetSize() > 0) {
// Normally, the correct way to calculate the position of the first sample would be:
// int64_t timestamp = lock->m_segment_audio_start_time + (int64_t) round((double) lock->m_segment_audio_samples_read / (double) m_audio_sample_rate * 1.0e6);
// int64_t pos = (int64_t) round((double) (lock->m_time_offset + (timestamp - segment_start_time)) * 1.0e-6 * (double) m_audio_sample_rate);
// Simplified:
// int64_t pos = (int64_t) round((double) (lock->m_time_offset + (lock->m_segment_audio_start_time - segment_start_time)) * 1.0e-6 * (double) m_audio_sample_rate)
// + lock->m_segment_audio_samples_read;
// The first part of the expression is constant, so it only has to be calculated at the start of the segment. After that the increase in position is always
// equal to the number of samples written. Samples are only dropped at the start of the segment, so actually
// the position doesn't have to be calculated anymore after that, since it is assumed to be equal to lock->m_audio_samples.
if(lock->m_segment_audio_can_drop) {
// calculate the offset of the first sample
int64_t pos = (int64_t) round((double) (lock->m_time_offset + (lock->m_segment_audio_start_time - segment_start_time)) * 1.0e-6 * (double) m_output_format->m_audio_sample_rate)
+ lock->m_segment_audio_samples_read;
// drop samples that are too early
if(pos < lock->m_audio_samples) {
int64_t n = std::min(lock->m_audio_samples - pos, (int64_t) lock->m_audio_buffer.GetSize() / m_output_format->m_audio_channels);
lock->m_audio_buffer.Pop(n * m_output_format->m_audio_channels);
lock->m_segment_audio_samples_read += n;
}
}
int64_t samples_left = std::min(samples_max, (int64_t) lock->m_audio_buffer.GetSize() / m_output_format->m_audio_channels);
// add new block to sync diagram
if(m_sync_diagram != NULL && samples_left > 0) {
double t = (double) lock->m_audio_samples / (double) m_output_format->m_audio_sample_rate;
m_sync_diagram->AddBlock(3, t, t + (double) samples_left / (double) m_output_format->m_audio_sample_rate, QColor(0, 255, 0));
}
// send the samples to the encoder
while(samples_left > 0) {
lock->m_segment_audio_can_drop = false;
// copy samples until either the partial frame is full or there are no samples left
//TODO// do direct copy/conversion to new audio frame?
int64_t n = std::min((int64_t) (m_output_format->m_audio_frame_size - lock->m_partial_audio_frame_samples), samples_left);
lock->m_audio_buffer.Pop(lock->m_partial_audio_frame.GetData() + lock->m_partial_audio_frame_samples * m_output_format->m_audio_channels, n * m_output_format->m_audio_channels);
lock->m_segment_audio_samples_read += n;
lock->m_partial_audio_frame_samples += n;
lock->m_audio_samples += n;
samples_left -= n;
// is the partial frame full?
if(lock->m_partial_audio_frame_samples == m_output_format->m_audio_frame_size) {
// allocate a frame
#if SSR_USE_AVUTIL_PLANAR_SAMPLE_FMT
unsigned int planes = (m_output_format->m_audio_sample_format == AV_SAMPLE_FMT_S16P ||
m_output_format->m_audio_sample_format == AV_SAMPLE_FMT_FLTP)? m_output_format->m_audio_channels : 1;
#else
unsigned int planes = 1;
#endif
std::unique_ptr<AVFrameWrapper> audio_frame = CreateAudioFrame(m_output_format->m_audio_channels, m_output_format->m_audio_sample_rate,
m_output_format->m_audio_frame_size, planes, m_output_format->m_audio_sample_format);
audio_frame->GetFrame()->pts = lock->m_audio_samples;
// copy/convert the samples
switch(m_output_format->m_audio_sample_format) {
case AV_SAMPLE_FMT_S16: {
float *data_in = (float*) lock->m_partial_audio_frame.GetData();
int16_t *data_out = (int16_t*) audio_frame->GetFrame()->data[0];
SampleCopy(m_output_format->m_audio_frame_size * m_output_format->m_audio_channels, data_in, 1, data_out, 1);
break;
}
case AV_SAMPLE_FMT_FLT: {
float *data_in = (float*) lock->m_partial_audio_frame.GetData();
float *data_out = (float*) audio_frame->GetFrame()->data[0];
memcpy(data_out, data_in, m_output_format->m_audio_frame_size * m_output_format->m_audio_channels * sizeof(float));
break;
}
#if SSR_USE_AVUTIL_PLANAR_SAMPLE_FMT
case AV_SAMPLE_FMT_S16P: {
for(unsigned int p = 0; p < planes; ++p) {
float *data_in = (float*) lock->m_partial_audio_frame.GetData() + p;
int16_t *data_out = (int16_t*) audio_frame->GetFrame()->data[p];
SampleCopy(m_output_format->m_audio_frame_size, data_in, planes, data_out, 1);
}
break;
}
case AV_SAMPLE_FMT_FLTP: {
for(unsigned int p = 0; p < planes; ++p) {
float *data_in = (float*) lock->m_partial_audio_frame.GetData() + p;
float *data_out = (float*) audio_frame->GetFrame()->data[p];
SampleCopy(m_output_format->m_audio_frame_size, data_in, planes, data_out, 1);
}
break;
}
#endif
default: {
assert(false);
break;
}
}
lock->m_partial_audio_frame_samples = 0;
//Logger::LogInfo("[Synchronizer::FlushAudioBuffer] Encoded audio frame [" + QString::number(lock->m_partial_audio_frame->pts) + "].");
m_output_manager->AddAudioFrame(std::move(audio_frame));
}
}
}
}
