int64_t Synchronizer::GetTotalTime(Synchronizer::SharedData* lock) { if(lock->m_segment_video_started && lock->m_segment_audio_started) { int64_t segment_start_time, segment_stop_time; GetSegmentStartStop(lock, &segment_start_time, &segment_stop_time); return lock->m_time_offset + std::max((int64_t) 0, segment_stop_time - segment_start_time); } else { return lock->m_time_offset; } }
void Synchronizer::NewSegment(SharedData* lock) { FlushBuffers(lock); if(lock->m_segment_video_started && lock->m_segment_audio_started) { int64_t segment_start_time, segment_stop_time; GetSegmentStartStop(lock, &segment_start_time, &segment_stop_time); lock->m_time_offset += std::max((int64_t) 0, segment_stop_time - segment_start_time); } lock->m_video_buffer.clear(); lock->m_audio_buffer.Clear(); InitSegment(lock); }
void Synchronizer::FlushBuffers(SharedData* lock) { if(!lock->m_segment_video_started || !lock->m_segment_audio_started) return; int64_t segment_start_time, segment_stop_time; GetSegmentStartStop(lock, &segment_start_time, &segment_stop_time); // flush video if(m_output_format->m_video_enabled) FlushVideoBuffer(lock, segment_start_time, segment_stop_time); // flush audio if(m_output_format->m_audio_enabled) FlushAudioBuffer(lock, segment_start_time, segment_stop_time); }
void Synchronizer::FlushBuffers(SharedData* lock) { if(!lock->m_segment_video_started || !lock->m_segment_audio_started) return; // Sometimes long delays between video frames can occur, e.g. when a game is showing a loading screen. // Not all codecs/players can handle that. It's also a problem for streaming. To fix this, long delays should be avoided by // duplicating the previous frame a few times when needed. Whenever a video frame is sent to the encoder, it is also copied, // with reference counting for the actual image to minimize overhead. When there is a gap, duplicate frames are inserted. // Duplicate frames are always inserted with a timestamp in the past, because we don't want to drop a real frame if it is captured // right after the duplicate was inserted. MAX_INPUT_LATENCY simulates the latency from the capturing of a frame to the synchronizer, // i.e. any new frame is assumed to have a timestamp higher than the current time minus MAX_INPUT_LATENCY. The duplicate // frame will have a timestamp that's one frame earlier than that time, so it will never interfere with the real frame. // There are two situations where duplicate frames can be inserted: // (1) The queue is not empty, but there is a gap between frames that is too large. // (2) The queue is empty and the last timestamp is too long ago (relative to the end of the video segment). // It is perfectly possible that *both* happen, each possibly multiple times, in just one function call. int64_t segment_start_time, segment_stop_time; GetSegmentStartStop(lock, &segment_start_time, &segment_stop_time); // flush video //int64_t segment_stop_video_pts = (int64_t) round((double) (lock->m_time_offset + (segment_stop_time - segment_start_time)) * 1.0e-6 * (double) m_video_frame_rate); int64_t segment_stop_video_pts = (lock->m_time_offset + (segment_stop_time - segment_start_time)) * (int64_t) m_video_frame_rate / (int64_t) 1000000; int64_t delay_time_per_frame = 1000000 / m_video_frame_rate + 1; // add one to avoid endless accumulation for( ; ; ) { // get/predict the timestamp of the next frame int64_t next_timestamp = (lock->m_video_buffer.empty())? lock->m_segment_video_stop_time - (int64_t) (1000000 / m_video_frame_rate) : lock->m_video_buffer.front()->GetFrame()->pts; int64_t next_pts = (lock->m_time_offset + (next_timestamp - segment_start_time)) * (int64_t) m_video_frame_rate / (int64_t) 1000000; // insert delays if needed, up to either the next frame or the segment end // It doesn't really matter where the delays end up, they will never cause real frames to be dropped, only duplicates. while(lock->m_segment_video_accumulated_delay >= delay_time_per_frame && lock->m_video_pts < std::min(next_pts, segment_stop_video_pts)) { lock->m_segment_video_accumulated_delay -= delay_time_per_frame; lock->m_video_pts += 1; //Logger::LogInfo("[Synchronizer::FlushBuffers] Delay [" + QString::number(lock->m_video_pts - 1) + "] acc " + QString::number(lock->m_segment_video_accumulated_delay) + "."); } // insert duplicate frames if needed, up to either the next frame or the segment end if(lock->m_last_video_frame_data != NULL) { while(lock->m_video_pts + m_video_max_frames_skipped < std::min(next_pts, segment_stop_video_pts)) { // create duplicate frame std::unique_ptr<AVFrameWrapper> duplicate_frame = CreateVideoFrameYUV(m_video_width, m_video_height, &lock->m_last_video_frame_data); duplicate_frame->GetFrame()->pts = lock->m_video_pts + m_video_max_frames_skipped; if(lock->m_sync_diagram != NULL) { double t = (double) duplicate_frame->GetFrame()->pts / (double) m_video_frame_rate; lock->m_sync_diagram->AddBlock(2, t, t + 1.0 / (double) m_video_frame_rate, QColor(255, 196, 0)); } lock->m_segment_video_accumulated_delay = std::max((int64_t) 0, lock->m_segment_video_accumulated_delay - (duplicate_frame->GetFrame()->pts - lock->m_video_pts) * delay_time_per_frame); lock->m_video_pts = duplicate_frame->GetFrame()->pts + 1; //Logger::LogInfo("[Synchronizer::FlushBuffers] Encoded video frame [" + QString::number(duplicate_frame->GetFrame()->pts) + "] (duplicate) acc " + QString::number(lock->m_segment_video_accumulated_delay) + "."); m_video_encoder->AddFrame(std::move(duplicate_frame)); } } // if there are no frames, or they are beyond the segment end, stop if(lock->m_video_buffer.empty() || next_pts >= segment_stop_video_pts) break; // get the frame std::unique_ptr<AVFrameWrapper> frame = std::move(lock->m_video_buffer.front()); lock->m_video_buffer.pop_front(); frame->GetFrame()->pts = next_pts; lock->m_last_video_frame_data = frame->GetFrameData(); // if the frame is way too early, drop it if(frame->GetFrame()->pts < lock->m_video_pts - 1) { //Logger::LogInfo("[Synchronizer::FlushBuffers] Dropped video frame [" + QString::number(frame->GetFrame()->pts) + "] acc " + QString::number(lock->m_segment_video_accumulated_delay) + "."); continue; } // if the frame is just a little too early, move it if(frame->GetFrame()->pts < lock->m_video_pts) frame->GetFrame()->pts = lock->m_video_pts; // if this is the first video frame, always set the pts to zero if(lock->m_video_pts == 0) frame->GetFrame()->pts = 0; if(lock->m_sync_diagram != NULL) { double t = (double) frame->GetFrame()->pts / (double) m_video_frame_rate; lock->m_sync_diagram->AddBlock(2, t, t + 1.0 / (double) m_video_frame_rate, QColor(255, 0, 0)); } // send the frame to the encoder lock->m_segment_video_accumulated_delay = std::max((int64_t) 0, lock->m_segment_video_accumulated_delay - (frame->GetFrame()->pts - lock->m_video_pts) * delay_time_per_frame); lock->m_video_pts = frame->GetFrame()->pts + 1; //Logger::LogInfo("[Synchronizer::FlushBuffers] Encoded video frame [" + QString::number(frame->GetFrame()->pts) + "]."); m_video_encoder->AddFrame(std::move(frame)); } // flush audio double sample_length = (double) (segment_stop_time - lock->m_segment_audio_start_time) * 1.0e-6; int64_t samples_max = (int64_t) ceil(sample_length * (double) m_audio_sample_rate) - lock->m_segment_audio_samples_read; if(lock->m_audio_buffer.GetSize() > 0) { // Normally, the correct way to calculate the position of the first sample would be: // int64_t timestamp = lock->m_segment_audio_start_time + (int64_t) round((double) lock->m_segment_audio_samples_read / (double) m_audio_sample_rate * 1.0e6); // int64_t pos = (int64_t) round((double) (lock->m_time_offset + (timestamp - segment_start_time)) * 1.0e-6 * (double) m_audio_sample_rate); // Simplified: // int64_t pos = (int64_t) round((double) (lock->m_time_offset + (lock->m_segment_audio_start_time - segment_start_time)) * 1.0e-6 * (double) m_audio_sample_rate) // + lock->m_segment_audio_samples_read; // The first part of the expression is constant, so it only has to be calculated at the start of the segment. After that the increase in position is always // equal to the number of samples written. Samples are only dropped at the start of the segment, so actually // the position doesn't have to be calculated anymore after that, since it is assumed to be equal to lock->m_audio_samples. if(lock->m_segment_audio_can_drop) { // calculate the offset of the first sample int64_t pos = (int64_t) round((double) (lock->m_time_offset + (lock->m_segment_audio_start_time - segment_start_time)) * 1.0e-6 * (double) m_audio_sample_rate) + lock->m_segment_audio_samples_read; // drop samples that are too early if(pos < lock->m_audio_samples) { int64_t n = std::min(lock->m_audio_samples - pos, (int64_t) lock->m_audio_buffer.GetSize() / m_audio_sample_size); lock->m_audio_buffer.Drop(n * m_audio_sample_size); lock->m_segment_audio_samples_read += n; } } int64_t samples_left = std::min(samples_max, (int64_t) lock->m_audio_buffer.GetSize() / m_audio_sample_size); if(lock->m_sync_diagram != NULL && samples_left > 0) { double t = (double) lock->m_audio_samples / (double) m_audio_sample_rate; lock->m_sync_diagram->AddBlock(3, t, t + (double) samples_left / (double) m_audio_sample_rate, QColor(0, 255, 0)); } // send the samples to the encoder while(samples_left > 0) { lock->m_segment_audio_can_drop = false; // copy samples until either the partial frame is full or there are no samples left int64_t n = std::min((int64_t) (m_audio_required_frame_size - lock->m_partial_audio_frame_samples), samples_left); lock->m_audio_buffer.Read(lock->m_partial_audio_frame.data() + lock->m_partial_audio_frame_samples * m_audio_sample_size, n * m_audio_sample_size); lock->m_segment_audio_samples_read += n; lock->m_partial_audio_frame_samples += n; lock->m_audio_samples += n; samples_left -= n; // is the partial frame full? if(lock->m_partial_audio_frame_samples == m_audio_required_frame_size) { // allocate a frame #if SSR_USE_AVUTIL_PLANAR_SAMPLE_FMT unsigned int planes = (m_audio_required_sample_format == AV_SAMPLE_FMT_S16P || m_audio_required_sample_format == AV_SAMPLE_FMT_FLTP)? m_audio_channels : 1; #else unsigned int planes = 1; #endif std::unique_ptr<AVFrameWrapper> audio_frame = CreateAudioFrame(planes, m_audio_required_frame_size, m_audio_required_sample_size, m_audio_required_sample_format); audio_frame->GetFrame()->pts = lock->m_audio_samples; // copy/convert the samples switch(m_audio_required_sample_format) { case AV_SAMPLE_FMT_S16: { memcpy(audio_frame->GetFrame()->data[0], lock->m_partial_audio_frame.data(), m_audio_required_frame_size * m_audio_required_sample_size); break; } case AV_SAMPLE_FMT_FLT: { int16_t *data_in = (int16_t*) lock->m_partial_audio_frame.data(); float *data_out = (float*) audio_frame->GetFrame()->data[0]; for(unsigned int i = 0; i < m_audio_required_frame_size * m_audio_channels; ++i) { *(data_out++) = (float) *(data_in++) / 32768.0f; } break; } #if SSR_USE_AVUTIL_PLANAR_SAMPLE_FMT case AV_SAMPLE_FMT_S16P: { for(unsigned int p = 0; p < planes; ++p) { int16_t *data_in = (int16_t*) lock->m_partial_audio_frame.data() + p; int16_t *data_out = (int16_t*) audio_frame->GetFrame()->data[p]; for(unsigned int i = 0; i < m_audio_required_frame_size; ++i) { *data_out = *data_in; data_in += planes; data_out++; } } break; } case AV_SAMPLE_FMT_FLTP: { for(unsigned int p = 0; p < planes; ++p) { int16_t *data_in = (int16_t*) lock->m_partial_audio_frame.data() + p; float *data_out = (float*) audio_frame->GetFrame()->data[p]; for(unsigned int i = 0; i < m_audio_required_frame_size; ++i) { *data_out = (float) *data_in / 32768.0f; data_in += planes; data_out++; } } break; } #endif default: { Q_ASSERT(false); break; } } lock->m_partial_audio_frame_samples = 0; //Logger::LogInfo("[Synchronizer::FlushBuffers] Encoded audio frame [" + QString::number(lock->m_partial_audio_frame->pts) + "]."); m_audio_encoder->AddFrame(std::move(audio_frame)); } } } }