Example #1
0
int JACKInput::ProcessCallback(jack_nframes_t nframes, void* arg) {
	JACKInput *input = (JACKInput*) arg;

	// deal with holes
	if(input->m_jackthread_hole) {
		char *message = input->m_message_queue.PrepareWriteMessage(sizeof(enum_eventtype));
		if(message == NULL)
			return 0;
		*((enum_eventtype*) message) = EVENTTYPE_HOLE;
		input->m_message_queue.WriteMessage();
		input->m_jackthread_hole = false;
	}

	// This function is called from a real-time thread, so it's not a good idea to do actual work here.
	// The data is moved to a queue, and a second thread will do the work as usual.
	//TODO// if nframes is small, then combine multiple blocks into one?
	char *message = input->m_message_queue.PrepareWriteMessage(sizeof(enum_eventtype) + sizeof(Event_Data) + nframes * input->m_channels * sizeof(float));
	if(message == NULL) {
		input->m_jackthread_hole = true;
		return 0;
	}
	*((enum_eventtype*) message) = EVENTTYPE_DATA;
	message += sizeof(enum_eventtype);
	((Event_Data*) message)->m_timestamp = hrt_time_micro() - (int64_t) nframes * (int64_t) 1000000 / (int64_t) input->m_jackthread_sample_rate;
	((Event_Data*) message)->m_sample_rate = input->m_jackthread_sample_rate;
	((Event_Data*) message)->m_sample_count = nframes;
	message += sizeof(Event_Data);
	for(unsigned int p = 0; p < input->m_channels; ++p) {
		SampleCopy(nframes, (float*) jack_port_get_buffer(input->m_jack_ports[p], nframes), 1, (float*) message + p, input->m_channels);
	}
	input->m_message_queue.WriteMessage();

	return 0;
}
Example #2
0
void Synchronizer::FlushAudioBuffer(Synchronizer::SharedData* lock, int64_t segment_start_time, int64_t segment_stop_time) {

	double sample_length = (double) (segment_stop_time - lock->m_segment_audio_start_time) * 1.0e-6;
	int64_t samples_max = (int64_t) ceil(sample_length * (double) m_output_format->m_audio_sample_rate) - lock->m_segment_audio_samples_read;
	if(lock->m_audio_buffer.GetSize() > 0) {

		// Normally, the correct way to calculate the position of the first sample would be:
		//     int64_t timestamp = lock->m_segment_audio_start_time + (int64_t) round((double) lock->m_segment_audio_samples_read / (double) m_audio_sample_rate * 1.0e6);
		//     int64_t pos = (int64_t) round((double) (lock->m_time_offset + (timestamp - segment_start_time)) * 1.0e-6 * (double) m_audio_sample_rate);
		// Simplified:
		//     int64_t pos = (int64_t) round((double) (lock->m_time_offset + (lock->m_segment_audio_start_time - segment_start_time)) * 1.0e-6 * (double) m_audio_sample_rate)
		//                   + lock->m_segment_audio_samples_read;
		// The first part of the expression is constant, so it only has to be calculated at the start of the segment. After that the increase in position is always
		// equal to the number of samples written. Samples are only dropped at the start of the segment, so actually
		// the position doesn't have to be calculated anymore after that, since it is assumed to be equal to lock->m_audio_samples.

		if(lock->m_segment_audio_can_drop) {

			// calculate the offset of the first sample
			int64_t pos = (int64_t) round((double) (lock->m_time_offset + (lock->m_segment_audio_start_time - segment_start_time)) * 1.0e-6 * (double) m_output_format->m_audio_sample_rate)
						  + lock->m_segment_audio_samples_read;

			// drop samples that are too early
			if(pos < lock->m_audio_samples) {
				int64_t n = std::min(lock->m_audio_samples - pos, (int64_t) lock->m_audio_buffer.GetSize() / m_output_format->m_audio_channels);
				lock->m_audio_buffer.Pop(n * m_output_format->m_audio_channels);
				lock->m_segment_audio_samples_read += n;
			}

		}

		int64_t samples_left = std::min(samples_max, (int64_t) lock->m_audio_buffer.GetSize() / m_output_format->m_audio_channels);

		// add new block to sync diagram
		if(m_sync_diagram != NULL && samples_left > 0) {
			double t = (double) lock->m_audio_samples / (double) m_output_format->m_audio_sample_rate;
			m_sync_diagram->AddBlock(3, t, t + (double) samples_left / (double) m_output_format->m_audio_sample_rate, QColor(0, 255, 0));
		}

		// send the samples to the encoder
		while(samples_left > 0) {

			lock->m_segment_audio_can_drop = false;

			// copy samples until either the partial frame is full or there are no samples left
			//TODO// do direct copy/conversion to new audio frame?
			int64_t n = std::min((int64_t) (m_output_format->m_audio_frame_size - lock->m_partial_audio_frame_samples), samples_left);
			lock->m_audio_buffer.Pop(lock->m_partial_audio_frame.GetData() + lock->m_partial_audio_frame_samples * m_output_format->m_audio_channels, n * m_output_format->m_audio_channels);
			lock->m_segment_audio_samples_read += n;
			lock->m_partial_audio_frame_samples += n;
			lock->m_audio_samples += n;
			samples_left -= n;

			// is the partial frame full?
			if(lock->m_partial_audio_frame_samples == m_output_format->m_audio_frame_size) {

				// allocate a frame
#if SSR_USE_AVUTIL_PLANAR_SAMPLE_FMT
				unsigned int planes = (m_output_format->m_audio_sample_format == AV_SAMPLE_FMT_S16P ||
									   m_output_format->m_audio_sample_format == AV_SAMPLE_FMT_FLTP)? m_output_format->m_audio_channels : 1;
#else
				unsigned int planes = 1;
#endif
				std::unique_ptr<AVFrameWrapper> audio_frame = CreateAudioFrame(m_output_format->m_audio_channels, m_output_format->m_audio_sample_rate,
																			   m_output_format->m_audio_frame_size, planes, m_output_format->m_audio_sample_format);
				audio_frame->GetFrame()->pts = lock->m_audio_samples;

				// copy/convert the samples
				switch(m_output_format->m_audio_sample_format) {
					case AV_SAMPLE_FMT_S16: {
						float *data_in = (float*) lock->m_partial_audio_frame.GetData();
						int16_t *data_out = (int16_t*) audio_frame->GetFrame()->data[0];
						SampleCopy(m_output_format->m_audio_frame_size * m_output_format->m_audio_channels, data_in, 1, data_out, 1);
						break;
					}
					case AV_SAMPLE_FMT_FLT: {
						float *data_in = (float*) lock->m_partial_audio_frame.GetData();
						float *data_out = (float*) audio_frame->GetFrame()->data[0];
						memcpy(data_out, data_in, m_output_format->m_audio_frame_size * m_output_format->m_audio_channels * sizeof(float));
						break;
					}
#if SSR_USE_AVUTIL_PLANAR_SAMPLE_FMT
					case AV_SAMPLE_FMT_S16P: {
						for(unsigned int p = 0; p < planes; ++p) {
							float *data_in = (float*) lock->m_partial_audio_frame.GetData() + p;
							int16_t *data_out = (int16_t*) audio_frame->GetFrame()->data[p];
							SampleCopy(m_output_format->m_audio_frame_size, data_in, planes, data_out, 1);
						}
						break;
					}
					case AV_SAMPLE_FMT_FLTP: {
						for(unsigned int p = 0; p < planes; ++p) {
							float *data_in = (float*) lock->m_partial_audio_frame.GetData() + p;
							float *data_out = (float*) audio_frame->GetFrame()->data[p];
							SampleCopy(m_output_format->m_audio_frame_size, data_in, planes, data_out, 1);
						}
						break;
					}
#endif
					default: {
						assert(false);
						break;
					}
				}
				lock->m_partial_audio_frame_samples = 0;

				//Logger::LogInfo("[Synchronizer::FlushAudioBuffer] Encoded audio frame [" + QString::number(lock->m_partial_audio_frame->pts) + "].");
				m_output_manager->AddAudioFrame(std::move(audio_frame));
			}

		}

	}

}
Example #3
0
void Resampler::Resample(unsigned int in_channels, unsigned int in_sample_rate, AVSampleFormat in_format, unsigned int in_sample_count, const uint8_t* in_data,
						 unsigned int out_channels, unsigned int out_sample_rate, AVSampleFormat out_format, unsigned int* out_sample_count, const uint8_t** out_data) {
	Q_ASSERT(in_channels > 0 && out_channels > 0);
	Q_ASSERT(in_sample_rate > 0 && out_sample_rate > 0);
	Q_ASSERT(in_channels == out_channels);
	Q_ASSERT(out_format == AV_SAMPLE_FMT_FLT);

	if(m_started) {
		Q_ASSERT(m_out_channels == out_channels);
		Q_ASSERT(m_out_sample_rate == out_sample_rate);
	} else {
		m_started = true;
		m_in_sample_rate = 0; // trigger creation of new resampler
		m_out_channels = out_channels;
		m_out_sample_rate = out_sample_rate;
	}

	// prepare output samples
	unsigned int out_pos = 0;
	m_out_data.alloc(16 * 1024);

	// do we need a new resampler?
	if(in_sample_rate != m_in_sample_rate) {

		// delete old resampler
		if(m_soxr != NULL) {

			// flush resampler
			for( ; ; ) {
				size_t out_done;
				soxr_error_t error = soxr_process(m_soxr,
												  NULL, 0, NULL,
												  m_out_data.data() + out_pos * out_channels * sizeof(float), m_out_data.size() / (out_channels * sizeof(float)) - out_pos, &out_done);
				if(error != NULL) {
					Logger::LogError("[Resampler::Resample] " + QObject::tr("Error: Flushing resampler failed! Reason: %s").arg(soxr_strerror(error)));
					throw SoxrException();
				}
				out_pos += out_done;
				if(out_pos < m_out_data.size() / (out_channels * sizeof(float)))
					break;
				m_out_data.realloc(m_out_data.size() * 2);
			}

			soxr_delete(m_soxr);
			m_soxr = NULL;
		}

		m_in_sample_rate = in_sample_rate;

		// do we really need a resampler?
		if(m_in_sample_rate != m_out_sample_rate) {
			Logger::LogInfo("[Resampler::Resampler] " + QObject::tr("Resampling from %1 to %2.").arg(m_in_sample_rate).arg(m_out_sample_rate));
			soxr_error_t error;
			soxr_quality_spec_t quality = soxr_quality_spec(SOXR_MQ, 0);
			m_soxr = soxr_create((double) m_in_sample_rate, (double) m_out_sample_rate, out_channels, &error, NULL, &quality, NULL);
			if(m_soxr == NULL || error != NULL) {
				m_in_sample_rate = 0;
				Logger::LogError("[Resampler::Resampler] " + QObject::tr("Error: Can't create resampler! Reason: %s").arg(soxr_strerror(error)));
				throw SoxrException();
			}
		} else {
			Logger::LogInfo("[Resampler::Resampler] " + QObject::tr("Resampling not needed."));
		}

	}

	// prepare input samples
	uint8_t *in_data_float;
	unsigned int in_pos = 0;
	if(in_format == AV_SAMPLE_FMT_FLT) {
		in_data_float = (uint8_t*) in_data;
	} else if(in_format == AV_SAMPLE_FMT_S16) {
		m_in_data.alloc(in_sample_count * out_channels * sizeof(float));
		SampleCopy(in_sample_count * out_channels, (const int16_t*) in_data, 1, (float*) m_in_data.data(), 1);
		in_data_float = (uint8_t*) m_in_data.data();
	} else {
		Q_ASSERT(false); // unsupported input format
		return;
	}

	// no resampling needed?
	if(m_in_sample_rate == m_out_sample_rate) {
		if(out_pos == 0) {
			*out_sample_count = in_sample_count;
			*out_data = in_data_float;
		} else {
			m_out_data.realloc((out_pos + in_sample_count) * out_channels * sizeof(float));
			memcpy(m_out_data.data() + out_pos * out_channels * sizeof(float), in_data_float, in_sample_count * out_channels * sizeof(float));
			*out_sample_count = out_pos + in_sample_count;
			*out_data = m_out_data.data();
		}
		return;
	}

	// resample
	for( ; ; ) {
		size_t in_done, out_done;
		soxr_error_t error = soxr_process(m_soxr,
										  in_data_float + in_pos * out_channels * sizeof(float), in_sample_count - in_pos, &in_done,
										  m_out_data.data() + out_pos * out_channels * sizeof(float), m_out_data.size() / (out_channels * sizeof(float)) - out_pos, &out_done);
		if(error != NULL) {
			Logger::LogError("[Resampler::Resample] " + QObject::tr("Error: Resampling failed!"));
			throw SoxrException();
		}
		in_pos += in_done;
		out_pos += out_done;
		if(in_pos == in_sample_count)
			break;
		m_out_data.realloc(m_out_data.size() * 2);
	}

	*out_sample_count = out_pos;
	*out_data = m_out_data.data();

}
Example #4
0
void Synchronizer::ReadAudioSamples(unsigned int channels, unsigned int sample_rate, AVSampleFormat format, unsigned int sample_count, const uint8_t* data, int64_t timestamp) {
	assert(m_audio_encoder != NULL);
	assert(channels == m_audio_channels); // remixing isn't supported yet

	// sanity check
	if(sample_count == 0)
		return;

	// add new block to sync diagram
	if(m_sync_diagram != NULL)
		m_sync_diagram->AddBlock(1, (double) timestamp * 1.0e-6, (double) timestamp * 1.0e-6 + (double) sample_count / (double) sample_rate, QColor(0, 255, 0));

	AudioLock audiolock(&m_audio_data);

	// check the timestamp
	if(timestamp < audiolock->m_last_timestamp) {
		if(timestamp < audiolock->m_last_timestamp - 10000)
			Logger::LogWarning("[Synchronizer::ReadAudioSamples] " + Logger::tr("Warning: Received audio samples with non-monotonic timestamp."));
		timestamp = audiolock->m_last_timestamp;
	}

	// update the timestamps
	int64_t previous_timestamp;
	if(audiolock->m_first_timestamp == AV_NOPTS_VALUE) {
		audiolock->m_first_timestamp = timestamp;
		previous_timestamp = timestamp;
	} else {
		previous_timestamp = audiolock->m_last_timestamp;
	}
	audiolock->m_last_timestamp = timestamp;

	// calculate drift
	double current_drift = ((double) audiolock->m_samples_written + audiolock->m_fast_resampler->GetOutputLatency()) / (double) m_audio_sample_rate
			- (double) (timestamp - audiolock->m_first_timestamp) * 1.0e-6;

	// if there are too many audio samples, drop the frame (unlikely)
	if(current_drift > DRIFT_ERROR_THRESHOLD) {
		Logger::LogWarning("[Synchronizer::ReadAudioSamples] " + Logger::tr("Warning: Too many audio samples, dropping samples to keep the audio in sync with the video."));
		return;
	}

	// if there are not enough audio samples, insert zeros
	unsigned int sample_count_out = 0;
	if(current_drift < -DRIFT_ERROR_THRESHOLD || audiolock->m_insert_zeros) {

		if(!audiolock->m_insert_zeros)
			Logger::LogWarning("[Synchronizer::ReadAudioSamples] " + Logger::tr("Warning: Not enough audio samples, inserting silence to keep the audio in sync with the video."));
		audiolock->m_insert_zeros = false;

		// insert zeros
		unsigned int n = std::max(0, (int) round(-current_drift * (double) sample_rate));
		audiolock->m_temp_input_buffer.Alloc(n * m_audio_channels);
		std::fill_n(audiolock->m_temp_input_buffer.GetData(), n * m_audio_channels, 0.0f);
		sample_count_out = audiolock->m_fast_resampler->Resample((double) sample_rate / (double) m_audio_sample_rate, 1.0,
																 audiolock->m_temp_input_buffer.GetData(), n, &audiolock->m_temp_output_buffer, sample_count_out);

		// recalculate drift
		current_drift = ((double) (audiolock->m_samples_written + sample_count_out) + audiolock->m_fast_resampler->GetOutputLatency()) / (double) m_audio_sample_rate
				- (double) (timestamp - audiolock->m_first_timestamp) * 1.0e-6;

	}

	// do drift correction
	// The point of drift correction is to keep video and audio in sync even when the clocks are not running at exactly the same speed.
	// This can happen because the sample rate of the sound card is not always 100% accurate. Even a 0.1% error will result in audio that is
	// seconds too early or too late at the end of a one hour video. This problem doesn't occur on all computers though (I'm not sure why).
	// Another cause of desynchronization is problems/glitches with PulseAudio (e.g. jumps in time when switching between sources).
	double dt = fmin((double) (timestamp - previous_timestamp) * 1.0e-6, DRIFT_MAX_BLOCK);
	audiolock->m_average_drift = clamp(audiolock->m_average_drift + DRIFT_CORRECTION_I * current_drift * dt, -0.5, 0.5);
	if(audiolock->m_average_drift < -0.02 && audiolock->m_warn_desync) {
		audiolock->m_warn_desync = false;
		Logger::LogWarning("[Synchronizer::ReadAudioSamples] " + Logger::tr("Warning: Audio input is more than 2% too slow!"));
	}
	if(audiolock->m_average_drift > 0.02 && audiolock->m_warn_desync) {
		audiolock->m_warn_desync = false;
		Logger::LogWarning("[Synchronizer::ReadAudioSamples] " + Logger::tr("Warning: Audio input is more than 2% too fast!"));
	}
	double length = (double) sample_count / (double) sample_rate;
	double drift_correction = clamp(DRIFT_CORRECTION_P * current_drift + audiolock->m_average_drift, -0.5, 0.5) * fmin(1.0, DRIFT_MAX_BLOCK / length);

	//qDebug() << "current_drift" << current_drift << "average_drift" << audiolock->m_average_drift << "drift_correction" << drift_correction;

	// convert the samples
	const float *data_float;
	if(format == AV_SAMPLE_FMT_FLT) {
		data_float = (const float*) data;
	} else if(format == AV_SAMPLE_FMT_S16) {
		audiolock->m_temp_input_buffer.Alloc(sample_count * m_audio_channels);
		data_float = audiolock->m_temp_input_buffer.GetData();
		SampleCopy(sample_count * m_audio_channels, (const int16_t*) data, 1, audiolock->m_temp_input_buffer.GetData(), 1);
	} else {
		Logger::LogError("[Synchronizer::ReadAudioSamples] " + Logger::tr("Error: Audio sample format is not supported!"));
		throw ResamplerException();
	}

	// resample
	sample_count_out = audiolock->m_fast_resampler->Resample((double) sample_rate / (double) m_audio_sample_rate, 1.0 / (1.0 - drift_correction),
															 data_float, sample_count, &audiolock->m_temp_output_buffer, sample_count_out);
	audiolock->m_samples_written += sample_count_out;

	SharedLock lock(&m_shared_data);

	// avoid memory problems by limiting the audio buffer size
	if(lock->m_audio_buffer.GetSize() / m_audio_channels >= MAX_AUDIO_SAMPLES_BUFFERED) {
		if(lock->m_segment_video_started) {
			Logger::LogWarning("[Synchronizer::ReadAudioSamples] " + Logger::tr("Warning: Audio buffer overflow, starting new segment to keep the audio in sync with the video "
																				 "(some video and/or audio may be lost). The video input seems to be too slow."));
			NewSegment(lock.get());
		} else {
			// If the video hasn't started yet, it makes more sense to drop the oldest samples.
			// Shifting the start time like this isn't completely accurate, but this shouldn't happen often anyway.
			// The number of samples dropped is calculated so that the buffer will be 90% full after this.
			size_t n = lock->m_audio_buffer.GetSize() / m_audio_channels - MAX_AUDIO_SAMPLES_BUFFERED * 9 / 10;
			lock->m_audio_buffer.Pop(n * m_audio_channels);
			lock->m_segment_audio_start_time += (int64_t) round((double) n / (double) m_audio_sample_rate * 1.0e6);
		}
	}

	// start audio
	if(!lock->m_segment_audio_started) {
		lock->m_segment_audio_started = true;
		lock->m_segment_audio_start_time = timestamp;
		lock->m_segment_audio_stop_time = timestamp;
	}

	// store the samples
	lock->m_audio_buffer.Push(audiolock->m_temp_output_buffer.GetData(), sample_count_out * m_audio_channels);

	// increase segment stop time
	double new_sample_length = (double) (lock->m_segment_audio_samples_read + lock->m_audio_buffer.GetSize() / m_audio_channels) / (double) m_audio_sample_rate;
	lock->m_segment_audio_stop_time = lock->m_segment_audio_start_time + (int64_t) round(new_sample_length * 1.0e6);

}