void RtpAudioStream::writeSilence(int samples)
{
if (samples < 1 || audio_out_rate_ == 0) return;
unsigned silence_bytes = samples * sample_bytes_;
char *silence_buff = (char *) g_malloc0(silence_bytes);
RTP_STREAM_DEBUG("Writing %u silence samples", samples);
tempfile_->write(silence_buff, silence_bytes);
g_free(silence_buff);
QVector<qint16> visual_fill(samples * visual_sample_rate_ / audio_out_rate_, 0);
visual_samples_ += visual_fill;
}
void RtpAudioStream::decode()
{
if (rtp_packets_.size() < 1) return;
// gtk/rtp_player.c:decode_rtp_stream
// XXX This is more messy than it should be.
gsize resample_buff_len = 0x1000;
SAMPLE *resample_buff = (SAMPLE *) g_malloc(resample_buff_len);
spx_uint32_t cur_in_rate = 0, visual_out_rate = 0;
char *write_buff = NULL;
qint64 write_bytes = 0;
unsigned channels = 0;
unsigned sample_rate = 0;
int last_sequence = 0;
double rtp_time_prev = 0.0;
double arrive_time_prev = 0.0;
double pack_period = 0.0;
double start_time = 0.0;
double start_rtp_time = 0.0;
guint32 start_timestamp = 0;
size_t decoded_bytes_prev = 0;
for (int cur_packet = 0; cur_packet < rtp_packets_.size(); cur_packet++) {
SAMPLE *decode_buff = NULL;
// XXX The GTK+ UI updates a progress bar here.
rtp_packet_t *rtp_packet = rtp_packets_[cur_packet];
stop_rel_time_ = start_rel_time_ + rtp_packet->arrive_offset;
ws_codec_resampler_get_rate(visual_resampler_, &cur_in_rate, &visual_out_rate);
QString payload_name;
if (rtp_packet->info->info_payload_type_str) {
payload_name = rtp_packet->info->info_payload_type_str;
} else {
payload_name = try_val_to_str_ext(rtp_packet->info->info_payload_type, &rtp_payload_type_short_vals_ext);
}
if (!payload_name.isEmpty()) {
payload_names_ << payload_name;
}
if (cur_packet < 1) { // First packet
start_timestamp = rtp_packet->info->info_timestamp;
start_rtp_time = 0;
rtp_time_prev = 0;
last_sequence = rtp_packet->info->info_seq_num - 1;
}
size_t decoded_bytes = decode_rtp_packet(rtp_packet, &decode_buff, decoders_hash_, &channels, &sample_rate);
if (decoded_bytes == 0 || sample_rate == 0) {
// We didn't decode anything. Clean up and prep for the next packet.
last_sequence = rtp_packet->info->info_seq_num;
continue;
}
if (audio_out_rate_ == 0) { // First non-zero wins
audio_out_rate_ = sample_rate;
RTP_STREAM_DEBUG("Audio sample rate is %u", audio_out_rate_);
// Prepend silence to match our sibling streams.
tempfile_->seek(0);
int prepend_samples = (start_rel_time_ - global_start_rel_time_) * audio_out_rate_;
if (prepend_samples > 0) {
writeSilence(prepend_samples);
}
}
if (rtp_packet->info->info_seq_num != last_sequence+1) {
out_of_seq_timestamps_.append(stop_rel_time_);
}
last_sequence = rtp_packet->info->info_seq_num;
double rtp_time = (double)(rtp_packet->info->info_timestamp-start_timestamp)/sample_rate - start_rtp_time;
double arrive_time;
if (timing_mode_ == Uninterrupted) {
arrive_time = rtp_time;
} else {
arrive_time = (double)rtp_packet->arrive_offset/1000 - start_time;
}
double diff = qAbs(arrive_time - rtp_time);
if (diff*1000 > jitter_buffer_size_ && timing_mode_ == Uninterrupted) {
// rtp_player.c:628
jitter_drop_timestamps_.append(stop_rel_time_);
RTP_STREAM_DEBUG("Packet drop by jitter buffer exceeded %f > %d", diff*1000, jitter_buffer_size_);
/* if there was a silence period (more than two packetization period) resync the source */
if ( (rtp_time - rtp_time_prev) > pack_period*2 ) {
int silence_samples;
RTP_STREAM_DEBUG("Resync...");
silence_samples = (int)((arrive_time - arrive_time_prev)*sample_rate - decoded_bytes_prev / sample_bytes_);
/* Fix for bug 4119/5902: don't insert too many silence frames.
* XXX - is there a better thing to do here?
*/
silence_samples = qMin(silence_samples, max_silence_samples_);
writeSilence(silence_samples);
silence_timestamps_.append(stop_rel_time_);
decoded_bytes_prev = 0;
/* defined start_timestmp to avoid overflow in timestamp. TODO: handle the timestamp correctly */
/* XXX: if timestamps (RTP) are missing/ignored try use packet arrive time only (see also "rtp_time") */
start_timestamp = rtp_packet->info->info_timestamp;
start_rtp_time = 0;
start_time = (double)rtp_packet->arrive_offset/1000;
rtp_time_prev = 0;
}
} else {
// rtp_player.c:664
/* Add silence if it is necessary */
int silence_samples;
if (timing_mode_ == Uninterrupted) {
silence_samples = 0;
} else {
silence_samples = (int)((rtp_time - rtp_time_prev)*sample_rate - decoded_bytes_prev / sample_bytes_);
}
if (silence_samples != 0) {
wrong_timestamp_timestamps_.append(stop_rel_time_);
}
if (silence_samples > 0) {
/* Fix for bug 4119/5902: don't insert too many silence frames.
* XXX - is there a better thing to do here?
*/
silence_samples = qMin(silence_samples, max_silence_samples_);
writeSilence(silence_samples);
silence_timestamps_.append(stop_rel_time_);
}
// XXX rtp_player.c:696 adds audio here.
rtp_time_prev = rtp_time;
pack_period = (double) decoded_bytes / sample_bytes_ / sample_rate;
decoded_bytes_prev = decoded_bytes;
arrive_time_prev = arrive_time;
}
// Write samples to our file.
write_buff = (char *) decode_buff;
write_bytes = rtp_packet->info->info_payload_len * sample_bytes_;
if (audio_out_rate_ != sample_rate) {
// Resample the audio to match our previous output rate.
if (!audio_resampler_) {
audio_resampler_ = ws_codec_resampler_init(1, sample_rate, audio_out_rate_, 10, NULL);
ws_codec_resampler_skip_zeros(audio_resampler_);
RTP_STREAM_DEBUG("Started resampling from %u to (out) %u Hz.", sample_rate, audio_out_rate_);
} else {
spx_uint32_t audio_out_rate;
ws_codec_resampler_get_rate(audio_resampler_, &cur_in_rate, &audio_out_rate);
// Adjust rates if needed.
if (sample_rate != cur_in_rate) {
ws_codec_resampler_set_rate(audio_resampler_, sample_rate, audio_out_rate);
ws_codec_resampler_set_rate(visual_resampler_, sample_rate, visual_out_rate);
RTP_STREAM_DEBUG("Changed input rate from %u to %u Hz. Out is %u.", cur_in_rate, sample_rate, audio_out_rate_);
}
}
spx_uint32_t in_len = (spx_uint32_t)rtp_packet->info->info_payload_len;
spx_uint32_t out_len = (audio_out_rate_ * (spx_uint32_t)rtp_packet->info->info_payload_len / sample_rate) + (audio_out_rate_ % sample_rate != 0);
if (out_len * sample_bytes_ > resample_buff_len) {
while ((out_len * sample_bytes_ > resample_buff_len))
resample_buff_len *= 2;
resample_buff = (SAMPLE *) g_realloc(resample_buff, resample_buff_len);
}
ws_codec_resampler_process_int(audio_resampler_, 0, decode_buff, &in_len, resample_buff, &out_len);
write_buff = (char *) decode_buff;
write_bytes = out_len * sample_bytes_;
}
// Write the decoded, possibly-resampled audio to our temp file.
tempfile_->write(write_buff, write_bytes);
// Collect our visual samples.
spx_uint32_t in_len = (spx_uint32_t)rtp_packet->info->info_payload_len;
spx_uint32_t out_len = (visual_out_rate * in_len / sample_rate) + (visual_out_rate % sample_rate != 0);
if (out_len * sample_bytes_ > resample_buff_len) {
while ((out_len * sample_bytes_ > resample_buff_len))
resample_buff_len *= 2;
resample_buff = (SAMPLE *) g_realloc(resample_buff, resample_buff_len);
}
ws_codec_resampler_process_int(visual_resampler_, 0, decode_buff, &in_len, resample_buff, &out_len);
for (unsigned i = 0; i < out_len; i++) {
packet_timestamps_[stop_rel_time_ + (double) i / visual_out_rate] = rtp_packet->frame_num;
if (qAbs(resample_buff[i]) > max_sample_val_) max_sample_val_ = qAbs(resample_buff[i]);
visual_samples_.append(resample_buff[i]);
}
// Finally, write the resampled audio to our temp file and clean up.
g_free(decode_buff);
}
g_free(resample_buff);
}
void RtpPlayerDialog::rescanPackets(bool rescale_axes)
{
int row_count = ui->streamTreeWidget->topLevelItemCount();
// Clear existing graphs and reset stream values
for (int row = 0; row < row_count; row++) {
QTreeWidgetItem *ti = ui->streamTreeWidget->topLevelItem(row);
RtpAudioStream *audio_stream = ti->data(stream_data_col_, Qt::UserRole).value<RtpAudioStream*>();
audio_stream->reset(start_rel_time_);
ti->setData(graph_data_col_, Qt::UserRole, QVariant());
}
ui->audioPlot->clearGraphs();
bool show_legend = false;
bool relative_timestamps = !ui->todCheckBox->isChecked();
ui->audioPlot->xAxis->setTickLabelType(relative_timestamps ? QCPAxis::ltNumber : QCPAxis::ltDateTime);
for (int row = 0; row < row_count; row++) {
QTreeWidgetItem *ti = ui->streamTreeWidget->topLevelItem(row);
RtpAudioStream *audio_stream = ti->data(stream_data_col_, Qt::UserRole).value<RtpAudioStream*>();
int y_offset = row_count - row - 1;
audio_stream->setJitterBufferSize((int) ui->jitterSpinBox->value());
RtpAudioStream::TimingMode timing_mode = RtpAudioStream::JitterBuffer;
switch (ui->timingComboBox->currentIndex()) {
case RtpAudioStream::RtpTimestamp:
timing_mode = RtpAudioStream::RtpTimestamp;
break;
case RtpAudioStream::Uninterrupted:
timing_mode = RtpAudioStream::Uninterrupted;
break;
default:
break;
}
audio_stream->setTimingMode(timing_mode);
audio_stream->decode();
// Waveform
QCPGraph *audio_graph = ui->audioPlot->addGraph();
QPen wf_pen(audio_stream->color());
wf_pen.setWidthF(wf_graph_normal_width_);
audio_graph->setPen(wf_pen);
wf_pen.setWidthF(wf_graph_selected_width_);
audio_graph->setSelectedPen(wf_pen);
audio_graph->setSelectable(false);
audio_graph->setData(audio_stream->visualTimestamps(relative_timestamps), audio_stream->visualSamples(y_offset));
audio_graph->removeFromLegend();
ti->setData(graph_data_col_, Qt::UserRole, QVariant::fromValue<QCPGraph *>(audio_graph));
RTP_STREAM_DEBUG("Plotting %s, %d samples", ti->text(src_addr_col_).toUtf8().constData(), audio_graph->data()->keys().length());
QString span_str = QString("%1 - %2 (%3)")
.arg(QString::number(audio_stream->startRelTime(), 'g', 3))
.arg(QString::number(audio_stream->stopRelTime(), 'g', 3))
.arg(QString::number(audio_stream->stopRelTime() - audio_stream->startRelTime(), 'g', 3));
ti->setText(time_span_col_, span_str);
ti->setText(sample_rate_col_, QString::number(audio_stream->sampleRate()));
ti->setText(payload_col_, audio_stream->payloadNames().join(", "));
if (audio_stream->outOfSequence() > 0) {
// Sequence numbers
QCPGraph *seq_graph = ui->audioPlot->addGraph();
seq_graph->setLineStyle(QCPGraph::lsNone);
seq_graph->setScatterStyle(QCPScatterStyle(QCPScatterStyle::ssSquare, tango_aluminium_6, Qt::white, 4)); // Arbitrary
seq_graph->setSelectable(false);
seq_graph->setData(audio_stream->outOfSequenceTimestamps(relative_timestamps), audio_stream->outOfSequenceSamples(y_offset));
if (row < 1) {
seq_graph->setName(tr("Out of Sequence"));
show_legend = true;
} else {
seq_graph->removeFromLegend();
}
}
if (audio_stream->jitterDropped() > 0) {
// Jitter drops
QCPGraph *seq_graph = ui->audioPlot->addGraph();
seq_graph->setLineStyle(QCPGraph::lsNone);
seq_graph->setScatterStyle(QCPScatterStyle(QCPScatterStyle::ssCircle, tango_scarlet_red_5, Qt::white, 4)); // Arbitrary
seq_graph->setSelectable(false);
seq_graph->setData(audio_stream->jitterDroppedTimestamps(relative_timestamps), audio_stream->jitterDroppedSamples(y_offset));
if (row < 1) {
seq_graph->setName(tr("Jitter Drops"));
show_legend = true;
} else {
seq_graph->removeFromLegend();
}
}
if (audio_stream->wrongTimestamps() > 0) {
// Wrong timestamps
QCPGraph *seq_graph = ui->audioPlot->addGraph();
seq_graph->setLineStyle(QCPGraph::lsNone);
seq_graph->setScatterStyle(QCPScatterStyle(QCPScatterStyle::ssDiamond, tango_sky_blue_5, Qt::white, 4)); // Arbitrary
seq_graph->setSelectable(false);
seq_graph->setData(audio_stream->wrongTimestampTimestamps(relative_timestamps), audio_stream->wrongTimestampSamples(y_offset));
if (row < 1) {
seq_graph->setName(tr("Wrong Timestamps"));
show_legend = true;
} else {
seq_graph->removeFromLegend();
}
}
if (audio_stream->insertedSilences() > 0) {
// Inserted silence
QCPGraph *seq_graph = ui->audioPlot->addGraph();
seq_graph->setLineStyle(QCPGraph::lsNone);
seq_graph->setScatterStyle(QCPScatterStyle(QCPScatterStyle::ssTriangle, tango_butter_5, Qt::white, 4)); // Arbitrary
seq_graph->setSelectable(false);
seq_graph->setData(audio_stream->insertedSilenceTimestamps(relative_timestamps), audio_stream->insertedSilenceSamples(y_offset));
if (row < 1) {
seq_graph->setName(tr("Inserted Silence"));
show_legend = true;
} else {
seq_graph->removeFromLegend();
}
}
}
ui->audioPlot->legend->setVisible(show_legend);
for (int col = 0; col < ui->streamTreeWidget->columnCount() - 1; col++) {
ui->streamTreeWidget->resizeColumnToContents(col);
}
ui->audioPlot->replot();
if (rescale_axes) resetXAxis();
updateWidgets();
}
