void UpdateSampleRateIfNeeded(AudioNodeStream* aStream, uint32_t aChannels)
{
if (mPlaybackRateTimeline.HasSimpleValue()) {
mPlaybackRate = mPlaybackRateTimeline.GetValue();
} else {
mPlaybackRate = mPlaybackRateTimeline.GetValueAtTime(aStream->GetCurrentPosition());
}
// Make sure the playback rate and the doppler shift are something
// our resampler can work with.
if (ComputeFinalOutSampleRate(aStream->SampleRate()) == 0) {
mPlaybackRate = 1.0;
mDopplerShift = 1.0;
}
uint32_t currentOutSampleRate, currentInSampleRate;
if (ShouldResample(aStream->SampleRate())) {
SpeexResamplerState* resampler = Resampler(aStream, aChannels);
speex_resampler_get_rate(resampler, ¤tInSampleRate, ¤tOutSampleRate);
uint32_t finalSampleRate = ComputeFinalOutSampleRate(aStream->SampleRate());
if (currentOutSampleRate != finalSampleRate) {
speex_resampler_set_rate(resampler, currentInSampleRate, finalSampleRate);
}
}
}
void AudioSegment::ResampleChunks(SpeexResamplerState* aResampler)
{
uint32_t inRate, outRate;
if (mChunks.IsEmpty()) {
return;
}
speex_resampler_get_rate(aResampler, &inRate, &outRate);
AudioSampleFormat format = AUDIO_FORMAT_SILENCE;
for (ChunkIterator ci(*this); !ci.IsEnded(); ci.Next()) {
if (ci->mBufferFormat != AUDIO_FORMAT_SILENCE) {
format = ci->mBufferFormat;
}
}
switch (format) {
// If the format is silence at this point, all the chunks are silent. The
// actual function we use does not matter, it's just a matter of changing
// the chunks duration.
case AUDIO_FORMAT_SILENCE:
case AUDIO_FORMAT_FLOAT32:
Resample<float>(aResampler, inRate, outRate);
break;
case AUDIO_FORMAT_S16:
Resample<int16_t>(aResampler, inRate, outRate);
break;
default:
MOZ_ASSERT(false);
break;
}
}
static void resample_process_ms2(MSFilter *obj) {
ResampleData *dt=(ResampleData*)obj->data;
mblk_t *m;
if (dt->output_rate==dt->input_rate) {
while((m=ms_queue_get(obj->inputs[0]))!=NULL) {
ms_queue_put(obj->outputs[0],m);
}
return;
}
ms_filter_lock(obj);
if (dt->handle!=NULL) {
unsigned int inrate=0, outrate=0;
speex_resampler_get_rate(dt->handle,&inrate,&outrate);
if (inrate!=dt->input_rate || outrate!=dt->output_rate) {
speex_resampler_destroy(dt->handle);
dt->handle=0;
}
}
if (dt->handle==NULL) {
int err=0;
dt->handle=speex_resampler_init(dt->nchannels, dt->input_rate, dt->output_rate, SPEEX_RESAMPLER_QUALITY_VOIP, &err);
}
while((m=ms_queue_get(obj->inputs[0]))!=NULL) {
unsigned int inlen=(m->b_wptr-m->b_rptr)/(2*dt->nchannels);
unsigned int outlen=((inlen*dt->output_rate)/dt->input_rate)+1;
unsigned int inlen_orig=inlen;
mblk_t *om=allocb(outlen*2*dt->nchannels,0);
if (dt->nchannels==1) {
speex_resampler_process_int(dt->handle,
0,
(int16_t*)m->b_rptr,
&inlen,
(int16_t*)om->b_wptr,
&outlen);
} else {
speex_resampler_process_interleaved_int(dt->handle,
(int16_t*)m->b_rptr,
&inlen,
(int16_t*)om->b_wptr,
&outlen);
}
if (inlen_orig!=inlen) {
ms_error("Bug in resampler ! only %u samples consumed instead of %u, out=%u",
inlen,inlen_orig,outlen);
}
om->b_wptr+=outlen*2*dt->nchannels;
mblk_set_timestamp_info(om,dt->ts);
dt->ts+=outlen;
ms_queue_put(obj->outputs[0],om);
freemsg(m);
}
ms_filter_unlock(obj);
}
size_t rl_resampler_in_samples( rl_resampler_t* resampler, size_t out_samples )
{
if ( resampler != (rl_resampler_t*)&passthrough )
{
SpeexResamplerState* st = (SpeexResamplerState*)resampler;
spx_uint32_t in_rate, out_rate;
speex_resampler_get_rate( st, &in_rate, &out_rate );
return out_samples * in_rate / out_rate;
}
else
{
return out_samples;
}
}
void resample(SpeexResamplerState * resampler, int16_t * buffer, size_t & filled, std::vector<int16_t> & out)
{
uint32_t inputRate, outputRate;
speex_resampler_get_rate(resampler, &inputRate, &outputRate);
uint32_t inlen = filled;
size_t oldSize = out.size();
out.resize(oldSize + (static_cast<int64_t>(inlen) * outputRate / inputRate) + 1);
out.resize(out.capacity());
uint32_t outlen = out.size() - oldSize;
speex_resampler_process_int(resampler, 0, buffer, &inlen, &out[oldSize], &outlen);
out.resize(oldSize + outlen);
// out.insert(out.end(), buffer, buffer + inlen);
memmove(buffer, buffer + inlen, (filled - inlen) * 2);
filled -= inlen;
}
void RtpAudioStream::decode()
{
if (rtp_packets_.size() < 1) return;
// gtk/rtp_player.c:decode_rtp_stream
// XXX This is more messy than it should be.
gsize resample_buff_len = 0x1000;
SAMPLE *resample_buff = (SAMPLE *) g_malloc(resample_buff_len);
spx_uint32_t cur_in_rate = 0, visual_out_rate = 0;
char *write_buff = NULL;
qint64 write_bytes = 0;
unsigned channels = 0;
unsigned sample_rate = 0;
int last_sequence = 0;
double rtp_time_prev = 0.0;
double arrive_time_prev = 0.0;
double pack_period = 0.0;
double start_time = 0.0;
double start_rtp_time = 0.0;
guint32 start_timestamp = 0;
size_t decoded_bytes_prev = 0;
for (int cur_packet = 0; cur_packet < rtp_packets_.size(); cur_packet++) {
SAMPLE *decode_buff = NULL;
// XXX The GTK+ UI updates a progress bar here.
rtp_packet_t *rtp_packet = rtp_packets_[cur_packet];
stop_rel_time_ = start_rel_time_ + rtp_packet->arrive_offset;
speex_resampler_get_rate(visual_resampler_, &cur_in_rate, &visual_out_rate);
QString payload_name;
if (rtp_packet->info->info_payload_type_str) {
payload_name = rtp_packet->info->info_payload_type_str;
} else {
payload_name = try_val_to_str_ext(rtp_packet->info->info_payload_type, &rtp_payload_type_short_vals_ext);
}
if (!payload_name.isEmpty()) {
payload_names_ << payload_name;
}
if (cur_packet < 1) { // First packet
start_timestamp = rtp_packet->info->info_timestamp;
start_rtp_time = 0;
rtp_time_prev = 0;
last_sequence = rtp_packet->info->info_seq_num - 1;
}
size_t decoded_bytes = decode_rtp_packet(rtp_packet, &decode_buff, decoders_hash_, &channels, &sample_rate);
if (decoded_bytes == 0 || sample_rate == 0) {
// We didn't decode anything. Clean up and prep for the next packet.
last_sequence = rtp_packet->info->info_seq_num;
g_free(decode_buff);
continue;
}
if (audio_out_rate_ == 0) { // First non-zero wins
audio_out_rate_ = sample_rate;
RTP_STREAM_DEBUG("Audio sample rate is %u", audio_out_rate_);
// Prepend silence to match our sibling streams.
tempfile_->seek(0);
int prepend_samples = (start_rel_time_ - global_start_rel_time_) * audio_out_rate_;
if (prepend_samples > 0) {
writeSilence(prepend_samples);
}
}
if (rtp_packet->info->info_seq_num != last_sequence+1) {
out_of_seq_timestamps_.append(stop_rel_time_);
}
last_sequence = rtp_packet->info->info_seq_num;
double rtp_time = (double)(rtp_packet->info->info_timestamp-start_timestamp)/sample_rate - start_rtp_time;
double arrive_time;
if (timing_mode_ == RtpTimestamp) {
arrive_time = rtp_time;
} else {
arrive_time = rtp_packet->arrive_offset - start_time;
}
double diff = qAbs(arrive_time - rtp_time);
if (diff*1000 > jitter_buffer_size_ && timing_mode_ != Uninterrupted) {
// rtp_player.c:628
jitter_drop_timestamps_.append(stop_rel_time_);
RTP_STREAM_DEBUG("Packet drop by jitter buffer exceeded %f > %d", diff*1000, jitter_buffer_size_);
/* if there was a silence period (more than two packetization period) resync the source */
if ((rtp_time - rtp_time_prev) > pack_period*2) {
int silence_samples;
RTP_STREAM_DEBUG("Resync...");
silence_samples = (int)((arrive_time - arrive_time_prev)*sample_rate - decoded_bytes_prev / sample_bytes_);
/* Fix for bug 4119/5902: don't insert too many silence frames.
* XXX - is there a better thing to do here?
*/
silence_samples = qMin(silence_samples, max_silence_samples_);
writeSilence(silence_samples);
silence_timestamps_.append(stop_rel_time_);
decoded_bytes_prev = 0;
/* defined start_timestmp to avoid overflow in timestamp. TODO: handle the timestamp correctly */
/* XXX: if timestamps (RTP) are missing/ignored try use packet arrive time only (see also "rtp_time") */
start_timestamp = rtp_packet->info->info_timestamp;
start_rtp_time = 0;
start_time = rtp_packet->arrive_offset;
rtp_time_prev = 0;
}
} else {
// rtp_player.c:664
/* Add silence if it is necessary */
int silence_samples;
if (timing_mode_ == Uninterrupted) {
silence_samples = 0;
} else {
silence_samples = (int)((rtp_time - rtp_time_prev)*sample_rate - decoded_bytes_prev / sample_bytes_);
}
if (silence_samples != 0) {
wrong_timestamp_timestamps_.append(stop_rel_time_);
}
if (silence_samples > 0) {
/* Fix for bug 4119/5902: don't insert too many silence frames.
* XXX - is there a better thing to do here?
*/
silence_samples = qMin(silence_samples, max_silence_samples_);
writeSilence(silence_samples);
silence_timestamps_.append(stop_rel_time_);
}
// XXX rtp_player.c:696 adds audio here.
rtp_time_prev = rtp_time;
pack_period = (double) decoded_bytes / sample_bytes_ / sample_rate;
decoded_bytes_prev = decoded_bytes;
arrive_time_prev = arrive_time;
}
// Write samples to our file.
write_buff = (char *) decode_buff;
write_bytes = decoded_bytes;
if (audio_out_rate_ != sample_rate) {
// Resample the audio to match our previous output rate.
if (!audio_resampler_) {
audio_resampler_ = speex_resampler_init(1, sample_rate, audio_out_rate_, 10, NULL);
speex_resampler_skip_zeros(audio_resampler_);
RTP_STREAM_DEBUG("Started resampling from %u to (out) %u Hz.", sample_rate, audio_out_rate_);
} else {
spx_uint32_t audio_out_rate;
speex_resampler_get_rate(audio_resampler_, &cur_in_rate, &audio_out_rate);
// Adjust rates if needed.
if (sample_rate != cur_in_rate) {
speex_resampler_set_rate(audio_resampler_, sample_rate, audio_out_rate);
speex_resampler_set_rate(visual_resampler_, sample_rate, visual_out_rate);
RTP_STREAM_DEBUG("Changed input rate from %u to %u Hz. Out is %u.", cur_in_rate, sample_rate, audio_out_rate_);
}
}
spx_uint32_t in_len = (spx_uint32_t)rtp_packet->info->info_payload_len;
spx_uint32_t out_len = (audio_out_rate_ * (spx_uint32_t)rtp_packet->info->info_payload_len / sample_rate) + (audio_out_rate_ % sample_rate != 0);
if (out_len * sample_bytes_ > resample_buff_len) {
while ((out_len * sample_bytes_ > resample_buff_len))
resample_buff_len *= 2;
resample_buff = (SAMPLE *) g_realloc(resample_buff, resample_buff_len);
}
speex_resampler_process_int(audio_resampler_, 0, decode_buff, &in_len, resample_buff, &out_len);
write_buff = (char *) decode_buff;
write_bytes = out_len * sample_bytes_;
}
// Write the decoded, possibly-resampled audio to our temp file.
tempfile_->write(write_buff, write_bytes);
// Collect our visual samples.
spx_uint32_t in_len = (spx_uint32_t)rtp_packet->info->info_payload_len;
spx_uint32_t out_len = (visual_out_rate * in_len / sample_rate) + (visual_out_rate % sample_rate != 0);
if (out_len * sample_bytes_ > resample_buff_len) {
while ((out_len * sample_bytes_ > resample_buff_len))
resample_buff_len *= 2;
resample_buff = (SAMPLE *) g_realloc(resample_buff, resample_buff_len);
}
speex_resampler_process_int(visual_resampler_, 0, decode_buff, &in_len, resample_buff, &out_len);
for (unsigned i = 0; i < out_len; i++) {
packet_timestamps_[stop_rel_time_ + (double) i / visual_out_rate] = rtp_packet->frame_num;
if (qAbs(resample_buff[i]) > max_sample_val_) max_sample_val_ = qAbs(resample_buff[i]);
visual_samples_.append(resample_buff[i]);
}
// Finally, write the resampled audio to our temp file and clean up.
g_free(decode_buff);
}
g_free(resample_buff);
}
static void resample_process_ms2(MSFilter *obj){
ResampleData *dt=(ResampleData*)obj->data;
mblk_t *im, *om = NULL, *om_chan = NULL;
if (dt->output_rate==dt->input_rate){
while((im=ms_queue_get(obj->inputs[0]))!=NULL){
if (resample_channel_adapt(dt->in_nchannels, dt->out_nchannels, im, &om) == 0) {
ms_queue_put(obj->outputs[0], im);
} else {
ms_queue_put(obj->outputs[0], om);
freemsg(im);
}
}
return;
}
ms_filter_lock(obj);
if (dt->handle!=NULL){
unsigned int inrate=0, outrate=0;
speex_resampler_get_rate(dt->handle,&inrate,&outrate);
if (inrate!=dt->input_rate || outrate!=dt->output_rate){
speex_resampler_destroy(dt->handle);
dt->handle=0;
}
}
if (dt->handle==NULL){
resample_init_speex(dt);
}
while((im=ms_queue_get(obj->inputs[0]))!=NULL){
unsigned int inlen=(im->b_wptr-im->b_rptr)/(2*dt->in_nchannels);
unsigned int outlen=((inlen*dt->output_rate)/dt->input_rate)+1;
unsigned int inlen_orig=inlen;
om=allocb(outlen*2*dt->in_nchannels,0);
mblk_meta_copy(im, om);
if (dt->in_nchannels==1){
speex_resampler_process_int(dt->handle,
0,
(int16_t*)im->b_rptr,
&inlen,
(int16_t*)om->b_wptr,
&outlen);
}else{
speex_resampler_process_interleaved_int(dt->handle,
(int16_t*)im->b_rptr,
&inlen,
(int16_t*)om->b_wptr,
&outlen);
}
if (inlen_orig!=inlen){
ms_error("Bug in resampler ! only %u samples consumed instead of %u, out=%u",
inlen,inlen_orig,outlen);
}
om->b_wptr+=outlen*2*dt->in_nchannels;
mblk_set_timestamp_info(om,dt->ts);
dt->ts+=outlen;
if (resample_channel_adapt(dt->in_nchannels, dt->out_nchannels, om, &om_chan) == 0) {
ms_queue_put(obj->outputs[0], om);
} else {
ms_queue_put(obj->outputs[0], om_chan);
freemsg(om);
}
freemsg(im);
}
ms_filter_unlock(obj);
}
