void JackAudioSystem::initializeOutput() {
QMutexLocker lock(&qmWait);
if (!jasys->bJackIsGood) {
return;
}
AudioOutputPtr ao = g.ao;
JackAudioOutput * const jao = dynamic_cast<JackAudioOutput *>(ao.get());
allocOutputBuffer(iBufferSize);
if (jao) {
jao->qmMutex.lock();
}
for (unsigned int i = 0; i < iOutPorts; ++i) {
char name[10];
snprintf(name, 10, "output_%d", i + 1);
out_ports[i] = jack_port_register(client, name, JACK_DEFAULT_AUDIO_TYPE, JackPortIsOutput, 0);
if (out_ports[i] == NULL) {
qWarning("JackAudioSystem: unable to register 'output' port");
break;
}
}
bOutputIsGood = true;
if (jao) {
jao->qmMutex.unlock();
}
}
void JackAudioSystem::destroyOutput() {
AudioOutputPtr ao = g.ao;
JackAudioOutput * const jao = dynamic_cast<JackAudioOutput *>(ao.get());
if (jao) {
jao->qmMutex.lock();
}
delete [] output_buffer;
output_buffer = NULL;
for (unsigned int i = 0; i < iOutPorts; ++i) {
if (out_ports[i] != NULL) {
int err = jack_port_unregister(client, out_ports[i]);
if (err != 0) {
qWarning("JackAudioSystem: unable to unregister out port - jack_port_unregister() returned %i", err);
}
out_ports[i] = NULL;
}
}
bOutputIsGood = false;
if (jao) {
jao->qmMutex.unlock();
}
}
void LoopPlayer::addFrame(const QByteArray &packet, int seq) {
if (DOUBLE_RAND < g.s.dPacketLoss) {
qWarning("Drop");
return;
}
bool restart = (qtLastFetch.elapsed() > 100);
{
QMutexLocker l(&qmLock);
double time = qtTicker.elapsed();
double r;
if (restart)
r = 0.0;
else
r = DOUBLE_RAND * g.s.dMaxPacketDelay;
qmPackets.insert(static_cast<float>(time + r), Packet(seq, packet));
}
// Restart check
if (qtLastFetch.elapsed() > 100) {
AudioOutputPtr ao = g.ao;
if (ao) {
ao->addFrameToBuffer(this, QByteArray(), 0);
}
}
}
void ClientUser::remove(unsigned int uiSession) {
QWriteLocker lock(&c_qrwlUsers);
ClientUser *p = c_qmUsers.take(uiSession);
if (p && p->cChannel)
p->cChannel->removeUser(p);
if (p) {
AudioOutputPtr ao = g.ao;
if (ao)
ao->removeBuffer(p);
}
}
void ClientUser::remove(unsigned int uiSession) {
QWriteLocker lock(&c_qrwlUsers);
ClientUser *p = c_qmUsers.take(uiSession);
if (p) {
if (p->cChannel)
p->cChannel->removeUser(p);
AudioOutputPtr ao = g.ao;
if (ao)
ao->removeBuffer(p);
if (p->tsState != Settings::Passive) {
QWriteLocker writeLock(&c_qrwlTalking);
c_qlTalking.removeAll(p);
}
}
}
void LoopPlayer::fetchFrames() {
QMutexLocker l(&qmLock);
AudioOutputPtr ao = g.ao;
if (!ao || qmPackets.isEmpty())
return;
double cmp = qtTicker.elapsed();
QMultiMap<float, Packet>::iterator i = qmPackets.begin();
while (i != qmPackets.end()) {
if (i.key() > cmp)
break;
ao->addFrameToBuffer(this, i.value().second, i.value().first);
i = qmPackets.erase(i);
}
qtLastFetch.restart();
}
void JackAudioSystem::allocOutputBuffer(jack_nframes_t frames) {
iBufferSize = frames;
AudioOutputPtr ao = g.ao;
JackAudioOutput * const jao = dynamic_cast<JackAudioOutput *>(ao.get());
if (jao) {
jao->qmMutex.lock();
}
if (output_buffer) {
delete [] output_buffer;
output_buffer = NULL;
}
output_buffer = new jack_default_audio_sample_t[frames * iOutPorts];
if (output_buffer == NULL) {
bJackIsGood = false;
}
if (jao) {
jao->qmMutex.unlock();
}
}
//加入到音频播放队列
void LoopUser::addFrame(const QByteArray &packet) {
if (DOUBLE_RAND < g_struct.s.dPacketLoss) {
Trace("Drop");
return;
}
bool restart = (qtLastFetch.elapsed()*CLOCKS_PER_SEC > 100);
{
MutexLocker l(&qmLock);
double time = qtTicker.elapsed();
double r;
if (restart)
r = 0.0;
else
r = DOUBLE_RAND * g_struct.s.dMaxPacketDelay;
qmPackets.insert(make_pair(static_cast<float>(time + r), packet));
}
// Restart check
if (qtLastFetch.elapsed()*CLOCKS_PER_SEC > 100) {
AudioOutputPtr ao = g_struct.ao;
if (ao) {
MessageHandler::UDPMessageType msgType = MessageHandler::UDPVoiceAACPlus/*MessageHandler::UDPVoiceCELT*/;//static_cast<MessageHandler::UDPMessageType>((packet.at(0) >> 5) & 0x7);
for (int i=0; i<PER_FRAME_OF_SAMEPLE; i++)
{
ao->addFrameToBuffer(this, QByteArray(), 0, msgType);
char buf[512] = {0};
sprintf(buf,"LoopUser::addFrame QByteArray() t=%d\n",qtLastFetch.elapsed()*CLOCKS_PER_SEC );
//OutputDebugStringA(buf);
}
}
}
}
//从音频队列中取帧
void LoopUser::fetchFrames() {
MutexLocker l(&qmLock);
AudioOutputPtr ao = g_struct.ao;
if (!ao || qmPackets.empty()) {
return;
}
double cmp = qtTicker.elapsed();
std::multimap<float, QByteArray>::iterator i = qmPackets.begin();
while (i != qmPackets.end()) {
// if (i->first > cmp)
// {
// char buf[512] = {0};
// sprintf(buf,"LoopUser::fetchFrames() cmp=%f\n",i->first );
// OutputDebugStringA(buf);
// break;
// }
const QByteArray &data = i->second;
PacketDataStream pds((char*)&data[0], data.size());
unsigned int msgFlags = 0;
int iSeq = static_cast<unsigned int>(pds.next8());
QByteArray qba;
pds.dataBlock(pds.left(), qba);
ao->addFrameToBuffer(this, qba, iSeq, MessageHandler::UDPVoiceAACPlus/*MessageHandler::UDPVoiceCELT*/);
i = qmPackets.erase(i);
}
qtLastFetch.restart();
}
void PulseAudioSystem::write_callback(pa_stream *s, size_t bytes, void *userdata) {
PulseAudioSystem *pas = reinterpret_cast<PulseAudioSystem *>(userdata);
Q_ASSERT(s == pas->pasOutput);
AudioOutputPtr ao = g.ao;
PulseAudioOutput *pao = dynamic_cast<PulseAudioOutput *>(ao.get());
unsigned char buffer[bytes];
if (! pao) {
// Transitioning, but most likely transitions back, so just zero.
memset(buffer, 0, bytes);
pa_stream_write(s, buffer, bytes, NULL, 0, PA_SEEK_RELATIVE);
pas->wakeup();
return;
}
const pa_sample_spec *pss = pa_stream_get_sample_spec(s);
const pa_channel_map *pcm = pa_stream_get_channel_map(pas->pasOutput);
if (!pa_sample_spec_equal(pss, &pao->pss) || !pa_channel_map_equal(pcm, &pao->pcm)) {
pao->pss = *pss;
pao->pcm = *pcm;
if (pss->format == PA_SAMPLE_FLOAT32NE)
pao->eSampleFormat = PulseAudioOutput::SampleFloat;
else
pao->eSampleFormat = PulseAudioOutput::SampleShort;
pao->iMixerFreq = pss->rate;
pao->iChannels = pss->channels;
unsigned int chanmasks[pss->channels];
for (int i=0;i<pss->channels;++i) {
unsigned int cm = 0;
switch (pcm->map[i]) {
case PA_CHANNEL_POSITION_LEFT:
cm = SPEAKER_FRONT_LEFT;
break;
case PA_CHANNEL_POSITION_RIGHT:
cm = SPEAKER_FRONT_RIGHT;
break;
case PA_CHANNEL_POSITION_CENTER:
cm = SPEAKER_FRONT_CENTER;
break;
case PA_CHANNEL_POSITION_REAR_LEFT:
cm = SPEAKER_BACK_LEFT;
break;
case PA_CHANNEL_POSITION_REAR_RIGHT:
cm = SPEAKER_BACK_RIGHT;
break;
case PA_CHANNEL_POSITION_REAR_CENTER:
cm = SPEAKER_BACK_CENTER;
break;
case PA_CHANNEL_POSITION_LFE:
cm = SPEAKER_LOW_FREQUENCY;
break;
case PA_CHANNEL_POSITION_SIDE_LEFT:
cm = SPEAKER_SIDE_LEFT;
break;
case PA_CHANNEL_POSITION_SIDE_RIGHT:
cm = SPEAKER_SIDE_RIGHT;
break;
case PA_CHANNEL_POSITION_FRONT_LEFT_OF_CENTER:
cm = SPEAKER_FRONT_LEFT_OF_CENTER;
break;
case PA_CHANNEL_POSITION_FRONT_RIGHT_OF_CENTER:
cm = SPEAKER_FRONT_RIGHT_OF_CENTER;
break;
default:
cm = 0;
break;
}
chanmasks[i] = cm;
}
pao->initializeMixer(chanmasks);
}
const unsigned int iSampleSize = pao->iSampleSize;
const unsigned int samples = static_cast<unsigned int>(bytes) / iSampleSize;
bool oldAttenuation = pas->bAttenuating;
// do we have some mixed output?
if (pao->mix(buffer, samples)) {
// attenuate if instructed to or it's in settings
pas->bAttenuating = (g.bAttenuateOthers || g.s.bAttenuateOthers);
} else {
memset(buffer, 0, bytes);
// attenuate if intructed to (self-activated)
pas->bAttenuating = g.bAttenuateOthers;
}
// if the attenuation state has changed
if (oldAttenuation != pas->bAttenuating) {
pas->setVolumes();
}
pa_stream_write(s, buffer, iSampleSize * samples, NULL, 0, PA_SEEK_RELATIVE);
}
void PulseAudioSystem::eventCallback(pa_mainloop_api *api, pa_defer_event *) {
api->defer_enable(pade, false);
if (! bSourceDone || ! bSinkDone || ! bServerDone)
return;
AudioInputPtr ai = g.ai;
AudioOutputPtr ao = g.ao;
AudioInput *raw_ai = ai.get();
AudioOutput *raw_ao = ao.get();
PulseAudioInput *pai = dynamic_cast<PulseAudioInput *>(raw_ai);
PulseAudioOutput *pao = dynamic_cast<PulseAudioOutput *>(raw_ao);
if (raw_ao) {
QString odev = outputDevice();
pa_stream_state ost = pasOutput ? pa_stream_get_state(pasOutput) : PA_STREAM_TERMINATED;
bool do_stop = false;
bool do_start = false;
if (! pao && (ost == PA_STREAM_READY)) {
do_stop = true;
} else if (pao) {
switch (ost) {
case PA_STREAM_TERMINATED: {
if (pasOutput)
pa_stream_unref(pasOutput);
pa_sample_spec pss = qhSpecMap.value(odev);
pa_channel_map pcm = qhChanMap.value(odev);
if ((pss.format != PA_SAMPLE_FLOAT32NE) && (pss.format != PA_SAMPLE_S16NE))
pss.format = PA_SAMPLE_FLOAT32NE;
if (pss.rate == 0)
pss.rate = SAMPLE_RATE;
if ((pss.channels == 0) || (! g.s.doPositionalAudio()))
pss.channels = 1;
pasOutput = pa_stream_new(pacContext, mumble_sink_input, &pss, (pss.channels == 1) ? NULL : &pcm);
pa_stream_set_state_callback(pasOutput, stream_callback, this);
pa_stream_set_write_callback(pasOutput, write_callback, this);
}
case PA_STREAM_UNCONNECTED:
do_start = true;
break;
case PA_STREAM_READY: {
if (g.s.iOutputDelay != iDelayCache) {
do_stop = true;
} else if (g.s.doPositionalAudio() != bPositionalCache) {
do_stop = true;
} else if (odev != qsOutputCache) {
do_stop = true;
}
break;
}
default:
break;
}
}
if (do_stop) {
qWarning("PulseAudio: Stopping output");
pa_stream_disconnect(pasOutput);
iSinkId = -1;
} else if (do_start) {
qWarning("PulseAudio: Starting output: %s", qPrintable(odev));
pa_buffer_attr buff;
const pa_sample_spec *pss = pa_stream_get_sample_spec(pasOutput);
const size_t sampleSize = (pss->format == PA_SAMPLE_FLOAT32NE) ? sizeof(float) : sizeof(short);
const unsigned int iBlockLen = ((pao->iFrameSize * pss->rate) / SAMPLE_RATE) * pss->channels * static_cast<unsigned int>(sampleSize);
buff.tlength = iBlockLen * (g.s.iOutputDelay+1);
buff.minreq = iBlockLen;
buff.maxlength = -1;
buff.prebuf = -1;
buff.fragsize = iBlockLen;
iDelayCache = g.s.iOutputDelay;
bPositionalCache = g.s.doPositionalAudio();
qsOutputCache = odev;
pa_stream_connect_playback(pasOutput, qPrintable(odev), &buff, PA_STREAM_ADJUST_LATENCY, NULL, NULL);
pa_context_get_sink_info_by_name(pacContext, qPrintable(odev), sink_info_callback, this);
}
}
if (raw_ai) {
QString idev = inputDevice();
pa_stream_state ist = pasInput ? pa_stream_get_state(pasInput) : PA_STREAM_TERMINATED;
bool do_stop = false;
bool do_start = false;
if (! pai && (ist == PA_STREAM_READY)) {
do_stop = true;
} else if (pai) {
switch (ist) {
case PA_STREAM_TERMINATED: {
if (pasInput)
pa_stream_unref(pasInput);
pa_sample_spec pss = qhSpecMap.value(idev);
if ((pss.format != PA_SAMPLE_FLOAT32NE) && (pss.format != PA_SAMPLE_S16NE))
pss.format = PA_SAMPLE_FLOAT32NE;
if (pss.rate == 0)
pss.rate = SAMPLE_RATE;
pss.channels = 1;
pasInput = pa_stream_new(pacContext, "Microphone", &pss, NULL);
pa_stream_set_state_callback(pasInput, stream_callback, this);
pa_stream_set_read_callback(pasInput, read_callback, this);
}
case PA_STREAM_UNCONNECTED:
do_start = true;
break;
case PA_STREAM_READY: {
if (idev != qsInputCache) {
do_stop = true;
}
break;
}
default:
break;
}
}
if (do_stop) {
qWarning("PulseAudio: Stopping input");
pa_stream_disconnect(pasInput);
} else if (do_start) {
qWarning("PulseAudio: Starting input %s",qPrintable(idev));
pa_buffer_attr buff;
const pa_sample_spec *pss = pa_stream_get_sample_spec(pasInput);
const size_t sampleSize = (pss->format == PA_SAMPLE_FLOAT32NE) ? sizeof(float) : sizeof(short);
const unsigned int iBlockLen = ((pai->iFrameSize * pss->rate) / SAMPLE_RATE) * pss->channels * static_cast<unsigned int>(sampleSize);
buff.tlength = iBlockLen;
buff.minreq = iBlockLen;
buff.maxlength = -1;
buff.prebuf = -1;
buff.fragsize = iBlockLen;
qsInputCache = idev;
pa_stream_connect_record(pasInput, qPrintable(idev), &buff, PA_STREAM_ADJUST_LATENCY);
}
}
if (raw_ai) {
QString odev = outputDevice();
QString edev = qhEchoMap.value(odev);
pa_stream_state est = pasSpeaker ? pa_stream_get_state(pasSpeaker) : PA_STREAM_TERMINATED;
bool do_stop = false;
bool do_start = false;
if ((! pai || ! g.s.doEcho()) && (est == PA_STREAM_READY)) {
do_stop = true;
} else if (pai && g.s.doEcho()) {
switch (est) {
case PA_STREAM_TERMINATED: {
if (pasSpeaker)
pa_stream_unref(pasSpeaker);
pa_sample_spec pss = qhSpecMap.value(edev);
pa_channel_map pcm = qhChanMap.value(edev);
if ((pss.format != PA_SAMPLE_FLOAT32NE) && (pss.format != PA_SAMPLE_S16NE))
pss.format = PA_SAMPLE_FLOAT32NE;
if (pss.rate == 0)
pss.rate = SAMPLE_RATE;
if ((pss.channels == 0) || (! g.s.bEchoMulti))
pss.channels = 1;
pasSpeaker = pa_stream_new(pacContext, mumble_echo, &pss, (pss.channels == 1) ? NULL : &pcm);
pa_stream_set_state_callback(pasSpeaker, stream_callback, this);
pa_stream_set_read_callback(pasSpeaker, read_callback, this);
}
case PA_STREAM_UNCONNECTED:
do_start = true;
break;
case PA_STREAM_READY: {
if (g.s.bEchoMulti != bEchoMultiCache) {
do_stop = true;
} else if (edev != qsEchoCache) {
do_stop = true;
}
break;
}
default:
break;
}
}
if (do_stop) {
qWarning("PulseAudio: Stopping echo");
pa_stream_disconnect(pasSpeaker);
} else if (do_start) {
qWarning("PulseAudio: Starting echo: %s", qPrintable(edev));
pa_buffer_attr buff;
const pa_sample_spec *pss = pa_stream_get_sample_spec(pasSpeaker);
const size_t sampleSize = (pss->format == PA_SAMPLE_FLOAT32NE) ? sizeof(float) : sizeof(short);
const unsigned int iBlockLen = ((pai->iFrameSize * pss->rate) / SAMPLE_RATE) * pss->channels * static_cast<unsigned int>(sampleSize);
buff.tlength = iBlockLen;
buff.minreq = iBlockLen;
buff.maxlength = -1;
buff.prebuf = -1;
buff.fragsize = iBlockLen;
bEchoMultiCache = g.s.bEchoMulti;
qsEchoCache = edev;
pa_stream_connect_record(pasSpeaker, qPrintable(edev), &buff, PA_STREAM_ADJUST_LATENCY);
}
}
}
void AudioInput::encodeAudioFrame() {
int iArg;
int i;
float sum;
short max;
short *psSource;
iFrameCounter++;
if (! bRunning)
return;
sum=1.0f;
max = 1;
for (i=0;i<iFrameSize;i++) {
sum += static_cast<float>(psMic[i] * psMic[i]);
max = std::max(static_cast<short>(abs(psMic[i])), max);
}
dPeakMic = qMax(20.0f*log10f(sqrtf(sum / static_cast<float>(iFrameSize)) / 32768.0f), -96.0f);
dMaxMic = max;
if (psSpeaker && (iEchoChannels > 0)) {
sum=1.0f;
for (i=0;i<iFrameSize;i++)
sum += static_cast<float>(psSpeaker[i] * psSpeaker[i]);
dPeakSpeaker = qMax(20.0f*log10f(sqrtf(sum / static_cast<float>(iFrameSize)) / 32768.0f), -96.0f);
} else {
dPeakSpeaker = 0.0;
}
QMutexLocker l(&qmSpeex);
resetAudioProcessor();
speex_preprocess_ctl(sppPreprocess, SPEEX_PREPROCESS_GET_AGC_GAIN, &iArg);
float gainValue = static_cast<float>(iArg);
iArg = g.s.iNoiseSuppress - iArg;
speex_preprocess_ctl(sppPreprocess, SPEEX_PREPROCESS_SET_NOISE_SUPPRESS, &iArg);
if (sesEcho && psSpeaker) {
speex_echo_cancellation(sesEcho, psMic, psSpeaker, psClean);
speex_preprocess_run(sppPreprocess, psClean);
psSource = psClean;
} else {
speex_preprocess_run(sppPreprocess, psMic);
psSource = psMic;
}
sum=1.0f;
for (i=0;i<iFrameSize;i++)
sum += static_cast<float>(psSource[i] * psSource[i]);
float micLevel = sqrtf(sum / static_cast<float>(iFrameSize));
dPeakSignal = qMax(20.0f*log10f(micLevel / 32768.0f), -96.0f);
spx_int32_t prob = 0;
speex_preprocess_ctl(sppPreprocess, SPEEX_PREPROCESS_GET_PROB, &prob);
fSpeechProb = static_cast<float>(prob) / 100.0f;
// clean microphone level: peak of filtered signal attenuated by AGC gain
dPeakCleanMic = qMax(dPeakSignal - gainValue, -96.0f);
float level = (g.s.vsVAD == Settings::SignalToNoise) ? fSpeechProb : (1.0f + dPeakCleanMic / 96.0f);
bool bIsSpeech = false;
if (level > g.s.fVADmax)
bIsSpeech = true;
else if (level > g.s.fVADmin && bPreviousVoice)
bIsSpeech = true;
if (! bIsSpeech) {
iHoldFrames++;
if (iHoldFrames < g.s.iVoiceHold)
bIsSpeech = true;
} else {
iHoldFrames = 0;
}
if (g.s.atTransmit == Settings::Continuous)
bIsSpeech = true;
else if (g.s.atTransmit == Settings::PushToTalk)
bIsSpeech = g.s.uiDoublePush && ((g.uiDoublePush < g.s.uiDoublePush) || (g.tDoublePush.elapsed() < g.s.uiDoublePush));
bIsSpeech = bIsSpeech || (g.iPushToTalk > 0);
ClientUser *p = ClientUser::get(g.uiSession);
if (g.s.bMute || ((g.s.lmLoopMode != Settings::Local) && p && (p->bMute || p->bSuppress)) || g.bPushToMute || (g.iTarget < 0)) {
bIsSpeech = false;
}
if (bIsSpeech) {
iSilentFrames = 0;
} else {
iSilentFrames++;
if (iSilentFrames > 500)
iFrameCounter = 0;
}
if (p) {
if (! bIsSpeech)
p->setTalking(Settings::Passive);
else if (g.iTarget == 0)
p->setTalking(Settings::Talking);
else
p->setTalking(Settings::Shouting);
}
if (g.s.bTxAudioCue && g.uiSession != 0) {
AudioOutputPtr ao = g.ao;
if (bIsSpeech && ! bPreviousVoice && ao)
ao->playSample(g.s.qsTxAudioCueOn);
else if (ao && !bIsSpeech && bPreviousVoice)
ao->playSample(g.s.qsTxAudioCueOff);
}
if (! bIsSpeech && ! bPreviousVoice) {
iBitrate = 0;
if (g.s.iaeIdleAction != Settings::Nothing && ((tIdle.elapsed() / 1000000ULL) > g.s.iIdleTime)) {
if (g.s.iaeIdleAction == Settings::Deafen && !g.s.bDeaf) {
tIdle.restart();
emit doDeaf();
} else if (g.s.iaeIdleAction == Settings::Mute && !g.s.bMute) {
tIdle.restart();
emit doMute();
}
}
spx_int32_t increment = 0;
speex_preprocess_ctl(sppPreprocess, SPEEX_PREPROCESS_SET_AGC_INCREMENT, &increment);
return;
} else {
spx_int32_t increment = 12;
speex_preprocess_ctl(sppPreprocess, SPEEX_PREPROCESS_SET_AGC_INCREMENT, &increment);
}
if (bIsSpeech && !bPreviousVoice) {
bResetEncoder = true;
}
tIdle.restart();
EncodingOutputBuffer buffer;
Q_ASSERT(buffer.size() >= static_cast<size_t>(iAudioQuality / 100 * iAudioFrames / 8));
int len = 0;
bool encoded = true;
if (!selectCodec())
return;
if (umtType == MessageHandler::UDPVoiceCELTAlpha || umtType == MessageHandler::UDPVoiceCELTBeta) {
len = encodeCELTFrame(psSource, buffer);
if (len <= 0) {
iBitrate = 0;
qWarning() << "encodeCELTFrame failed" << iBufferedFrames << iFrameSize << len;
return;
}
++iBufferedFrames;
} else if (umtType == MessageHandler::UDPVoiceOpus) {
encoded = false;
opusBuffer.insert(opusBuffer.end(), psSource, psSource + iFrameSize);
++iBufferedFrames;
if (!bIsSpeech || iBufferedFrames >= iAudioFrames) {
if (iBufferedFrames < iAudioFrames) {
// Stuff frame to framesize if speech ends and we don't have enough audio
// this way we are guaranteed to have a valid framecount and won't cause
// a codec configuration switch by suddenly using a wildly different
// framecount per packet.
const int missingFrames = iAudioFrames - iBufferedFrames;
opusBuffer.insert(opusBuffer.end(), iFrameSize * missingFrames, 0);
iBufferedFrames += missingFrames;
iFrameCounter += missingFrames;
}
Q_ASSERT(iBufferedFrames == iAudioFrames);
len = encodeOpusFrame(&opusBuffer[0], iBufferedFrames * iFrameSize, buffer);
opusBuffer.clear();
if (len <= 0) {
iBitrate = 0;
qWarning() << "encodeOpusFrame failed" << iBufferedFrames << iFrameSize << len;
iBufferedFrames = 0; // These are lost. Make sure not to mess up our sequence counter next flushCheck.
return;
}
encoded = true;
}
}
if (encoded) {
flushCheck(QByteArray(reinterpret_cast<char *>(&buffer[0]), len), !bIsSpeech);
}
if (! bIsSpeech)
iBitrate = 0;
bPreviousVoice = bIsSpeech;
}
void AudioInput::encodeAudioFrame() {
int iArg;
ClientPlayer *p=ClientPlayer::get(g.uiSession);
int i;
float sum;
short max;
short *psSource;
iFrameCounter++;
if (! bRunning) {
return;
}
sum=1.0f;
for (i=0;i<iFrameSize;i++)
sum += static_cast<float>(psMic[i] * psMic[i]);
dPeakMic=20.0f*log10f(sqrtf(sum / static_cast<float>(iFrameSize)) / 32768.0f);
if (dPeakMic < -96.0f)
dPeakMic = -96.0f;
max = 1;
for (i=0;i<iFrameSize;i++)
max = static_cast<short>(abs(psMic[i]) > max ? abs(psMic[i]) : max);
dMaxMic = max;
if (g.bEchoTest) {
STACKVAR(float, fft, iFrameSize);
STACKVAR(float, power, iFrameSize);
float scale = 1.f / static_cast<float>(iFrameSize);
for (i=0;i<iFrameSize;i++)
fft[i] = static_cast<float>(psMic[i]) * scale;
mumble_drft_forward(&fftTable, fft);
float mp = 0.0f;
int bin = 0;
power[0]=power[1]=0.0f;
for (i=2;i < iFrameSize / 2;i++) {
power[i] = sqrtf(fft[2*i]*fft[2*i]+fft[2*i-1]*fft[2*i-1]);
if (power[i] > mp) {
bin = i;
mp = power[i];
}
}
for (i=2;i< iFrameSize / 2;i++) {
if (power[i] * 2 > mp) {
if (i != bin)
bin = 0;
}
}
iBestBin = bin * 2;
}
if (iEchoChannels > 0) {
sum=1.0f;
for (i=0;i<iFrameSize;i++)
sum += static_cast<float>(psSpeaker[i] * psSpeaker[i]);
dPeakSpeaker=20.0f*log10f(sqrtf(sum / static_cast<float>(iFrameSize)) / 32768.0f);
if (dPeakSpeaker < -96.0f)
dPeakSpeaker = -96.0f;
} else {
dPeakSpeaker = 0.0;
}
QMutexLocker l(&qmSpeex);
if (bResetProcessor) {
if (sppPreprocess)
speex_preprocess_state_destroy(sppPreprocess);
if (sesEcho)
speex_echo_state_destroy(sesEcho);
sppPreprocess = speex_preprocess_state_init(iFrameSize, SAMPLE_RATE);
iArg = 1;
speex_preprocess_ctl(sppPreprocess, SPEEX_PREPROCESS_SET_VAD, &iArg);
speex_preprocess_ctl(sppPreprocess, SPEEX_PREPROCESS_SET_DENOISE, &iArg);
speex_preprocess_ctl(sppPreprocess, SPEEX_PREPROCESS_SET_AGC, &iArg);
speex_preprocess_ctl(sppPreprocess, SPEEX_PREPROCESS_SET_DEREVERB, &iArg);
iArg = 30000;
speex_preprocess_ctl(sppPreprocess, SPEEX_PREPROCESS_SET_AGC_TARGET, &iArg);
float v = 30000.0f / static_cast<float>(g.s.iMinLoudness);
iArg = lroundf(floorf(20.0f * log10f(v)));
speex_preprocess_ctl(sppPreprocess, SPEEX_PREPROCESS_SET_AGC_MAX_GAIN, &iArg);
iArg = g.s.iNoiseSuppress;
speex_preprocess_ctl(sppPreprocess, SPEEX_PREPROCESS_SET_NOISE_SUPPRESS, &iArg);
if (iEchoChannels > 0) {
sesEcho = speex_echo_state_init(iFrameSize, iFrameSize*10);
iArg = SAMPLE_RATE;
speex_echo_ctl(sesEcho, SPEEX_SET_SAMPLING_RATE, &iArg);
speex_preprocess_ctl(sppPreprocess, SPEEX_PREPROCESS_SET_ECHO_STATE, sesEcho);
jitter_buffer_reset(jb);
qWarning("AudioInput: ECHO CANCELLER ACTIVE");
} else {
sesEcho = NULL;
}
iFrames = 0;
speex_bits_reset(&sbBits);
bResetProcessor = false;
}
int iIsSpeech;
if (sesEcho) {
speex_echo_cancellation(sesEcho, psMic, psSpeaker, psClean);
iIsSpeech=speex_preprocess_run(sppPreprocess, psClean);
psSource = psClean;
} else {
iIsSpeech=speex_preprocess_run(sppPreprocess, psMic);
psSource = psMic;
}
sum=1.0f;
for (i=0;i<iFrameSize;i++)
sum += static_cast<float>(psSource[i] * psSource[i]);
float micLevel = sqrtf(sum / static_cast<float>(iFrameSize));
dPeakSignal=20.0f*log10f(micLevel / 32768.0f);
if (dPeakSignal < -96.0f)
dPeakSignal = -96.0f;
spx_int32_t prob = 0;
speex_preprocess_ctl(sppPreprocess, SPEEX_PREPROCESS_GET_PROB, &prob);
fSpeechProb = static_cast<float>(prob) / 100.0f;
float level = (g.s.vsVAD == Settings::SignalToNoise) ? fSpeechProb : (1.0f + dPeakMic / 96.0f);
if (level > g.s.fVADmax)
iIsSpeech = 1;
else if (level > g.s.fVADmin && bPreviousVoice)
iIsSpeech = 1;
else
iIsSpeech = 0;
if (! iIsSpeech) {
iHoldFrames++;
if (iHoldFrames < g.s.iVoiceHold)
iIsSpeech=1;
} else {
iHoldFrames = 0;
}
if (g.s.atTransmit == Settings::Continous)
iIsSpeech = 1;
else if (g.s.atTransmit == Settings::PushToTalk)
iIsSpeech = g.s.uiDoublePush && ((g.uiDoublePush < g.s.uiDoublePush) || (g.tDoublePush.elapsed() < g.s.uiDoublePush));
iIsSpeech = iIsSpeech || (g.iPushToTalk > 0) || (g.iAltSpeak > 0);
if (g.s.bMute || ((g.s.lmLoopMode != Settings::Local) && p && p->bMute) || g.bPushToMute) {
iIsSpeech = 0;
}
if (iIsSpeech) {
iSilentFrames = 0;
} else {
iSilentFrames++;
if (iSilentFrames > 200)
iFrameCounter = 0;
}
if (p)
p->setTalking(iIsSpeech, (g.iAltSpeak > 0));
if (g.s.bPushClick && (g.s.atTransmit == Settings::PushToTalk)) {
AudioOutputPtr ao = g.ao;
if (iIsSpeech && ! bPreviousVoice && ao)
ao->playSine(400.0f,1200.0f,5);
else if (ao && !iIsSpeech && bPreviousVoice && ao)
ao->playSine(620.0f,-1200.0f,5);
}
if (! iIsSpeech && ! bPreviousVoice) {
iBitrate = 0;
if (g.s.iIdleTime && ! g.s.bMute && ((tIdle.elapsed() / 1000000ULL) > g.s.iIdleTime)) {
emit doMute();
tIdle.restart();
}
return;
}
bPreviousVoice = iIsSpeech;
tIdle.restart();
if (! iIsSpeech) {
memset(psMic, 0, sizeof(short) * iFrameSize);
}
if (g.s.bTransmitPosition && g.p && ! g.bCenterPosition && (iFrames == 0) && g.p->fetch()) {
QByteArray q;
QDataStream ds(&q, QIODevice::WriteOnly);
ds << g.p->fPosition[0];
ds << g.p->fPosition[1];
ds << g.p->fPosition[2];
speex_bits_pack(&sbBits, 13, 5);
speex_bits_pack(&sbBits, q.size(), 4);
const unsigned char *d=reinterpret_cast<const unsigned char*>(q.data());
for (i=0;i<q.size();i++) {
speex_bits_pack(&sbBits, d[i], 8);
}
}
speex_encode_int(esEncState, psSource, &sbBits);
iFrames++;
speex_encoder_ctl(esEncState, SPEEX_GET_BITRATE, &iBitrate);
flushCheck();
}
void AudioInput::encodeAudioFrame() {
int iArg;
ClientUser *p=ClientUser::get(g.uiSession);
int i;
float sum;
short max;
short *psSource;
iFrameCounter++;
if (! bRunning)
return;
sum=1.0f;
for (i=0;i<iFrameSize;i++)
sum += static_cast<float>(psMic[i] * psMic[i]);
dPeakMic = qMax(20.0f*log10f(sqrtf(sum / static_cast<float>(iFrameSize)) / 32768.0f), -96.0f);
max = 1;
for (i=0;i<iFrameSize;i++)
max = static_cast<short>(abs(psMic[i]) > max ? abs(psMic[i]) : max);
dMaxMic = max;
if (psSpeaker && (iEchoChannels > 0)) {
sum=1.0f;
for (i=0;i<iFrameSize;i++)
sum += static_cast<float>(psSpeaker[i] * psSpeaker[i]);
dPeakSpeaker = qMax(20.0f*log10f(sqrtf(sum / static_cast<float>(iFrameSize)) / 32768.0f), -96.0f);
} else {
dPeakSpeaker = 0.0;
}
QMutexLocker l(&qmSpeex);
resetAudioProcessor();
speex_preprocess_ctl(sppPreprocess, SPEEX_PREPROCESS_GET_AGC_GAIN, &iArg);
float gainValue = static_cast<float>(iArg);
iArg = g.s.iNoiseSuppress - iArg;
speex_preprocess_ctl(sppPreprocess, SPEEX_PREPROCESS_SET_NOISE_SUPPRESS, &iArg);
if (sesEcho && psSpeaker) {
speex_echo_cancellation(sesEcho, psMic, psSpeaker, psClean);
speex_preprocess_run(sppPreprocess, psClean);
psSource = psClean;
} else {
speex_preprocess_run(sppPreprocess, psMic);
psSource = psMic;
}
sum=1.0f;
for (i=0;i<iFrameSize;i++)
sum += static_cast<float>(psSource[i] * psSource[i]);
float micLevel = sqrtf(sum / static_cast<float>(iFrameSize));
dPeakSignal = qMax(20.0f*log10f(micLevel / 32768.0f), -96.0f);
spx_int32_t prob = 0;
speex_preprocess_ctl(sppPreprocess, SPEEX_PREPROCESS_GET_PROB, &prob);
fSpeechProb = static_cast<float>(prob) / 100.0f;
// clean microphone level: peak of filtered signal attenuated by AGC gain
dPeakCleanMic = qMax(dPeakSignal - gainValue, -96.0f);
float level = (g.s.vsVAD == Settings::SignalToNoise) ? fSpeechProb : (1.0f + dPeakCleanMic / 96.0f);
bool bIsSpeech = false;
if (level > g.s.fVADmax)
bIsSpeech = true;
else if (level > g.s.fVADmin && bPreviousVoice)
bIsSpeech = true;
if (! bIsSpeech) {
iHoldFrames++;
if (iHoldFrames < g.s.iVoiceHold)
bIsSpeech = true;
} else {
iHoldFrames = 0;
}
if (g.s.atTransmit == Settings::Continous)
bIsSpeech = true;
else if (g.s.atTransmit == Settings::PushToTalk)
bIsSpeech = g.s.uiDoublePush && ((g.uiDoublePush < g.s.uiDoublePush) || (g.tDoublePush.elapsed() < g.s.uiDoublePush));
bIsSpeech = bIsSpeech || (g.iPushToTalk > 0);
if (g.s.bMute || ((g.s.lmLoopMode != Settings::Local) && p && (p->bMute || p->bSuppress)) || g.bPushToMute || (g.iTarget < 0)) {
bIsSpeech = false;
}
if (bIsSpeech) {
iSilentFrames = 0;
} else {
iSilentFrames++;
if (iSilentFrames > 500)
iFrameCounter = 0;
}
if (p) {
if (! bIsSpeech)
p->setTalking(Settings::Passive);
else if (g.iTarget == 0)
p->setTalking(Settings::Talking);
else
p->setTalking(Settings::Shouting);
}
if (g.s.bTxAudioCue && g.uiSession != 0) {
AudioOutputPtr ao = g.ao;
if (bIsSpeech && ! bPreviousVoice && ao)
ao->playSample(g.s.qsTxAudioCueOn);
else if (ao && !bIsSpeech && bPreviousVoice && ao)
ao->playSample(g.s.qsTxAudioCueOff);
}
if (! bIsSpeech && ! bPreviousVoice) {
iBitrate = 0;
if (g.s.iIdleTime && ! g.s.bDeaf && ((tIdle.elapsed() / 1000000ULL) > g.s.iIdleTime)) {
emit doDeaf();
tIdle.restart();
}
spx_int32_t increment = 0;
speex_preprocess_ctl(sppPreprocess, SPEEX_PREPROCESS_SET_AGC_INCREMENT, &increment);
return;
} else {
spx_int32_t increment = 12;
speex_preprocess_ctl(sppPreprocess, SPEEX_PREPROCESS_SET_AGC_INCREMENT, &increment);
}
tIdle.restart();
/*
int r = celt_encoder_ctl(ceEncoder, CELT_SET_POST_MDCT_CALLBACK(celtBack, NULL));
qWarning() << "Set Callback" << r;
*/
unsigned char buffer[512];
int len;
if (umtType != MessageHandler::UDPVoiceSpeex) {
len = encodeCELTFrame(psSource, buffer);
if (len == 0)
return;
} else {
len = encodeSpeexFrame(psSource, buffer);
}
flushCheck(QByteArray(reinterpret_cast<const char *>(buffer), len), ! bIsSpeech);
if (! bIsSpeech)
iBitrate = 0;
bPreviousVoice = bIsSpeech;
}
int JackAudioSystem::process_callback(jack_nframes_t nframes, void *arg) {
JackAudioSystem * const jas = static_cast<JackAudioSystem*>(arg);
if (jas && jas->bJackIsGood) {
AudioInputPtr ai = g.ai;
AudioOutputPtr ao = g.ao;
JackAudioInput * const jai = dynamic_cast<JackAudioInput *>(ai.get());
JackAudioOutput * const jao = dynamic_cast<JackAudioOutput *>(ao.get());
if (jai && jai->isRunning() && jai->iMicChannels > 0 && !jai->isFinished()) {
QMutexLocker(&jai->qmMutex);
void *input = jack_port_get_buffer(jas->in_port, nframes);
if (input != NULL) {
jai->addMic(input, nframes);
}
}
if (jao && jao->isRunning() && jao->iChannels > 0 && !jao->isFinished()) {
QMutexLocker(&jao->qmMutex);
jack_default_audio_sample_t *port_buffers[JACK_MAX_OUTPUT_PORTS];
for (unsigned int i = 0; i < jao->iChannels; ++i) {
port_buffers[i] = (jack_default_audio_sample_t*)jack_port_get_buffer(jas->out_ports[i], nframes);
if (port_buffers[i] == NULL) {
return 1;
}
}
jack_default_audio_sample_t * const buffer = jas->output_buffer;
memset(buffer, 0, sizeof(jack_default_audio_sample_t) * nframes * jao->iChannels);
jao->mix(buffer, nframes);
if (jao->iChannels == 1) {
memcpy(port_buffers[0], buffer, sizeof(jack_default_audio_sample_t) * nframes);
} else {
// de-interleave channels
for (unsigned int i = 0; i < nframes * jao->iChannels; ++i) {
port_buffers[i % jao->iChannels][i / jao->iChannels] = buffer[i];
}
}
}
}
return 0;
}
