int64_t AudioInjector::injectNextFrame() {
if (_state == AudioInjector::State::Finished) {
qDebug() << "AudioInjector::injectNextFrame called but AudioInjector has finished and was not restarted. Returning.";
return NEXT_FRAME_DELTA_ERROR_OR_FINISHED;
}
// if we haven't setup the packet to send then do so now
static int positionOptionOffset = -1;
static int volumeOptionOffset = -1;
static int audioDataOffset = -1;
if (!_currentPacket) {
if (_currentSendOffset < 0 ||
_currentSendOffset >= _audioData.size()) {
_currentSendOffset = 0;
}
// make sure we actually have samples downloaded to inject
if (_audioData.size()) {
int sampleSize = (_options.stereo ? 2 : 1) * sizeof(AudioConstants::AudioSample);
auto numSamples = static_cast<int>(_audioData.size() / sampleSize);
auto targetSize = numSamples * sampleSize;
if (targetSize != _audioData.size()) {
qDebug() << "Resizing audio that doesn't end at multiple of sample size, resizing from "
<< _audioData.size() << " to " << targetSize;
_audioData.resize(targetSize);
}
_outgoingSequenceNumber = 0;
_nextFrame = 0;
if (!_frameTimer) {
_frameTimer = std::unique_ptr<QElapsedTimer>(new QElapsedTimer);
}
_frameTimer->restart();
_currentPacket = NLPacket::create(PacketType::InjectAudio);
// setup the packet for injected audio
QDataStream audioPacketStream(_currentPacket.get());
// pack some placeholder sequence number for now
audioPacketStream << (quint16) 0;
// pack stream identifier (a generated UUID)
audioPacketStream << QUuid::createUuid();
// pack the stereo/mono type of the stream
audioPacketStream << _options.stereo;
// pack the flag for loopback
uchar loopbackFlag = (uchar)true;
audioPacketStream << loopbackFlag;
// pack the position for injected audio
positionOptionOffset = _currentPacket->pos();
audioPacketStream.writeRawData(reinterpret_cast<const char*>(&_options.position),
sizeof(_options.position));
// pack our orientation for injected audio
audioPacketStream.writeRawData(reinterpret_cast<const char*>(&_options.orientation),
sizeof(_options.orientation));
// pack zero for radius
float radius = 0;
audioPacketStream << radius;
// pack 255 for attenuation byte
volumeOptionOffset = _currentPacket->pos();
quint8 volume = MAX_INJECTOR_VOLUME;
audioPacketStream << volume;
audioPacketStream << _options.ignorePenumbra;
audioDataOffset = _currentPacket->pos();
} else {
// no samples to inject, return immediately
qDebug() << "AudioInjector::injectNextFrame() called with no samples to inject. Returning.";
return NEXT_FRAME_DELTA_ERROR_OR_FINISHED;
}
}
if (!_frameTimer->isValid()) {
// in the case where we have been restarted, the frame timer will be invalid and we need to start it back over here
_frameTimer->restart();
}
int totalBytesLeftToCopy = (_options.stereo ? 2 : 1) * AudioConstants::NETWORK_FRAME_BYTES_PER_CHANNEL;
if (!_options.loop) {
// If we aren't looping, let's make sure we don't read past the end
totalBytesLeftToCopy = std::min(totalBytesLeftToCopy, _audioData.size() - _currentSendOffset);
}
// Measure the loudness of this frame
_loudness = 0.0f;
for (int i = 0; i < totalBytesLeftToCopy; i += sizeof(int16_t)) {
_loudness += abs(*reinterpret_cast<int16_t*>(_audioData.data() + ((_currentSendOffset + i) % _audioData.size()))) /
(AudioConstants::MAX_SAMPLE_VALUE / 2.0f);
}
_loudness /= (float)(totalBytesLeftToCopy/ sizeof(int16_t));
_currentPacket->seek(0);
// pack the sequence number
_currentPacket->writePrimitive(_outgoingSequenceNumber);
_currentPacket->seek(positionOptionOffset);
_currentPacket->writePrimitive(_options.position);
_currentPacket->writePrimitive(_options.orientation);
quint8 volume = MAX_INJECTOR_VOLUME * _options.volume;
_currentPacket->seek(volumeOptionOffset);
_currentPacket->writePrimitive(volume);
_currentPacket->seek(audioDataOffset);
while (totalBytesLeftToCopy > 0) {
int bytesToCopy = std::min(totalBytesLeftToCopy, _audioData.size() - _currentSendOffset);
_currentPacket->write(_audioData.data() + _currentSendOffset, bytesToCopy);
_currentSendOffset += bytesToCopy;
totalBytesLeftToCopy -= bytesToCopy;
if (_options.loop && _currentSendOffset >= _audioData.size()) {
_currentSendOffset = 0;
}
}
// set the correct size used for this packet
_currentPacket->setPayloadSize(_currentPacket->pos());
// grab our audio mixer from the NodeList, if it exists
auto nodeList = DependencyManager::get<NodeList>();
SharedNodePointer audioMixer = nodeList->soloNodeOfType(NodeType::AudioMixer);
if (audioMixer) {
// send off this audio packet
nodeList->sendUnreliablePacket(*_currentPacket, *audioMixer);
_outgoingSequenceNumber++;
}
if (_currentSendOffset >= _audioData.size() && !_options.loop) {
finish();
return NEXT_FRAME_DELTA_ERROR_OR_FINISHED;
}
if (!_hasSentFirstFrame) {
_hasSentFirstFrame = true;
// ask AudioInjectorManager to call us right away again to
// immediately send the first two frames so the mixer can start using the audio right away
return NEXT_FRAME_DELTA_IMMEDIATELY;
}
const int MAX_ALLOWED_FRAMES_TO_FALL_BEHIND = 7;
int64_t currentTime = _frameTimer->nsecsElapsed() / 1000;
auto currentFrameBasedOnElapsedTime = currentTime / AudioConstants::NETWORK_FRAME_USECS;
if (currentFrameBasedOnElapsedTime - _nextFrame > MAX_ALLOWED_FRAMES_TO_FALL_BEHIND) {
// If we are falling behind by more frames than our threshold, let's skip the frames ahead
qDebug() << this << "injectNextFrame() skipping ahead, fell behind by " << (currentFrameBasedOnElapsedTime - _nextFrame) << " frames";
_nextFrame = currentFrameBasedOnElapsedTime;
_currentSendOffset = _nextFrame * AudioConstants::NETWORK_FRAME_BYTES_PER_CHANNEL * (_options.stereo ? 2 : 1) % _audioData.size();
}
int64_t playNextFrameAt = ++_nextFrame * AudioConstants::NETWORK_FRAME_USECS;
return std::max(INT64_C(0), playNextFrameAt - currentTime);
}
uint64_t AudioInjector::injectNextFrame() {
if (_state == AudioInjector::State::Finished) {
qDebug() << "AudioInjector::injectNextFrame called but AudioInjector has finished and was not restarted. Returning.";
return NEXT_FRAME_DELTA_ERROR_OR_FINISHED;
}
// if we haven't setup the packet to send then do so now
static int positionOptionOffset = -1;
static int volumeOptionOffset = -1;
static int audioDataOffset = -1;
if (!_currentPacket) {
if (_currentSendOffset < 0 ||
_currentSendOffset >= _audioData.size()) {
_currentSendOffset = 0;
}
// make sure we actually have samples downloaded to inject
if (_audioData.size()) {
_outgoingSequenceNumber = 0;
_nextFrame = 0;
if (!_frameTimer) {
_frameTimer = std::unique_ptr<QElapsedTimer>(new QElapsedTimer);
}
_frameTimer->restart();
_currentPacket = NLPacket::create(PacketType::InjectAudio);
// setup the packet for injected audio
QDataStream audioPacketStream(_currentPacket.get());
// pack some placeholder sequence number for now
audioPacketStream << (quint16) 0;
// pack stream identifier (a generated UUID)
audioPacketStream << QUuid::createUuid();
// pack the stereo/mono type of the stream
audioPacketStream << _options.stereo;
// pack the flag for loopback
uchar loopbackFlag = (uchar) true;
audioPacketStream << loopbackFlag;
// pack the position for injected audio
positionOptionOffset = _currentPacket->pos();
audioPacketStream.writeRawData(reinterpret_cast<const char*>(&_options.position),
sizeof(_options.position));
// pack our orientation for injected audio
audioPacketStream.writeRawData(reinterpret_cast<const char*>(&_options.orientation),
sizeof(_options.orientation));
// pack zero for radius
float radius = 0;
audioPacketStream << radius;
// pack 255 for attenuation byte
volumeOptionOffset = _currentPacket->pos();
quint8 volume = MAX_INJECTOR_VOLUME;
audioPacketStream << volume;
audioPacketStream << _options.ignorePenumbra;
audioDataOffset = _currentPacket->pos();
} else {
// no samples to inject, return immediately
qDebug() << "AudioInjector::injectNextFrame() called with no samples to inject. Returning.";
return NEXT_FRAME_DELTA_ERROR_OR_FINISHED;
}
}
int bytesToCopy = std::min((_options.stereo ? 2 : 1) * AudioConstants::NETWORK_FRAME_BYTES_PER_CHANNEL,
_audioData.size() - _currentSendOffset);
// Measure the loudness of this frame
_loudness = 0.0f;
for (int i = 0; i < bytesToCopy; i += sizeof(int16_t)) {
_loudness += abs(*reinterpret_cast<int16_t*>(_audioData.data() + _currentSendOffset + i)) /
(AudioConstants::MAX_SAMPLE_VALUE / 2.0f);
}
_loudness /= (float)(bytesToCopy / sizeof(int16_t));
_currentPacket->seek(0);
// pack the sequence number
_currentPacket->writePrimitive(_outgoingSequenceNumber);
_currentPacket->seek(positionOptionOffset);
_currentPacket->writePrimitive(_options.position);
_currentPacket->writePrimitive(_options.orientation);
quint8 volume = MAX_INJECTOR_VOLUME * _options.volume;
_currentPacket->seek(volumeOptionOffset);
_currentPacket->writePrimitive(volume);
_currentPacket->seek(audioDataOffset);
// copy the next NETWORK_BUFFER_LENGTH_BYTES_PER_CHANNEL bytes to the packet
_currentPacket->write(_audioData.data() + _currentSendOffset, bytesToCopy);
// set the correct size used for this packet
_currentPacket->setPayloadSize(_currentPacket->pos());
// grab our audio mixer from the NodeList, if it exists
auto nodeList = DependencyManager::get<NodeList>();
SharedNodePointer audioMixer = nodeList->soloNodeOfType(NodeType::AudioMixer);
if (audioMixer) {
// send off this audio packet
nodeList->sendUnreliablePacket(*_currentPacket, *audioMixer);
_outgoingSequenceNumber++;
}
_currentSendOffset += bytesToCopy;
if (_currentSendOffset >= _audioData.size()) {
// we're at the end of the audio data to send
if (_options.loop) {
// we were asked to loop, set our send offset to 0
_currentSendOffset = 0;
} else {
// we weren't to loop, say that we're done now
finish();
return NEXT_FRAME_DELTA_ERROR_OR_FINISHED;
}
}
if (_currentSendOffset == bytesToCopy) {
// ask AudioInjectorManager to call us right away again to
// immediately send the first two frames so the mixer can start using the audio right away
return NEXT_FRAME_DELTA_IMMEDIATELY;
} else {
return (++_nextFrame * AudioConstants::NETWORK_FRAME_USECS) - _frameTimer->nsecsElapsed() / 1000;
}
}
void AudioInjector::injectToMixer() {
if (_currentSendOffset < 0 ||
_currentSendOffset >= _audioData.size()) {
_currentSendOffset = 0;
}
auto nodeList = DependencyManager::get<NodeList>();
// make sure we actually have samples downloaded to inject
if (_audioData.size()) {
auto audioPacket = NLPacket::create(PacketType::InjectAudio);
// setup the packet for injected audio
QDataStream audioPacketStream(audioPacket.get());
// pack some placeholder sequence number for now
audioPacketStream << (quint16) 0;
// pack stream identifier (a generated UUID)
audioPacketStream << QUuid::createUuid();
// pack the stereo/mono type of the stream
audioPacketStream << _options.stereo;
// pack the flag for loopback
uchar loopbackFlag = (uchar) true;
audioPacketStream << loopbackFlag;
// pack the position for injected audio
int positionOptionOffset = audioPacket->pos();
audioPacketStream.writeRawData(reinterpret_cast<const char*>(&_options.position),
sizeof(_options.position));
// pack our orientation for injected audio
audioPacketStream.writeRawData(reinterpret_cast<const char*>(&_options.orientation),
sizeof(_options.orientation));
// pack zero for radius
float radius = 0;
audioPacketStream << radius;
// pack 255 for attenuation byte
int volumeOptionOffset = audioPacket->pos();
quint8 volume = MAX_INJECTOR_VOLUME * _options.volume;
audioPacketStream << volume;
audioPacketStream << _options.ignorePenumbra;
int audioDataOffset = audioPacket->pos();
QElapsedTimer timer;
timer.start();
int nextFrame = 0;
bool shouldLoop = _options.loop;
// loop to send off our audio in NETWORK_BUFFER_LENGTH_SAMPLES_PER_CHANNEL byte chunks
quint16 outgoingInjectedAudioSequenceNumber = 0;
while (_currentSendOffset < _audioData.size() && !_shouldStop) {
int bytesToCopy = std::min(((_options.stereo) ? 2 : 1) * AudioConstants::NETWORK_FRAME_BYTES_PER_CHANNEL,
_audioData.size() - _currentSendOffset);
// Measure the loudness of this frame
_loudness = 0.0f;
for (int i = 0; i < bytesToCopy; i += sizeof(int16_t)) {
_loudness += abs(*reinterpret_cast<int16_t*>(_audioData.data() + _currentSendOffset + i)) /
(AudioConstants::MAX_SAMPLE_VALUE / 2.0f);
}
_loudness /= (float)(bytesToCopy / sizeof(int16_t));
audioPacket->seek(0);
// pack the sequence number
audioPacket->writePrimitive(outgoingInjectedAudioSequenceNumber);
audioPacket->seek(positionOptionOffset);
audioPacket->writePrimitive(_options.position);
audioPacket->writePrimitive(_options.orientation);
volume = MAX_INJECTOR_VOLUME * _options.volume;
audioPacket->seek(volumeOptionOffset);
audioPacket->writePrimitive(volume);
audioPacket->seek(audioDataOffset);
// copy the next NETWORK_BUFFER_LENGTH_BYTES_PER_CHANNEL bytes to the packet
audioPacket->write(_audioData.data() + _currentSendOffset, bytesToCopy);
// set the correct size used for this packet
audioPacket->setPayloadSize(audioPacket->pos());
// grab our audio mixer from the NodeList, if it exists
SharedNodePointer audioMixer = nodeList->soloNodeOfType(NodeType::AudioMixer);
if (audioMixer) {
// send off this audio packet
nodeList->sendUnreliablePacket(*audioPacket, *audioMixer);
outgoingInjectedAudioSequenceNumber++;
}
_currentSendOffset += bytesToCopy;
// send two packets before the first sleep so the mixer can start playback right away
if (_currentSendOffset != bytesToCopy && _currentSendOffset < _audioData.size()) {
// process events in case we have been told to stop and be deleted
QCoreApplication::processEvents();
if (_shouldStop) {
break;
}
// not the first packet and not done
// sleep for the appropriate time
int usecToSleep = (++nextFrame * AudioConstants::NETWORK_FRAME_USECS) - timer.nsecsElapsed() / 1000;
if (usecToSleep > 0) {
usleep(usecToSleep);
}
}
if (shouldLoop && _currentSendOffset >= _audioData.size()) {
_currentSendOffset = 0;
}
}
}
setIsFinished(true);
_isPlaying = !_isFinished; // Which can be false if a restart was requested
}