bool AVOutput::writeData(const QByteArray &data)
{
Q_UNUSED(data);
//DPTR_D(AVOutput);
//locker(&mutex)
//TODO: make sure d.data thread safe. lock around here? for audio and video(main thread problem)?
/* you can use d.data directly in AVThread. In other thread, it's not safe, you must do something
* to make sure the data is not be modified in AVThread when using it*/
//d_func().data = data;
convertData(data);
bool result = write();
//write then pause: if capture when pausing, the displayed picture is captured
tryPause();
return result;
}
/*
*TODO:
* if output is null or dummy, the use duration to wait
*/
void AudioThread::run()
{
DPTR_D(AudioThread);
//No decoder or output. No audio output is ok, just display picture
if (!d.dec || !d.dec->isAvailable() || !d.outputSet)
return;
resetState();
Q_ASSERT(d.clock != 0);
AudioDecoder *dec = static_cast<AudioDecoder*>(d.dec);
AudioOutput *ao = 0;
// first() is not null even if list empty
if (!d.outputSet->outputs().isEmpty())
ao = static_cast<AudioOutput*>(d.outputSet->outputs().first()); //TODO: not here
d.init();
//TODO: bool need_sync in private class
bool is_external_clock = d.clock->clockType() == AVClock::ExternalClock;
Packet pkt;
while (!d.stop) {
processNextTask();
//TODO: why put it at the end of loop then playNextFrame() not work?
if (tryPause()) { //DO NOT continue, or playNextFrame() will fail
if (d.stop)
break; //the queue is empty and may block. should setBlocking(false) wake up cond empty?
} else {
if (isPaused())
continue;
}
if (d.packets.isEmpty() && !d.stop) {
d.stop = d.demux_end;
}
if (d.stop) {
qDebug("audio thread stop before take packet");
break;
}
if (!pkt.isValid()) {
pkt = d.packets.take(); //wait to dequeue
}
if (!pkt.isValid()) {
qDebug("Invalid packet! flush audio codec context!!!!!!!! audio queue size=%d", d.packets.size());
dec->flush();
continue;
}
bool skip_render = pkt.pts < d.render_pts0;
// audio has no key frame, skip rendering equals to skip decoding
if (skip_render) {
d.clock->updateValue(pkt.pts);
/*
* audio may be too fast than video if skip without sleep
* a frame is about 20ms. sleep time must be << frame time
*/
qreal a_v = pkt.pts - d.clock->videoPts();
//qDebug("skip audio decode at %f/%f v=%f a-v=%f", pkt.pts, d.render_pts0, d.clock->videoPts(), a_v);
if (a_v > 0)
msleep(qMin((ulong)300, ulong(a_v*1000.0)));
else
msleep(2);
pkt = Packet(); //mark invalid to take next
continue;
}
d.render_pts0 = 0;
if (is_external_clock) {
d.delay = pkt.pts - d.clock->value();
/*
*after seeking forward, a packet may be the old, v packet may be
*the new packet, then the d.delay is very large, omit it.
*TODO: 1. how to choose the value
* 2. use last delay when seeking
*/
if (qAbs(d.delay) < 2.718) {
if (d.delay < -kSyncThreshold) { //Speed up. drop frame?
//continue;
}
while (d.delay > kSyncThreshold) { //Slow down
//d.delay_cond.wait(&d.mutex, d.delay*1000); //replay may fail. why?
//qDebug("~~~~~wating for %f msecs", d.delay*1000);
usleep(kSyncThreshold * 1000000UL);
if (d.stop)
d.delay = 0;
else
d.delay -= kSyncThreshold;
}
if (d.delay > 0)
usleep(d.delay * 1000000UL);
} else { //when to drop off?
if (d.delay > 0) {
msleep(64);
} else {
//audio packet not cleaned up?
continue;
}
}
} else {
d.clock->updateValue(pkt.pts);
}
//DO NOT decode and convert if ao is not available or mute!
bool has_ao = ao && ao->isAvailable();
//if (!has_ao) {//do not decode?
// TODO: move resampler to AudioFrame, like VideoFrame does
if (has_ao && dec->resampler()) {
if (dec->resampler()->speed() != ao->speed()
|| dec->resampler()->outAudioFormat() != ao->audioFormat()) {
//resample later to ensure thread safe. TODO: test
if (d.resample) {
qDebug("decoder set speed: %.2f", ao->speed());
dec->resampler()->setOutAudioFormat(ao->audioFormat());
dec->resampler()->setSpeed(ao->speed());
dec->resampler()->prepare();
d.resample = false;
} else {
d.resample = true;
}
}
} else {
if (dec->resampler() && dec->resampler()->speed() != d.clock->speed()) {
if (d.resample) {
qDebug("decoder set speed: %.2f", d.clock->speed());
dec->resampler()->setSpeed(d.clock->speed());
dec->resampler()->prepare();
d.resample = false;
} else {
d.resample = true;
}
}
}
if (d.stop) {
qDebug("audio thread stop before decode()");
break;
}
QMutexLocker locker(&d.mutex);
Q_UNUSED(locker);
if (!dec->decode(pkt.data)) {
qWarning("Decode audio failed");
qreal dt = pkt.pts - d.last_pts;
if (dt > 0.618 || dt < 0) {
dt = 0;
}
//qDebug("sleep %f", dt);
//TODO: avoid acummulative error. External clock?
msleep((unsigned long)(dt*1000.0));
pkt = Packet();
d.last_pts = d.clock->value(); //not pkt.pts! the delay is updated!
continue;
}
QByteArray decoded(dec->data());
int decodedSize = decoded.size();
int decodedPos = 0;
qreal delay = 0;
//AudioFormat.durationForBytes() calculates int type internally. not accurate
AudioFormat &af = dec->resampler()->inAudioFormat();
qreal byte_rate = af.bytesPerSecond();
while (decodedSize > 0) {
if (d.stop) {
qDebug("audio thread stop after decode()");
break;
}
// TODO: set to format.bytesPerFrame()*1024?
const int chunk = qMin(decodedSize, has_ao ? ao->bufferSize() : 1024*4);//int(max_len*byte_rate));
//AudioFormat.bytesForDuration
const qreal chunk_delay = (qreal)chunk/(qreal)byte_rate;
pkt.pts += chunk_delay;
QByteArray decodedChunk(chunk, 0); //volume == 0 || mute
if (has_ao) {
//TODO: volume filter and other filters!!!
if (!ao->isMute()) {
decodedChunk = QByteArray::fromRawData(decoded.constData() + decodedPos, chunk);
qreal vol = ao->volume();
if (vol != 1.0) {
int len = decodedChunk.size()/ao->audioFormat().bytesPerSample();
switch (ao->audioFormat().sampleFormat()) {
case AudioFormat::SampleFormat_Unsigned8:
case AudioFormat::SampleFormat_Unsigned8Planar: {
quint8 *data = (quint8*)decodedChunk.data(); //TODO: other format?
for (int i = 0; i < len; data[i++] *= vol) {}
}
break;
case AudioFormat::SampleFormat_Signed16:
case AudioFormat::SampleFormat_Signed16Planar: {
qint16 *data = (qint16*)decodedChunk.data(); //TODO: other format?
for (int i = 0; i < len; data[i++] *= vol) {}
}
break;
case AudioFormat::SampleFormat_Signed32:
case AudioFormat::SampleFormat_Signed32Planar: {
qint32 *data = (qint32*)decodedChunk.data(); //TODO: other format?
for (int i = 0; i < len; data[i++] *= vol) {}
}
break;
case AudioFormat::SampleFormat_Float:
case AudioFormat::SampleFormat_FloatPlanar: {
float *data = (float*)decodedChunk.data(); //TODO: other format?
for (int i = 0; i < len; data[i++] *= vol) {}
}
break;
case AudioFormat::SampleFormat_Double:
case AudioFormat::SampleFormat_DoublePlanar: {
double *data = (double*)decodedChunk.data(); //TODO: other format?
for (int i = 0; i < len; data[i++] *= vol) {}
}
break;
default:
break;
}
}
}
ao->waitForNextBuffer();
ao->receiveData(decodedChunk, pkt.pts);
ao->play();
d.clock->updateValue(ao->timestamp());
} else {
d.clock->updateDelay(delay += chunk_delay);
/*
* why need this even if we add delay? and usleep sounds weird
* the advantage is if no audio device, the play speed is ok too
* So is portaudio blocking the thread when playing?
*/
static bool sWarn_no_ao = true; //FIXME: no warning when replay. warn only once
if (sWarn_no_ao) {
qDebug("Audio output not available! msleep(%lu)", (unsigned long)((qreal)chunk/(qreal)byte_rate * 1000));
sWarn_no_ao = false;
}
//TODO: avoid acummulative error. External clock?
msleep((unsigned long)(chunk_delay * 1000.0));
}
decodedPos += chunk;
decodedSize -= chunk;
}
int undecoded = dec->undecodedSize();
if (undecoded > 0) {
pkt.data.remove(0, pkt.data.size() - undecoded);
} else {
pkt = Packet();
}
d.last_pts = d.clock->value(); //not pkt.pts! the delay is updated!
}
qDebug("Audio thread stops running...");
}
void AVDemuxThread::run()
{
m_buffering = false;
end = false;
if (audio_thread && !audio_thread->isRunning())
audio_thread->start(QThread::HighPriority);
if (video_thread && !video_thread->isRunning())
video_thread->start();
int stream = 0;
Packet pkt;
pause(false);
qDebug("get av queue a/v thread = %p %p", audio_thread, video_thread);
PacketBuffer *aqueue = audio_thread ? audio_thread->packetQueue() : 0;
PacketBuffer *vqueue = video_thread ? video_thread->packetQueue() : 0;
// aqueue as a primary buffer: music with/without cover
AVThread* thread = !video_thread || (audio_thread && demuxer->hasAttacedPicture()) ? audio_thread : video_thread;
m_buffer = thread->packetQueue();
const qint64 buf2 = aqueue ? aqueue->bufferValue() : 1; // TODO: may be changed by user. Deal with audio track change
if (aqueue) {
aqueue->clear();
aqueue->setBlocking(true);
}
if (vqueue) {
vqueue->clear();
vqueue->setBlocking(true);
}
connect(thread, SIGNAL(seekFinished(qint64)), this, SIGNAL(seekFinished(qint64)), Qt::DirectConnection);
seek_tasks.clear();
bool was_end = false;
if (ademuxer) {
ademuxer->seek(0LL);
}
while (!end) {
processNextSeekTask();
//vthread maybe changed by AVPlayer.setPriority() from no dec case
vqueue = video_thread ? video_thread->packetQueue() : 0;
if (demuxer->atEnd()) {
// if avthread may skip 1st eof packet because of a/v sync
if (aqueue && (!was_end || aqueue->isEmpty())) {
aqueue->put(Packet::createEOF());
aqueue->blockEmpty(false); // do not block if buffer is not enough. block again on seek
}
if (vqueue && (!was_end || vqueue->isEmpty())) {
vqueue->put(Packet::createEOF());
vqueue->blockEmpty(false);
}
m_buffering = false;
Q_EMIT mediaStatusChanged(QtAV::BufferedMedia);
was_end = true;
// wait for a/v thread finished
msleep(100);
continue;
}
was_end = false;
if (tryPause()) {
continue; //the queue is empty and will block
}
updateBufferState();
if (!demuxer->readFrame()) {
continue;
}
stream = demuxer->stream();
pkt = demuxer->packet();
Packet apkt;
bool audio_has_pic = demuxer->hasAttacedPicture();
int a_ext = 0;
if (ademuxer) {
QMutexLocker locker(&buffer_mutex);
Q_UNUSED(locker);
if (ademuxer) {
a_ext = -1;
audio_has_pic = ademuxer->hasAttacedPicture();
// FIXME: buffer full but buffering!!!
// avoid read external track everytime. aqueue may not block full
// vqueue will not block if aqueue is not enough
if (!aqueue->isFull() || aqueue->isBuffering()) {
if (ademuxer->readFrame()) {
if (ademuxer->stream() == ademuxer->audioStream()) {
a_ext = 1;
apkt = ademuxer->packet();
}
}
// no continue otherwise. ademuxer finished earlier than demuxer
}
}
}
//qDebug("vqueue: %d, aqueue: %d/isbuffering %d isfull: %d, buffer: %d/%d", vqueue->size(), aqueue->size(), aqueue->isBuffering(), aqueue->isFull(), aqueue->buffered(), aqueue->bufferValue());
//QMutexLocker locker(&buffer_mutex); //TODO: seems we do not need to lock
//Q_UNUSED(locker);
/*1 is empty but another is enough, then do not block to
ensure the empty one can put packets immediatly.
But usually it will not happen, why?
*/
/* demux thread will be blocked only when 1 queue is full and still put
* if vqueue is full and aqueue becomes empty, then demux thread
* will be blocked. so we should wake up another queue when empty(or threshold?).
* TODO: the video stream and audio stream may be group by group. provide it
* stream data: aaaaaaavvvvvvvaaaaaaaavvvvvvvvvaaaaaa, it happens
* stream data: aavavvavvavavavavavavavavvvaavavavava, it's ok
*/
//TODO: use cache queue, take from cache queue if not empty?
const bool a_internal = stream == demuxer->audioStream();
if (a_internal || a_ext > 0) {//apkt.isValid()) {
if (a_internal && !a_ext) // internal is always read even if external audio used
apkt = demuxer->packet();
/* if vqueue if not blocked and full, and aqueue is empty, then put to
* vqueue will block demuex thread
*/
if (aqueue) {
if (!audio_thread || !audio_thread->isRunning()) {
aqueue->clear();
continue;
}
// must ensure bufferValue set correctly before continue
if (m_buffer != aqueue)
aqueue->setBufferValue(m_buffer->isBuffering() ? std::numeric_limits<qint64>::max() : buf2);
// always block full if no vqueue because empty callback may set false
// attached picture is cover for song, 1 frame
aqueue->blockFull(!video_thread || !video_thread->isRunning() || !vqueue || audio_has_pic);
// external audio: a_ext < 0, stream = audio_idx=>put invalid packet
if (a_ext >= 0)
aqueue->put(apkt); //affect video_thread
}
}
// always check video stream if use external audio
if (stream == demuxer->videoStream()) {
if (vqueue) {
if (!video_thread || !video_thread->isRunning()) {
vqueue->clear();
continue;
}
vqueue->blockFull(!audio_thread || !audio_thread->isRunning() || !aqueue || aqueue->isEnough());
vqueue->put(pkt); //affect audio_thread
}
} else if (demuxer->subtitleStreams().contains(stream)) { //subtitle
Q_EMIT internalSubtitlePacketRead(demuxer->subtitleStreams().indexOf(stream), pkt);
} else {
continue;
}
}
m_buffering = false;
m_buffer = 0;
while (audio_thread && audio_thread->isRunning()) {
qDebug("waiting audio thread.......");
aqueue->blockEmpty(false); //FIXME: why need this
audio_thread->wait(500);
}
while (video_thread && video_thread->isRunning()) {
qDebug("waiting video thread.......");
vqueue->blockEmpty(false);
video_thread->wait(500);
}
thread->disconnect(this, SIGNAL(seekFinished(qint64)));
qDebug("Demux thread stops running....");
emit mediaStatusChanged(QtAV::EndOfMedia);
}
void AVDemuxThread::run()
{
end = false;
if (audio_thread && !audio_thread->isRunning())
audio_thread->start(QThread::HighPriority);
if (video_thread && !video_thread->isRunning())
video_thread->start();
int running_threads = 0;
if (audio_thread)
++running_threads;
if (video_thread)
++running_threads;
qDebug("demux thread start running...%d avthreads", running_threads);
audio_stream = demuxer->audioStream();
video_stream = demuxer->videoStream();
int index = 0;
Packet pkt;
pause(false);
qDebug("get av queue a/v thread = %p %p", audio_thread, video_thread);
PacketQueue *aqueue = audio_thread ? audio_thread->packetQueue() : 0;
PacketQueue *vqueue = video_thread ? video_thread->packetQueue() : 0;
if (aqueue) {
aqueue->clear();
aqueue->setBlocking(true);
}
if (vqueue) {
vqueue->clear();
vqueue->setBlocking(true);
}
while (!end) {
processNextSeekTask();
#if RESUME_ONCE_ON_SEEK
// resume once is not enough. readFrame() failed after seek (why?) and we need video packet
processNextPauseTask();
if (tryPause()) {
#else
if (tryPause(ULONG_MAX)) {
#endif
continue; //the queue is empty and will block
}
running_threads = (audio_thread && audio_thread->isRunning()) + (video_thread && video_thread->isRunning());
if (!running_threads) {
qDebug("no running avthreads. exit demuxer thread");
break;
}
QMutexLocker locker(&buffer_mutex);
Q_UNUSED(locker);
if (end) {
break;
}
if (!demuxer->readFrame()) {
continue;
}
index = demuxer->stream();
pkt = *demuxer->packet(); //TODO: how to avoid additional copy?
//connect to stop is ok too
if (pkt.isEnd()) {
qDebug("read end packet %d A:%d V:%d", index, audio_stream, video_stream);
end = true;
//avthread can stop. do not clear queue, make sure all data are played
if (audio_thread) {
audio_thread->setDemuxEnded(true);
}
if (video_thread) {
video_thread->setDemuxEnded(true);
}
break;
}
/*1 is empty but another is enough, then do not block to
ensure the empty one can put packets immediatly.
But usually it will not happen, why?
*/
/* demux thread will be blocked only when 1 queue is full and still put
* if vqueue is full and aqueue becomes empty, then demux thread
* will be blocked. so we should wake up another queue when empty(or threshold?).
* TODO: the video stream and audio stream may be group by group. provide it
* stream data: aaaaaaavvvvvvvaaaaaaaavvvvvvvvvaaaaaa, it happens
* stream data: aavavvavvavavavavavavavavvvaavavavava, it's ok
*/
//TODO: use cache queue, take from cache queue if not empty?
if (index == audio_stream) {
/* if vqueue if not blocked and full, and aqueue is empty, then put to
* vqueue will block demuex thread
*/
if (aqueue) {
if (!audio_thread || !audio_thread->isRunning()) {
aqueue->clear();
continue;
}
if (vqueue)
aqueue->blockFull(vqueue->isEnough() || demuxer->hasAttacedPicture());
aqueue->put(pkt); //affect video_thread
}
} else if (index == video_stream) {
if (!video_thread || !video_thread->isRunning()) {
vqueue->clear();
continue;
}
if (vqueue) {
if (aqueue)
vqueue->blockFull(aqueue->isEnough());
vqueue->put(pkt); //affect audio_thread
}
} else { //subtitle
continue;
}
}
//flush. seeking will be omitted when stopped
if (aqueue)
aqueue->put(Packet());
if (vqueue)
vqueue->put(Packet());
while (audio_thread && audio_thread->isRunning()) {
qDebug("waiting audio thread.......");
audio_thread->wait(500);
}
while (video_thread && video_thread->isRunning()) {
qDebug("waiting video thread.......");
video_thread->wait(500);
}
qDebug("Demux thread stops running....");
}
bool AVDemuxThread::tryPause(unsigned long timeout)
{
if (!paused)
return false;
QMutexLocker lock(&buffer_mutex);
Q_UNUSED(lock);
cond.wait(&buffer_mutex, timeout);
return true;
}
} //namespace QtAV
//TODO: if output is null or dummy, the use duration to wait
void AudioThread::run()
{
DPTR_D(AudioThread);
//No decoder or output. No audio output is ok, just display picture
if (!d.dec || !d.dec->isAvailable())
return;
resetState();
Q_ASSERT(d.clock != 0);
AudioDecoder *dec = static_cast<AudioDecoder*>(d.dec);
AudioOutput *ao = static_cast<AudioOutput*>(d.writer);
int sample_rate = dec->codecContext()->sample_rate;
int channels = dec->codecContext()->channels;
int csf = channels * sample_rate * sizeof(float);
static const double max_len = 0.02;
d.last_pts = 0;
while (!d.stop) {
//TODO: why put it at the end of loop then playNextFrame() not work?
if (tryPause()) { //DO NOT continue, or playNextFrame() will fail
if (d.stop)
break; //the queue is empty and may block. should setBlocking(false) wake up cond empty?
}
QMutexLocker locker(&d.mutex);
Q_UNUSED(locker);
if (d.packets.isEmpty() && !d.stop) {
d.stop = d.demux_end;
if (d.stop) {
break;
}
}
Packet pkt = d.packets.take(); //wait to dequeue
if (!pkt.isValid()) {
qDebug("Invalid packet! flush audio codec context!!!!!!!!");
dec->flush();
continue;
}
d.clock->updateValue(pkt.pts);
//DO NOT decode and convert if ao is not available or mute!
if (dec->decode(pkt.data)) {
QByteArray decoded(dec->data());
int decodedSize = decoded.size();
int decodedPos = 0;
qreal delay =0;
while (decodedSize > 0) {
int chunk = qMin(decodedSize, int(max_len*csf));
d.clock->updateDelay(delay += (qreal)chunk/(qreal)csf);
QByteArray decodedChunk(chunk, 0); //volume == 0 || mute
if (ao && ao->isAvailable()) {
if (!ao->isMute()) {
decodedChunk = QByteArray::fromRawData(decoded.constData() + decodedPos, chunk);
qreal vol = ao->volume();
if (vol != 1.0) {
int len = decodedChunk.size()/sizeof(float); //TODO: why???
float *data = (float*)decodedChunk.data();
for (int i = 0; i < len; ++i)
data[i] *= vol;
}
}
ao->writeData(decodedChunk);
} else {
/*
* why need this even if we add delay? and usleep sounds weird
* the advantage is if no audio device, the play speed is ok too
* So is portaudio blocking the thread when playing?
*/
static bool sWarn_no_ao = true; //FIXME: no warning when replay. warn only once
if (sWarn_no_ao) {
qDebug("Audio output not available! msleep(%lu)", (unsigned long)((qreal)chunk/(qreal)csf * 1000));
sWarn_no_ao = false;
}
//TODO: avoid acummulative error. External clock?
msleep((unsigned long)((qreal)chunk/(qreal)csf * 1000.0));
}
decodedPos += chunk;
decodedSize -= chunk;
}
} else {
//qWarning("Decode audio failed");
qreal dt = pkt.pts - d.last_pts;
if (abs(dt) > 0.618 || dt < 0) {
dt = 0;
}
//qDebug("sleep %f", dt);
//TODO: avoid acummulative error. External clock?
msleep((unsigned long)(dt*1000.0));
}
d.last_pts = d.clock->value(); //not pkt.pts! the delay is updated!
}
qDebug("Audio thread stops running...");
}
