bool AVOutput::writeData(const QByteArray &data) { Q_UNUSED(data); //DPTR_D(AVOutput); //locker(&mutex) //TODO: make sure d.data thread safe. lock around here? for audio and video(main thread problem)? /* you can use d.data directly in AVThread. In other thread, it's not safe, you must do something * to make sure the data is not be modified in AVThread when using it*/ //d_func().data = data; convertData(data); bool result = write(); //write then pause: if capture when pausing, the displayed picture is captured tryPause(); return result; }
/* *TODO: * if output is null or dummy, the use duration to wait */ void AudioThread::run() { DPTR_D(AudioThread); //No decoder or output. No audio output is ok, just display picture if (!d.dec || !d.dec->isAvailable() || !d.outputSet) return; resetState(); Q_ASSERT(d.clock != 0); AudioDecoder *dec = static_cast<AudioDecoder*>(d.dec); AudioOutput *ao = 0; // first() is not null even if list empty if (!d.outputSet->outputs().isEmpty()) ao = static_cast<AudioOutput*>(d.outputSet->outputs().first()); //TODO: not here d.init(); //TODO: bool need_sync in private class bool is_external_clock = d.clock->clockType() == AVClock::ExternalClock; Packet pkt; while (!d.stop) { processNextTask(); //TODO: why put it at the end of loop then playNextFrame() not work? if (tryPause()) { //DO NOT continue, or playNextFrame() will fail if (d.stop) break; //the queue is empty and may block. should setBlocking(false) wake up cond empty? } else { if (isPaused()) continue; } if (d.packets.isEmpty() && !d.stop) { d.stop = d.demux_end; } if (d.stop) { qDebug("audio thread stop before take packet"); break; } if (!pkt.isValid()) { pkt = d.packets.take(); //wait to dequeue } if (!pkt.isValid()) { qDebug("Invalid packet! flush audio codec context!!!!!!!! audio queue size=%d", d.packets.size()); dec->flush(); continue; } bool skip_render = pkt.pts < d.render_pts0; // audio has no key frame, skip rendering equals to skip decoding if (skip_render) { d.clock->updateValue(pkt.pts); /* * audio may be too fast than video if skip without sleep * a frame is about 20ms. sleep time must be << frame time */ qreal a_v = pkt.pts - d.clock->videoPts(); //qDebug("skip audio decode at %f/%f v=%f a-v=%f", pkt.pts, d.render_pts0, d.clock->videoPts(), a_v); if (a_v > 0) msleep(qMin((ulong)300, ulong(a_v*1000.0))); else msleep(2); pkt = Packet(); //mark invalid to take next continue; } d.render_pts0 = 0; if (is_external_clock) { d.delay = pkt.pts - d.clock->value(); /* *after seeking forward, a packet may be the old, v packet may be *the new packet, then the d.delay is very large, omit it. *TODO: 1. how to choose the value * 2. use last delay when seeking */ if (qAbs(d.delay) < 2.718) { if (d.delay < -kSyncThreshold) { //Speed up. drop frame? //continue; } while (d.delay > kSyncThreshold) { //Slow down //d.delay_cond.wait(&d.mutex, d.delay*1000); //replay may fail. why? //qDebug("~~~~~wating for %f msecs", d.delay*1000); usleep(kSyncThreshold * 1000000UL); if (d.stop) d.delay = 0; else d.delay -= kSyncThreshold; } if (d.delay > 0) usleep(d.delay * 1000000UL); } else { //when to drop off? if (d.delay > 0) { msleep(64); } else { //audio packet not cleaned up? continue; } } } else { d.clock->updateValue(pkt.pts); } //DO NOT decode and convert if ao is not available or mute! bool has_ao = ao && ao->isAvailable(); //if (!has_ao) {//do not decode? // TODO: move resampler to AudioFrame, like VideoFrame does if (has_ao && dec->resampler()) { if (dec->resampler()->speed() != ao->speed() || dec->resampler()->outAudioFormat() != ao->audioFormat()) { //resample later to ensure thread safe. TODO: test if (d.resample) { qDebug("decoder set speed: %.2f", ao->speed()); dec->resampler()->setOutAudioFormat(ao->audioFormat()); dec->resampler()->setSpeed(ao->speed()); dec->resampler()->prepare(); d.resample = false; } else { d.resample = true; } } } else { if (dec->resampler() && dec->resampler()->speed() != d.clock->speed()) { if (d.resample) { qDebug("decoder set speed: %.2f", d.clock->speed()); dec->resampler()->setSpeed(d.clock->speed()); dec->resampler()->prepare(); d.resample = false; } else { d.resample = true; } } } if (d.stop) { qDebug("audio thread stop before decode()"); break; } QMutexLocker locker(&d.mutex); Q_UNUSED(locker); if (!dec->decode(pkt.data)) { qWarning("Decode audio failed"); qreal dt = pkt.pts - d.last_pts; if (dt > 0.618 || dt < 0) { dt = 0; } //qDebug("sleep %f", dt); //TODO: avoid acummulative error. External clock? msleep((unsigned long)(dt*1000.0)); pkt = Packet(); d.last_pts = d.clock->value(); //not pkt.pts! the delay is updated! continue; } QByteArray decoded(dec->data()); int decodedSize = decoded.size(); int decodedPos = 0; qreal delay = 0; //AudioFormat.durationForBytes() calculates int type internally. not accurate AudioFormat &af = dec->resampler()->inAudioFormat(); qreal byte_rate = af.bytesPerSecond(); while (decodedSize > 0) { if (d.stop) { qDebug("audio thread stop after decode()"); break; } // TODO: set to format.bytesPerFrame()*1024? const int chunk = qMin(decodedSize, has_ao ? ao->bufferSize() : 1024*4);//int(max_len*byte_rate)); //AudioFormat.bytesForDuration const qreal chunk_delay = (qreal)chunk/(qreal)byte_rate; pkt.pts += chunk_delay; QByteArray decodedChunk(chunk, 0); //volume == 0 || mute if (has_ao) { //TODO: volume filter and other filters!!! if (!ao->isMute()) { decodedChunk = QByteArray::fromRawData(decoded.constData() + decodedPos, chunk); qreal vol = ao->volume(); if (vol != 1.0) { int len = decodedChunk.size()/ao->audioFormat().bytesPerSample(); switch (ao->audioFormat().sampleFormat()) { case AudioFormat::SampleFormat_Unsigned8: case AudioFormat::SampleFormat_Unsigned8Planar: { quint8 *data = (quint8*)decodedChunk.data(); //TODO: other format? for (int i = 0; i < len; data[i++] *= vol) {} } break; case AudioFormat::SampleFormat_Signed16: case AudioFormat::SampleFormat_Signed16Planar: { qint16 *data = (qint16*)decodedChunk.data(); //TODO: other format? for (int i = 0; i < len; data[i++] *= vol) {} } break; case AudioFormat::SampleFormat_Signed32: case AudioFormat::SampleFormat_Signed32Planar: { qint32 *data = (qint32*)decodedChunk.data(); //TODO: other format? for (int i = 0; i < len; data[i++] *= vol) {} } break; case AudioFormat::SampleFormat_Float: case AudioFormat::SampleFormat_FloatPlanar: { float *data = (float*)decodedChunk.data(); //TODO: other format? for (int i = 0; i < len; data[i++] *= vol) {} } break; case AudioFormat::SampleFormat_Double: case AudioFormat::SampleFormat_DoublePlanar: { double *data = (double*)decodedChunk.data(); //TODO: other format? for (int i = 0; i < len; data[i++] *= vol) {} } break; default: break; } } } ao->waitForNextBuffer(); ao->receiveData(decodedChunk, pkt.pts); ao->play(); d.clock->updateValue(ao->timestamp()); } else { d.clock->updateDelay(delay += chunk_delay); /* * why need this even if we add delay? and usleep sounds weird * the advantage is if no audio device, the play speed is ok too * So is portaudio blocking the thread when playing? */ static bool sWarn_no_ao = true; //FIXME: no warning when replay. warn only once if (sWarn_no_ao) { qDebug("Audio output not available! msleep(%lu)", (unsigned long)((qreal)chunk/(qreal)byte_rate * 1000)); sWarn_no_ao = false; } //TODO: avoid acummulative error. External clock? msleep((unsigned long)(chunk_delay * 1000.0)); } decodedPos += chunk; decodedSize -= chunk; } int undecoded = dec->undecodedSize(); if (undecoded > 0) { pkt.data.remove(0, pkt.data.size() - undecoded); } else { pkt = Packet(); } d.last_pts = d.clock->value(); //not pkt.pts! the delay is updated! } qDebug("Audio thread stops running..."); }
void AVDemuxThread::run() { m_buffering = false; end = false; if (audio_thread && !audio_thread->isRunning()) audio_thread->start(QThread::HighPriority); if (video_thread && !video_thread->isRunning()) video_thread->start(); int stream = 0; Packet pkt; pause(false); qDebug("get av queue a/v thread = %p %p", audio_thread, video_thread); PacketBuffer *aqueue = audio_thread ? audio_thread->packetQueue() : 0; PacketBuffer *vqueue = video_thread ? video_thread->packetQueue() : 0; // aqueue as a primary buffer: music with/without cover AVThread* thread = !video_thread || (audio_thread && demuxer->hasAttacedPicture()) ? audio_thread : video_thread; m_buffer = thread->packetQueue(); const qint64 buf2 = aqueue ? aqueue->bufferValue() : 1; // TODO: may be changed by user. Deal with audio track change if (aqueue) { aqueue->clear(); aqueue->setBlocking(true); } if (vqueue) { vqueue->clear(); vqueue->setBlocking(true); } connect(thread, SIGNAL(seekFinished(qint64)), this, SIGNAL(seekFinished(qint64)), Qt::DirectConnection); seek_tasks.clear(); bool was_end = false; if (ademuxer) { ademuxer->seek(0LL); } while (!end) { processNextSeekTask(); //vthread maybe changed by AVPlayer.setPriority() from no dec case vqueue = video_thread ? video_thread->packetQueue() : 0; if (demuxer->atEnd()) { // if avthread may skip 1st eof packet because of a/v sync if (aqueue && (!was_end || aqueue->isEmpty())) { aqueue->put(Packet::createEOF()); aqueue->blockEmpty(false); // do not block if buffer is not enough. block again on seek } if (vqueue && (!was_end || vqueue->isEmpty())) { vqueue->put(Packet::createEOF()); vqueue->blockEmpty(false); } m_buffering = false; Q_EMIT mediaStatusChanged(QtAV::BufferedMedia); was_end = true; // wait for a/v thread finished msleep(100); continue; } was_end = false; if (tryPause()) { continue; //the queue is empty and will block } updateBufferState(); if (!demuxer->readFrame()) { continue; } stream = demuxer->stream(); pkt = demuxer->packet(); Packet apkt; bool audio_has_pic = demuxer->hasAttacedPicture(); int a_ext = 0; if (ademuxer) { QMutexLocker locker(&buffer_mutex); Q_UNUSED(locker); if (ademuxer) { a_ext = -1; audio_has_pic = ademuxer->hasAttacedPicture(); // FIXME: buffer full but buffering!!! // avoid read external track everytime. aqueue may not block full // vqueue will not block if aqueue is not enough if (!aqueue->isFull() || aqueue->isBuffering()) { if (ademuxer->readFrame()) { if (ademuxer->stream() == ademuxer->audioStream()) { a_ext = 1; apkt = ademuxer->packet(); } } // no continue otherwise. ademuxer finished earlier than demuxer } } } //qDebug("vqueue: %d, aqueue: %d/isbuffering %d isfull: %d, buffer: %d/%d", vqueue->size(), aqueue->size(), aqueue->isBuffering(), aqueue->isFull(), aqueue->buffered(), aqueue->bufferValue()); //QMutexLocker locker(&buffer_mutex); //TODO: seems we do not need to lock //Q_UNUSED(locker); /*1 is empty but another is enough, then do not block to ensure the empty one can put packets immediatly. But usually it will not happen, why? */ /* demux thread will be blocked only when 1 queue is full and still put * if vqueue is full and aqueue becomes empty, then demux thread * will be blocked. so we should wake up another queue when empty(or threshold?). * TODO: the video stream and audio stream may be group by group. provide it * stream data: aaaaaaavvvvvvvaaaaaaaavvvvvvvvvaaaaaa, it happens * stream data: aavavvavvavavavavavavavavvvaavavavava, it's ok */ //TODO: use cache queue, take from cache queue if not empty? const bool a_internal = stream == demuxer->audioStream(); if (a_internal || a_ext > 0) {//apkt.isValid()) { if (a_internal && !a_ext) // internal is always read even if external audio used apkt = demuxer->packet(); /* if vqueue if not blocked and full, and aqueue is empty, then put to * vqueue will block demuex thread */ if (aqueue) { if (!audio_thread || !audio_thread->isRunning()) { aqueue->clear(); continue; } // must ensure bufferValue set correctly before continue if (m_buffer != aqueue) aqueue->setBufferValue(m_buffer->isBuffering() ? std::numeric_limits<qint64>::max() : buf2); // always block full if no vqueue because empty callback may set false // attached picture is cover for song, 1 frame aqueue->blockFull(!video_thread || !video_thread->isRunning() || !vqueue || audio_has_pic); // external audio: a_ext < 0, stream = audio_idx=>put invalid packet if (a_ext >= 0) aqueue->put(apkt); //affect video_thread } } // always check video stream if use external audio if (stream == demuxer->videoStream()) { if (vqueue) { if (!video_thread || !video_thread->isRunning()) { vqueue->clear(); continue; } vqueue->blockFull(!audio_thread || !audio_thread->isRunning() || !aqueue || aqueue->isEnough()); vqueue->put(pkt); //affect audio_thread } } else if (demuxer->subtitleStreams().contains(stream)) { //subtitle Q_EMIT internalSubtitlePacketRead(demuxer->subtitleStreams().indexOf(stream), pkt); } else { continue; } } m_buffering = false; m_buffer = 0; while (audio_thread && audio_thread->isRunning()) { qDebug("waiting audio thread......."); aqueue->blockEmpty(false); //FIXME: why need this audio_thread->wait(500); } while (video_thread && video_thread->isRunning()) { qDebug("waiting video thread......."); vqueue->blockEmpty(false); video_thread->wait(500); } thread->disconnect(this, SIGNAL(seekFinished(qint64))); qDebug("Demux thread stops running...."); emit mediaStatusChanged(QtAV::EndOfMedia); }
void AVDemuxThread::run() { end = false; if (audio_thread && !audio_thread->isRunning()) audio_thread->start(QThread::HighPriority); if (video_thread && !video_thread->isRunning()) video_thread->start(); int running_threads = 0; if (audio_thread) ++running_threads; if (video_thread) ++running_threads; qDebug("demux thread start running...%d avthreads", running_threads); audio_stream = demuxer->audioStream(); video_stream = demuxer->videoStream(); int index = 0; Packet pkt; pause(false); qDebug("get av queue a/v thread = %p %p", audio_thread, video_thread); PacketQueue *aqueue = audio_thread ? audio_thread->packetQueue() : 0; PacketQueue *vqueue = video_thread ? video_thread->packetQueue() : 0; if (aqueue) { aqueue->clear(); aqueue->setBlocking(true); } if (vqueue) { vqueue->clear(); vqueue->setBlocking(true); } while (!end) { processNextSeekTask(); #if RESUME_ONCE_ON_SEEK // resume once is not enough. readFrame() failed after seek (why?) and we need video packet processNextPauseTask(); if (tryPause()) { #else if (tryPause(ULONG_MAX)) { #endif continue; //the queue is empty and will block } running_threads = (audio_thread && audio_thread->isRunning()) + (video_thread && video_thread->isRunning()); if (!running_threads) { qDebug("no running avthreads. exit demuxer thread"); break; } QMutexLocker locker(&buffer_mutex); Q_UNUSED(locker); if (end) { break; } if (!demuxer->readFrame()) { continue; } index = demuxer->stream(); pkt = *demuxer->packet(); //TODO: how to avoid additional copy? //connect to stop is ok too if (pkt.isEnd()) { qDebug("read end packet %d A:%d V:%d", index, audio_stream, video_stream); end = true; //avthread can stop. do not clear queue, make sure all data are played if (audio_thread) { audio_thread->setDemuxEnded(true); } if (video_thread) { video_thread->setDemuxEnded(true); } break; } /*1 is empty but another is enough, then do not block to ensure the empty one can put packets immediatly. But usually it will not happen, why? */ /* demux thread will be blocked only when 1 queue is full and still put * if vqueue is full and aqueue becomes empty, then demux thread * will be blocked. so we should wake up another queue when empty(or threshold?). * TODO: the video stream and audio stream may be group by group. provide it * stream data: aaaaaaavvvvvvvaaaaaaaavvvvvvvvvaaaaaa, it happens * stream data: aavavvavvavavavavavavavavvvaavavavava, it's ok */ //TODO: use cache queue, take from cache queue if not empty? if (index == audio_stream) { /* if vqueue if not blocked and full, and aqueue is empty, then put to * vqueue will block demuex thread */ if (aqueue) { if (!audio_thread || !audio_thread->isRunning()) { aqueue->clear(); continue; } if (vqueue) aqueue->blockFull(vqueue->isEnough() || demuxer->hasAttacedPicture()); aqueue->put(pkt); //affect video_thread } } else if (index == video_stream) { if (!video_thread || !video_thread->isRunning()) { vqueue->clear(); continue; } if (vqueue) { if (aqueue) vqueue->blockFull(aqueue->isEnough()); vqueue->put(pkt); //affect audio_thread } } else { //subtitle continue; } } //flush. seeking will be omitted when stopped if (aqueue) aqueue->put(Packet()); if (vqueue) vqueue->put(Packet()); while (audio_thread && audio_thread->isRunning()) { qDebug("waiting audio thread......."); audio_thread->wait(500); } while (video_thread && video_thread->isRunning()) { qDebug("waiting video thread......."); video_thread->wait(500); } qDebug("Demux thread stops running...."); } bool AVDemuxThread::tryPause(unsigned long timeout) { if (!paused) return false; QMutexLocker lock(&buffer_mutex); Q_UNUSED(lock); cond.wait(&buffer_mutex, timeout); return true; } } //namespace QtAV
//TODO: if output is null or dummy, the use duration to wait void AudioThread::run() { DPTR_D(AudioThread); //No decoder or output. No audio output is ok, just display picture if (!d.dec || !d.dec->isAvailable()) return; resetState(); Q_ASSERT(d.clock != 0); AudioDecoder *dec = static_cast<AudioDecoder*>(d.dec); AudioOutput *ao = static_cast<AudioOutput*>(d.writer); int sample_rate = dec->codecContext()->sample_rate; int channels = dec->codecContext()->channels; int csf = channels * sample_rate * sizeof(float); static const double max_len = 0.02; d.last_pts = 0; while (!d.stop) { //TODO: why put it at the end of loop then playNextFrame() not work? if (tryPause()) { //DO NOT continue, or playNextFrame() will fail if (d.stop) break; //the queue is empty and may block. should setBlocking(false) wake up cond empty? } QMutexLocker locker(&d.mutex); Q_UNUSED(locker); if (d.packets.isEmpty() && !d.stop) { d.stop = d.demux_end; if (d.stop) { break; } } Packet pkt = d.packets.take(); //wait to dequeue if (!pkt.isValid()) { qDebug("Invalid packet! flush audio codec context!!!!!!!!"); dec->flush(); continue; } d.clock->updateValue(pkt.pts); //DO NOT decode and convert if ao is not available or mute! if (dec->decode(pkt.data)) { QByteArray decoded(dec->data()); int decodedSize = decoded.size(); int decodedPos = 0; qreal delay =0; while (decodedSize > 0) { int chunk = qMin(decodedSize, int(max_len*csf)); d.clock->updateDelay(delay += (qreal)chunk/(qreal)csf); QByteArray decodedChunk(chunk, 0); //volume == 0 || mute if (ao && ao->isAvailable()) { if (!ao->isMute()) { decodedChunk = QByteArray::fromRawData(decoded.constData() + decodedPos, chunk); qreal vol = ao->volume(); if (vol != 1.0) { int len = decodedChunk.size()/sizeof(float); //TODO: why??? float *data = (float*)decodedChunk.data(); for (int i = 0; i < len; ++i) data[i] *= vol; } } ao->writeData(decodedChunk); } else { /* * why need this even if we add delay? and usleep sounds weird * the advantage is if no audio device, the play speed is ok too * So is portaudio blocking the thread when playing? */ static bool sWarn_no_ao = true; //FIXME: no warning when replay. warn only once if (sWarn_no_ao) { qDebug("Audio output not available! msleep(%lu)", (unsigned long)((qreal)chunk/(qreal)csf * 1000)); sWarn_no_ao = false; } //TODO: avoid acummulative error. External clock? msleep((unsigned long)((qreal)chunk/(qreal)csf * 1000.0)); } decodedPos += chunk; decodedSize -= chunk; } } else { //qWarning("Decode audio failed"); qreal dt = pkt.pts - d.last_pts; if (abs(dt) > 0.618 || dt < 0) { dt = 0; } //qDebug("sleep %f", dt); //TODO: avoid acummulative error. External clock? msleep((unsigned long)(dt*1000.0)); } d.last_pts = d.clock->value(); //not pkt.pts! the delay is updated! } qDebug("Audio thread stops running..."); }