streamContext *FFmpegImportFileHandle::ReadNextFrame()
{
// Get pointer to array of contiguous unique_ptrs
auto scs = mScs->get();
// This reinterpret_cast to array of plain pointers is innocent
return import_ffmpeg_read_next_frame
(mFormatContext, reinterpret_cast<streamContext**>(scs), mNumStreams);
}
//these next few look highly refactorable.
//get the right stream pointer.
streamContext* ODFFmpegDecoder::ReadNextFrame()
{
// Get pointer to array of contiguous unique_ptrs
auto scs = mScs->get();
// This reinterpret_cast to array of plain pointers is innocent
return import_ffmpeg_read_next_frame
(mContext->ic_ptr, reinterpret_cast<streamContext**>(scs), mChannels.size());
}
int FFmpegImportFileHandle::WriteData(streamContext *sc)
{
// Find the stream index in mScs array
int streamid = -1;
auto iter = mChannels.begin();
auto scs = mScs->get();
for (int i = 0; i < mNumStreams; ++iter, ++i)
{
if (scs[i].get() == sc)
{
streamid = i;
break;
}
}
// Stream is not found. This should not really happen
if (streamid == -1)
{
return 1;
}
// Allocate the buffer to store audio.
int insamples = sc->m_decodedAudioSamplesValidSiz / sc->m_samplesize;
int nChannels = sc->m_stream->codec->channels < sc->m_initialchannels ? sc->m_stream->codec->channels : sc->m_initialchannels;
uint8_t **tmp = (uint8_t **) malloc(sizeof(uint8_t *) * nChannels);
for (int chn = 0; chn < nChannels; chn++)
{
tmp[chn] = (uint8_t *) malloc(sc->m_osamplesize * (insamples / nChannels));
}
// Separate the channels and convert input sample format to 16-bit
uint8_t *in = sc->m_decodedAudioSamples;
int index = 0;
int pos = 0;
while (pos < insamples)
{
for (int chn = 0; chn < sc->m_stream->codec->channels; chn++)
{
if (chn < nChannels)
{
switch (sc->m_samplefmt)
{
case AV_SAMPLE_FMT_U8:
case AV_SAMPLE_FMT_U8P:
((int16_t *)tmp[chn])[index] = (int16_t) (*(uint8_t *)in - 0x80) << 8;
break;
case AV_SAMPLE_FMT_S16:
case AV_SAMPLE_FMT_S16P:
((int16_t *)tmp[chn])[index] = (int16_t) *(int16_t *)in;
break;
case AV_SAMPLE_FMT_S32:
case AV_SAMPLE_FMT_S32P:
((float *)tmp[chn])[index] = (float) *(int32_t *)in * (1.0 / (1 << 31));
break;
case AV_SAMPLE_FMT_FLT:
case AV_SAMPLE_FMT_FLTP:
((float *)tmp[chn])[index] = (float) *(float *)in;
break;
case AV_SAMPLE_FMT_DBL:
case AV_SAMPLE_FMT_DBLP:
((float *)tmp[chn])[index] = (float) *(double *)in;
break;
default:
wxLogError(wxT("Stream %d has unrecognized sample format %d."), streamid, sc->m_samplefmt);
for (int chn=0; chn < nChannels; chn++)
{
free(tmp[chn]);
}
free(tmp);
return 1;
break;
}
}
in += sc->m_samplesize;
pos++;
}
index++;
}
// Write audio into WaveTracks
auto iter2 = iter->begin();
for (int chn=0; chn < nChannels; ++iter2, ++chn)
{
iter2->get()->Append((samplePtr)tmp[chn],sc->m_osamplefmt,index);
free(tmp[chn]);
}
free(tmp);
// Try to update the progress indicator (and see if user wants to cancel)
int updateResult = eProgressSuccess;
int64_t filesize = avio_size(mFormatContext->pb);
// PTS (presentation time) is the proper way of getting current position
if (sc->m_pkt.pts != int64_t(AV_NOPTS_VALUE) && mFormatContext->duration != int64_t(AV_NOPTS_VALUE))
{
mProgressPos = sc->m_pkt.pts * sc->m_stream->time_base.num / sc->m_stream->time_base.den;
mProgressLen = (mFormatContext->duration > 0 ? mFormatContext->duration / AV_TIME_BASE: 1);
}
// When PTS is not set, use number of frames and number of current frame
else if (sc->m_stream->nb_frames > 0 && sc->m_codecCtx->frame_number > 0 && sc->m_codecCtx->frame_number <= sc->m_stream->nb_frames)
{
mProgressPos = sc->m_codecCtx->frame_number;
mProgressLen = sc->m_stream->nb_frames;
}
// When number of frames is unknown, use position in file
else if (filesize > 0 && sc->m_pkt.pos > 0 && sc->m_pkt.pos <= filesize)
{
mProgressPos = sc->m_pkt.pos;
mProgressLen = filesize;
}
updateResult = mProgress->Update(mProgressPos, mProgressLen != 0 ? mProgressLen : 1);
return updateResult;
}
int FFmpegImportFileHandle::Import(TrackFactory *trackFactory,
TrackHolders &outTracks,
Tags *tags)
{
outTracks.clear();
CreateProgress();
// Remove stream contexts which are not marked for importing and adjust mScs and mNumStreams accordingly
const auto scs = mScs->get();
for (int i = 0; i < mNumStreams;)
{
if (!scs[i]->m_use)
{
for (int j = i; j < mNumStreams - 1; j++)
{
scs[j] = std::move(scs[j+1]);
}
mNumStreams--;
}
else i++;
}
mChannels.resize(mNumStreams);
int s = -1;
for (auto &stream : mChannels)
{
++s;
auto sc = scs[s].get();
switch (sc->m_stream->codec->sample_fmt)
{
case AV_SAMPLE_FMT_U8:
case AV_SAMPLE_FMT_S16:
case AV_SAMPLE_FMT_U8P:
case AV_SAMPLE_FMT_S16P:
sc->m_osamplesize = sizeof(int16_t);
sc->m_osamplefmt = int16Sample;
break;
default:
sc->m_osamplesize = sizeof(float);
sc->m_osamplefmt = floatSample;
break;
}
// There is a possibility that number of channels will change over time, but we do not have WaveTracks for NEW channels. Remember the number of channels and stick to it.
sc->m_initialchannels = sc->m_stream->codec->channels;
stream.resize(sc->m_stream->codec->channels);
int c = -1;
for (auto &channel : stream)
{
++c;
channel = trackFactory->NewWaveTrack(sc->m_osamplefmt, sc->m_stream->codec->sample_rate);
if (sc->m_stream->codec->channels == 2)
{
switch (c)
{
case 0:
channel->SetChannel(Track::LeftChannel);
channel->SetLinked(true);
break;
case 1:
channel->SetChannel(Track::RightChannel);
break;
}
}
else
{
channel->SetChannel(Track::MonoChannel);
}
}
}
// Handles the start_time by creating silence. This may or may not be correct.
// There is a possibility that we should ignore first N milliseconds of audio instead. I do not know.
/// TODO: Nag FFmpeg devs about start_time until they finally say WHAT is this and HOW to handle it.
s = -1;
for (auto &stream : mChannels)
{
++s;
int64_t stream_delay = 0;
auto sc = scs[s].get();
if (sc->m_stream->start_time != int64_t(AV_NOPTS_VALUE) && sc->m_stream->start_time > 0)
{
stream_delay = sc->m_stream->start_time;
wxLogDebug(wxT("Stream %d start_time = %lld, that would be %f milliseconds."), s, (long long) sc->m_stream->start_time, double(sc->m_stream->start_time)/AV_TIME_BASE*1000);
}
if (stream_delay != 0)
{
int c = -1;
for (auto &channel : stream)
{
++c;
WaveTrack *t = channel.get();
t->InsertSilence(0,double(stream_delay)/AV_TIME_BASE);
}
}
}
// This is the heart of the importing process
// The result of Import() to be returend. It will be something other than zero if user canceled or some error appears.
int res = eProgressSuccess;
#ifdef EXPERIMENTAL_OD_FFMPEG
mUsingOD = false;
gPrefs->Read(wxT("/Library/FFmpegOnDemand"), &mUsingOD);
//at this point we know the file is good and that we have to load the number of channels in mScs[s]->m_stream->codec->channels;
//so for OD loading we create the tracks and releasee the modal lock after starting the ODTask.
if (mUsingOD) {
std::vector<ODDecodeFFmpegTask*> tasks;
//append blockfiles to each stream and add an individual ODDecodeTask for each one.
s = -1;
for (const auto &stream : mChannels) {
++s;
ODDecodeFFmpegTask* odTask =
new ODDecodeFFmpegTask(mScs, ODDecodeFFmpegTask::FromList(mChannels), mFormatContext, s);
odTask->CreateFileDecoder(mFilename);
//each stream has different duration. We need to know it if seeking is to be allowed.
sampleCount sampleDuration = 0;
auto sc = scs[s].get();
if (sc->m_stream->duration > 0)
sampleDuration = ((sampleCount)sc->m_stream->duration * sc->m_stream->time_base.num), sc->m_stream->codec->sample_rate / sc->m_stream->time_base.den;
else
sampleDuration = ((sampleCount)mFormatContext->duration *sc->m_stream->codec->sample_rate) / AV_TIME_BASE;
// printf(" OD duration samples %qi, sr %d, secs %d\n",sampleDuration, (int)sc->m_stream->codec->sample_rate, (int)sampleDuration/sc->m_stream->codec->sample_rate);
//for each wavetrack within the stream add coded blockfiles
for (int c = 0; c < sc->m_stream->codec->channels; c++) {
WaveTrack *t = stream[c].get();
odTask->AddWaveTrack(t);
sampleCount maxBlockSize = t->GetMaxBlockSize();
//use the maximum blockfile size to divide the sections (about 11secs per blockfile at 44.1khz)
for (sampleCount i = 0; i < sampleDuration; i += maxBlockSize) {
sampleCount blockLen = maxBlockSize;
if (i + blockLen > sampleDuration)
blockLen = sampleDuration - i;
t->AppendCoded(mFilename, i, blockLen, c, ODTask::eODFFMPEG);
// This only works well for single streams since we assume
// each stream is of the same duration and channels
res = mProgress->Update(i+sampleDuration*c+ sampleDuration*sc->m_stream->codec->channels*s,
sampleDuration*sc->m_stream->codec->channels*mNumStreams);
if (res != eProgressSuccess)
break;
}
}
tasks.push_back(odTask);
}
//Now we add the tasks and let them run, or DELETE them if the user cancelled
for(int i=0; i < (int)tasks.size(); i++) {
if(res==eProgressSuccess)
ODManager::Instance()->AddNewTask(tasks[i]);
else
{
delete tasks[i];
}
}
} else {
#endif
// Read next frame.
for (streamContext *sc; (sc = ReadNextFrame()) != NULL && (res == eProgressSuccess);)
{
// ReadNextFrame returns 1 if stream is not to be imported
if (sc != (streamContext*)1)
{
// Decode frame until it is not possible to decode any further
while (sc->m_pktRemainingSiz > 0 && (res == eProgressSuccess || res == eProgressStopped))
{
if (DecodeFrame(sc,false) < 0)
break;
// If something useable was decoded - write it to mChannels
if (sc->m_frameValid)
res = WriteData(sc);
}
// Cleanup after frame decoding
if (sc->m_pktValid)
{
av_free_packet(&sc->m_pkt);
sc->m_pktValid = 0;
}
}
}
// Flush the decoders.
if ((mNumStreams != 0) && (res == eProgressSuccess || res == eProgressStopped))
{
for (int i = 0; i < mNumStreams; i++)
{
auto sc = scs[i].get();
if (DecodeFrame(sc, true) == 0)
{
WriteData(sc);
if (sc->m_pktValid)
{
av_free_packet(&sc->m_pkt);
sc->m_pktValid = 0;
}
}
}
}
#ifdef EXPERIMENTAL_OD_FFMPEG
} // else -- !mUsingOD == true
#endif //EXPERIMENTAL_OD_FFMPEG
// Something bad happened - destroy everything!
if (res == eProgressCancelled || res == eProgressFailed)
return res;
//else if (res == 2), we just stop the decoding as if the file has ended
// Copy audio from mChannels to newly created tracks (destroying mChannels elements in process)
for (auto &stream : mChannels)
{
for(auto &channel : stream)
{
channel->Flush();
outTracks.push_back(std::move(channel));
}
}
// Save metadata
WriteMetadata(tags);
return res;
}
bool FFmpegImportFileHandle::InitCodecs()
{
// Allocate the array of pointers to hold stream contexts pointers
// Some of the allocated space may be unused (corresponds to video, subtitle, or undecodeable audio streams)
mScs = std::make_shared<Scs>(mFormatContext->nb_streams);
// Fill the stream contexts
for (unsigned int i = 0; i < mFormatContext->nb_streams; i++)
{
if (mFormatContext->streams[i]->codec->codec_type == AVMEDIA_TYPE_AUDIO)
{
//Create a context
auto sc = std::make_unique<streamContext>();
sc->m_stream = mFormatContext->streams[i];
sc->m_codecCtx = sc->m_stream->codec;
AVCodec *codec = avcodec_find_decoder(sc->m_codecCtx->codec_id);
if (codec == NULL)
{
wxLogError(wxT("FFmpeg : avcodec_find_decoder() failed. Index[%02d], Codec[%02x - %s]"),i,sc->m_codecCtx->codec_id,sc->m_codecCtx->codec_name);
//FFmpeg can't decode this stream, skip it
continue;
}
if (codec->type != sc->m_codecCtx->codec_type)
{
wxLogError(wxT("FFmpeg : Codec type mismatch, skipping. Index[%02d], Codec[%02x - %s]"),i,sc->m_codecCtx->codec_id,sc->m_codecCtx->codec_name);
//Non-audio codec reported as audio? Nevertheless, we don't need THIS.
continue;
}
if (avcodec_open2(sc->m_codecCtx, codec, NULL) < 0)
{
wxLogError(wxT("FFmpeg : avcodec_open() failed. Index[%02d], Codec[%02x - %s]"),i,sc->m_codecCtx->codec_id,sc->m_codecCtx->codec_name);
//Can't open decoder - skip this stream
continue;
}
// Stream is decodeable and it is audio. Add it and its decription to the arrays
wxString strinfo;
int duration = 0;
if (sc->m_stream->duration > 0)
duration = sc->m_stream->duration * sc->m_stream->time_base.num / sc->m_stream->time_base.den;
else
duration = mFormatContext->duration / AV_TIME_BASE;
wxString bitrate = wxT("");
if (sc->m_codecCtx->bit_rate > 0)
bitrate.Printf(wxT("%d"),sc->m_codecCtx->bit_rate);
else
bitrate.Printf(wxT("?"));
AVDictionaryEntry *tag = av_dict_get(sc->m_stream->metadata, "language", NULL, 0);
wxString lang;
if (tag)
{
lang.FromUTF8(tag->value);
}
strinfo.Printf(_("Index[%02x] Codec[%s], Language[%s], Bitrate[%s], Channels[%d], Duration[%d]"),sc->m_stream->id,codec->name,lang.c_str(),bitrate.c_str(),sc->m_stream->codec->channels, duration);
mStreamInfo->Add(strinfo);
mScs->get()[mNumStreams++] = std::move(sc);
}
//for video and unknown streams do nothing
}
//It doesn't really returns false, but GetStreamCount() will return 0 if file is composed entierly of unreadable streams
return true;
}
int ODFFmpegDecoder::Decode(SampleBuffer & data, sampleFormat & format, sampleCount start, size_t len, unsigned int channel)
{
auto mFormatContext = mContext->ic_ptr;
auto scs = mScs->get();
auto sci = scs[mStreamIndex].get();
format = sci->m_osamplefmt;
data.Allocate(len, format);
samplePtr bufStart = data.ptr();
streamContext* sc = NULL;
// printf("start %llu len %lu\n", start, len);
//TODO update this to work with seek - this only works linearly now.
if(mCurrentPos > start && mCurrentPos <= start+len + kDecodeSampleAllowance)
{
//this next call takes data, start and len as reference variables and updates them to reflect the NEW area that is needed.
FillDataFromCache(bufStart, format, start,len,channel);
}
bool seeking = false;
//look at the decoding timestamp and see if the next sample that will be decoded is not the next sample we need.
if(len && (mCurrentPos > start + len || mCurrentPos + kDecodeSampleAllowance < start ) && SeekingAllowed()) {
sc = sci;
AVStream* st = sc->m_stream;
int stindex = -1;
uint64_t targetts;
//printf("attempting seek to %llu\n", start);
//we have to find the index for this stream.
for (unsigned int i = 0; i < mFormatContext->nb_streams; i++) {
if (mFormatContext->streams[i] == sc->m_stream )
stindex =i;
}
if(stindex >=0) {
int numAttempts = 0;
//reset mCurrentPos to a bogus value
mCurrentPos = start+len +1;
while(numAttempts++ < kMaxSeekRewindAttempts && mCurrentPos > start) {
//we want to move slightly before the start of the block file, but not too far ahead
targetts =
(start - kDecodeSampleAllowance * numAttempts / kMaxSeekRewindAttempts)
.as_long_long() *
((double)st->time_base.den/(st->time_base.num * st->codec->sample_rate ));
if(targetts<0)
targetts=0;
//printf("attempting seek to %llu, attempts %d\n", targetts, numAttempts);
if(av_seek_frame(mFormatContext,stindex,targetts,0) >= 0){
//find out the dts we've seekd to.
sampleCount actualDecodeStart = 0.5 + st->codec->sample_rate * st->cur_dts * ((double)st->time_base.num/st->time_base.den); //this is mostly safe because den is usually 1 or low number but check for high values.
mCurrentPos = actualDecodeStart;
seeking = true;
//if the seek was past our desired position, rewind a bit.
//printf("seek ok to %llu samps, float: %f\n",actualDecodeStart,actualDecodeStartDouble);
} else {
printf("seek failed");
break;
}
}
if(mCurrentPos>start){
mSeekingAllowedStatus = (bool)ODFFMPEG_SEEKING_TEST_FAILED;
// url_fseek(mFormatContext->pb,sc->m_pkt.pos,SEEK_SET);
printf("seek fail, reverting to previous pos\n");
return -1;
}
}
}
bool firstpass = true;
//we decode up to the end of the blockfile
while (len>0 && (mCurrentPos < start+len) && (sc = ReadNextFrame()) != NULL)
{
// ReadNextFrame returns 1 if stream is not to be imported
if (sc != (streamContext*)1)
{
//find out the dts we've seekd to. can't use the stream->cur_dts because it is faulty. also note that until we do the first seek, pkt.dts can be false and will change for the same samples after the initial seek.
auto actualDecodeStart = mCurrentPos;
// we need adjacent samples, so don't use dts most of the time which will leave gaps between frames
// for some formats
// The only other case for inserting silence is for initial offset and ImportFFmpeg.cpp does this for us
if (seeking) {
actualDecodeStart = 0.52 + (sc->m_stream->codec->sample_rate * sc->m_pkt->dts
* ((double)sc->m_stream->time_base.num / sc->m_stream->time_base.den));
//this is mostly safe because den is usually 1 or low number but check for high values.
//hack to get rounding to work to neareset frame size since dts isn't exact
if (sc->m_stream->codec->frame_size) {
actualDecodeStart = ((actualDecodeStart + sc->m_stream->codec->frame_size/2) / sc->m_stream->codec->frame_size) * sc->m_stream->codec->frame_size;
}
// reset for the next one
seeking = false;
}
if(actualDecodeStart != mCurrentPos)
printf("ts not matching - now:%llu , last:%llu, lastlen:%lu, start %llu, len %lu\n",actualDecodeStart.as_long_long(), mCurrentPos.as_long_long(), mCurrentLen, start.as_long_long(), len);
//if we've skipped over some samples, fill the gap with silence. This could happen often in the beginning of the file.
if(actualDecodeStart>start && firstpass) {
// find the number of samples for the leading silence
// UNSAFE_SAMPLE_COUNT_TRUNCATION
// -- but used only experimentally as of this writing
// Is there a proof size_t will not overflow size_t?
// Result is surely nonnegative.
auto amt = (actualDecodeStart - start).as_size_t();
auto cache = make_movable<FFMpegDecodeCache>();
//printf("skipping/zeroing %i samples. - now:%llu (%f), last:%llu, lastlen:%lu, start %llu, len %lu\n",amt,actualDecodeStart, actualDecodeStartdouble, mCurrentPos, mCurrentLen, start, len);
//put it in the cache so the other channels can use it.
// wxASSERT(sc->m_stream->codec->channels > 0);
cache->numChannels = sc->m_stream->codec->channels;
cache->len = amt;
cache->start=start;
// 8 bit and 16 bit audio output from ffmpeg means
// 16 bit int out.
// 32 bit int, float, double mean float out.
if (format == int16Sample)
cache->samplefmt = AV_SAMPLE_FMT_S16;
else
cache->samplefmt = AV_SAMPLE_FMT_FLT;
cache->samplePtr = (uint8_t*) malloc(amt * cache->numChannels * SAMPLE_SIZE(format));
memset(cache->samplePtr, 0, amt * cache->numChannels * SAMPLE_SIZE(format));
InsertCache(std::move(cache));
}
firstpass=false;
mCurrentPos = actualDecodeStart;
//decode the entire packet (unused bits get saved in cache, so as long as cache size limit is bigger than the
//largest packet size, we're ok.
while (sc->m_pktRemainingSiz > 0)
//Fill the cache with decoded samples
if (DecodeFrame(sc,false) < 0)
break;
// Cleanup after frame decoding
sc->m_pkt.reset();
}
}
// Flush the decoders if we're done.
if((!sc || sc == (streamContext*) 1)&& len>0)
{
for (int i = 0; i < mChannels.size(); i++)
{
sc = scs[i].get();
sc->m_pkt.create();
if (DecodeFrame(sc, true) == 0)
{
sc->m_pkt.reset();
}
}
}
//this next call takes data, start and len as reference variables and updates them to reflect the NEW area that is needed.
FillDataFromCache(bufStart, format, start, len, channel);
// CHECK: not sure if we need this. In any case it has to be updated for the NEW float case (not just int16)
//if for some reason we couldn't get the samples, fill them with silence
/*
int16_t* outBuf = (int16_t*) bufStart;
for(int i=0;i<len;i++)
outBuf[i]=0;
*/
return 1;
}