void VideoReaderUnit::ProcessFrame(FrameSetPtr input, list<FrameSetPtr>* output) {
if (!used_as_root_) {
input->push_back(shared_ptr<VideoFrame>(ReadNextFrame()));
output->push_back(input);
++frame_num_;
}
}
int H264FileReader::Init()
{
//assert(IsOpened());
//assert(0 == ftell(m_fp));
assert(m_ptr == search_start_code(m_ptr, m_bytes));
long offset = 0;
size_t count = 0;
bool spspps = true;
const unsigned char* nalu = m_ptr;
do
{
const unsigned char* nalu2 = ReadNextFrame();
nalu = m_ptr + m_offset;
int nal_unit_type = h264_nal_type(nalu);
assert(0 != nal_unit_type);
if(nal_unit_type <= 5)
{
if(m_sps.size() > 0) spspps = false; // don't need more sps/pps
long n = ftell(m_fp);
vframe_t frame;
frame.offset = offset;
frame.bytes = (nalu2 ? toOffset(nalu2) : n) - offset;
frame.time = 40 * count++;
frame.idr = 5 == nal_unit_type; // IDR-frame
m_videos.push_back(frame);
offset += frame.bytes;
}
else if(NAL_SPS == nal_unit_type || NAL_PPS == nal_unit_type)
{
assert(nalu2);
if(spspps && nalu2)
{
size_t n = 0x01 == nalu[2] ? 3 : 4;
sps_t sps(nalu2 - nalu - n);
memcpy(&sps[0], nalu+n, nalu2-nalu-n);
// filter last 0x00 bytes
while(sps.size() > 0 && !*sps.rbegin())
sps.resize(sps.size()-1);
m_sps.push_back(sps);
}
}
nalu = nalu2;
m_offset = nalu - m_ptr;
} while(nalu);
m_duration = 40 * count;
return 0;
}
bool
CMNGAnimation::ReadNextFrame(BGRA* frame_buffer)
{
if (!ReadNextFrame((RGBA*)frame_buffer)) {
return false;
}
// switch red and blue
for (int i = 0; i < m_width * m_height; i++) {
std::swap(frame_buffer[i].red, frame_buffer[i].blue);
}
return true;
}
bool VideoReaderUnit::PostProcess(list<FrameSetPtr>* append) {
if (used_as_root_) {
VideoFrame* next_frame = ReadNextFrame();
if (next_frame != nullptr) {
// Make new frameset and push VideoFrame.
FrameSetPtr frame_set (new FrameSet());
frame_set->push_back(shared_ptr<VideoFrame>(next_frame));
append->push_back(frame_set);
++frame_num_;
return true;
} else {
return false;
}
}
return false;
}
int ODFFmpegDecoder::Decode(SampleBuffer & data, sampleFormat & format, sampleCount start, sampleCount len, unsigned int channel)
{
format = mScs[mStreamIndex]->m_osamplefmt;
data.Allocate(len, format);
samplePtr bufStart = data.ptr();
streamContext* sc = NULL;
// printf("start %llu len %llu\n", start, len);
//TODO update this to work with seek - this only works linearly now.
if(mCurrentPos > start && mCurrentPos <= start+len + kDecodeSampleAllowance)
{
//this next call takes data, start and len as reference variables and updates them to reflect the NEW area that is needed.
FillDataFromCache(bufStart, format, start,len,channel);
}
bool seeking = false;
//look at the decoding timestamp and see if the next sample that will be decoded is not the next sample we need.
if(len && (mCurrentPos > start + len || mCurrentPos + kDecodeSampleAllowance < start ) && SeekingAllowed()) {
sc = mScs[mStreamIndex];
AVStream* st = sc->m_stream;
int stindex = -1;
uint64_t targetts;
//printf("attempting seek to %llu\n", start);
//we have to find the index for this stream.
for (unsigned int i = 0; i < mFormatContext->nb_streams; i++) {
if (mFormatContext->streams[i] == sc->m_stream )
stindex =i;
}
if(stindex >=0) {
int numAttempts = 0;
//reset mCurrentPos to a bogus value
mCurrentPos = start+len +1;
while(numAttempts++ < kMaxSeekRewindAttempts && mCurrentPos > start) {
//we want to move slightly before the start of the block file, but not too far ahead
targetts = (start-kDecodeSampleAllowance*numAttempts/kMaxSeekRewindAttempts) * ((double)st->time_base.den/(st->time_base.num * st->codec->sample_rate ));
if(targetts<0)
targetts=0;
//printf("attempting seek to %llu, attempts %d\n", targetts, numAttempts);
if(av_seek_frame(mFormatContext,stindex,targetts,0) >= 0){
//find out the dts we've seekd to.
sampleCount actualDecodeStart = 0.5 + st->codec->sample_rate * st->cur_dts * ((double)st->time_base.num/st->time_base.den); //this is mostly safe because den is usually 1 or low number but check for high values.
mCurrentPos = actualDecodeStart;
seeking = true;
//if the seek was past our desired position, rewind a bit.
//printf("seek ok to %llu samps, float: %f\n",actualDecodeStart,actualDecodeStartDouble);
} else {
printf("seek failed");
break;
}
}
if(mCurrentPos>start){
mSeekingAllowedStatus = (bool)ODFFMPEG_SEEKING_TEST_FAILED;
// url_fseek(mFormatContext->pb,sc->m_pkt.pos,SEEK_SET);
printf("seek fail, reverting to previous pos\n");
return -1;
}
}
}
bool firstpass = true;
//we decode up to the end of the blockfile
while (len>0 && (mCurrentPos < start+len) && (sc = ReadNextFrame()) != NULL)
{
// ReadNextFrame returns 1 if stream is not to be imported
if (sc != (streamContext*)1)
{
//find out the dts we've seekd to. can't use the stream->cur_dts because it is faulty. also note that until we do the first seek, pkt.dts can be false and will change for the same samples after the initial seek.
sampleCount actualDecodeStart = mCurrentPos;
// we need adjacent samples, so don't use dts most of the time which will leave gaps between frames
// for some formats
// The only other case for inserting silence is for initial offset and ImportFFmpeg.cpp does this for us
if (seeking) {
actualDecodeStart = 0.52 + (sc->m_stream->codec->sample_rate * sc->m_pkt.dts
* ((double)sc->m_stream->time_base.num / sc->m_stream->time_base.den));
//this is mostly safe because den is usually 1 or low number but check for high values.
//hack to get rounding to work to neareset frame size since dts isn't exact
if (sc->m_stream->codec->frame_size) {
actualDecodeStart = ((actualDecodeStart + sc->m_stream->codec->frame_size/2) / sc->m_stream->codec->frame_size) * sc->m_stream->codec->frame_size;
}
// reset for the next one
seeking = false;
}
if(actualDecodeStart != mCurrentPos)
printf("ts not matching - now:%llu , last:%llu, lastlen:%llu, start %llu, len %llu\n",actualDecodeStart, mCurrentPos, mCurrentLen, start, len);
//if we've skipped over some samples, fill the gap with silence. This could happen often in the beginning of the file.
if(actualDecodeStart>start && firstpass) {
// find the number of samples for the leading silence
int amt = actualDecodeStart - start;
FFMpegDecodeCache* cache = new FFMpegDecodeCache;
//printf("skipping/zeroing %i samples. - now:%llu (%f), last:%llu, lastlen:%llu, start %llu, len %llu\n",amt,actualDecodeStart, actualDecodeStartdouble, mCurrentPos, mCurrentLen, start, len);
//put it in the cache so the other channels can use it.
cache->numChannels = sc->m_stream->codec->channels;
cache->len = amt;
cache->start=start;
// 8 bit and 16 bit audio output from ffmpeg means
// 16 bit int out.
// 32 bit int, float, double mean float out.
if (format == int16Sample)
cache->samplefmt = SAMPLE_FMT_S16;
else
cache->samplefmt = SAMPLE_FMT_FLT;
cache->samplePtr = (uint8_t*) malloc(amt * cache->numChannels * SAMPLE_SIZE(format));
memset(cache->samplePtr, 0, amt * cache->numChannels * SAMPLE_SIZE(format));
InsertCache(cache);
}
firstpass=false;
mCurrentPos = actualDecodeStart;
//decode the entire packet (unused bits get saved in cache, so as long as cache size limit is bigger than the
//largest packet size, we're ok.
while (sc->m_pktRemainingSiz > 0)
//Fill the cache with decoded samples
if (DecodeFrame(sc,false) < 0)
break;
// Cleanup after frame decoding
if (sc->m_pktValid)
{
av_free_packet(&sc->m_pkt);
sc->m_pktValid = 0;
}
}
}
// Flush the decoders if we're done.
if((!sc || sc == (streamContext*) 1)&& len>0)
{
for (int i = 0; i < mNumStreams; i++)
{
if (DecodeFrame(mScs[i], true) == 0)
{
if (mScs[i]->m_pktValid)
{
av_free_packet(&mScs[i]->m_pkt);
mScs[i]->m_pktValid = 0;
}
}
}
}
//this next call takes data, start and len as reference variables and updates them to reflect the NEW area that is needed.
FillDataFromCache(bufStart, format, start, len, channel);
// CHECK: not sure if we need this. In any case it has to be updated for the NEW float case (not just int16)
//if for some reason we couldn't get the samples, fill them with silence
/*
int16_t* outBuf = (int16_t*) bufStart;
for(int i=0;i<len;i++)
outBuf[i]=0;
*/
return 1;
}
int FFmpegImportFileHandle::Import(TrackFactory *trackFactory,
Track ***outTracks,
int *outNumTracks,
Tags *tags)
{
CreateProgress();
// Remove stream contexts which are not marked for importing and adjust mScs and mNumStreams accordingly
for (int i = 0; i < mNumStreams;)
{
if (!mScs[i]->m_use)
{
delete mScs[i];
for (int j = i; j < mNumStreams - 1; j++)
{
mScs[j] = mScs[j+1];
}
mNumStreams--;
}
else i++;
}
mChannels = new WaveTrack **[mNumStreams];
mNumSamples = 0;
for (int s = 0; s < mNumStreams; s++)
{
// As you can see, it's really a number of frames.
// TODO: use something other than nb_frames for progress reporting (nb_frames is not available for some formats). Maybe something from the format context?
mNumSamples += mScs[s]->m_stream->nb_frames;
// There is a possibility that number of channels will change over time, but we do not have WaveTracks for new channels. Remember the number of channels and stick to it.
mScs[s]->m_initialchannels = mScs[s]->m_stream->codec->channels;
mChannels[s] = new WaveTrack *[mScs[s]->m_stream->codec->channels];
int c;
for (c = 0; c < mScs[s]->m_stream->codec->channels; c++)
{
mChannels[s][c] = trackFactory->NewWaveTrack(int16Sample, mScs[s]->m_stream->codec->sample_rate);
if (mScs[s]->m_stream->codec->channels == 2)
{
switch (c)
{
case 0:
mChannels[s][c]->SetChannel(Track::LeftChannel);
mChannels[s][c]->SetLinked(true);
break;
case 1:
mChannels[s][c]->SetChannel(Track::RightChannel);
mChannels[s][c]->SetTeamed(true);
break;
}
}
else
{
mChannels[s][c]->SetChannel(Track::MonoChannel);
}
}
}
// Handles the start_time by creating silence. This may or may not be correct.
// There is a possibility that we should ignore first N milliseconds of audio instead. I do not know.
/// TODO: Nag FFmpeg devs about start_time until they finally say WHAT is this and HOW to handle it.
for (int s = 0; s < mNumStreams; s++)
{
int64_t stream_delay = 0;
if (mScs[s]->m_stream->start_time != int64_t(AV_NOPTS_VALUE))
{
stream_delay = mScs[s]->m_stream->start_time;
wxLogMessage(wxT("Stream %d start_time = %d, that would be %f milliseconds."), s, mScs[s]->m_stream->start_time, double(mScs[s]->m_stream->start_time)/AV_TIME_BASE*1000);
}
if (stream_delay != 0)
{
for (int c = 0; c < mScs[s]->m_stream->codec->channels; c++)
{
WaveTrack *t = mChannels[s][c];
t->InsertSilence(0,double(stream_delay)/AV_TIME_BASE);
}
}
}
// This is the heart of the importing process
streamContext *sc = NULL;
// The result of Import() to be returend. It will be something other than zero if user canceled or some error appears.
int res = 0;
// Read next frame.
while ((sc = ReadNextFrame()) != NULL && (res == 0))
{
// ReadNextFrame returns 1 if stream is not to be imported
if (sc != (streamContext*)1)
{
// Decode frame until it is not possible to decode any further
while (sc->m_pktRemainingSiz > 0 && (res == 0))
{
if (DecodeFrame(sc,false) < 0)
break;
// If something useable was decoded - write it to mChannels
if (sc->m_frameValid)
res = WriteData(sc);
}
// Cleanup after frame decoding
if (sc->m_pktValid)
{
av_free_packet(&sc->m_pkt);
sc->m_pktValid = 0;
}
}
}
// Flush the decoders.
if ((mNumStreams != 0) && (res == 0))
{
for (int i = 0; i < mNumStreams; i++)
{
if (DecodeFrame(mScs[i], true) == 0)
{
WriteData(mScs[i]);
if (mScs[i]->m_pktValid)
{
av_free_packet(&mScs[i]->m_pkt);
mScs[i]->m_pktValid = 0;
}
}
}
}
// Something bad happened - destroy everything!
if (res)
{
for (int s = 0; s < mNumStreams; s++)
{
delete[] mChannels[s];
}
delete[] mChannels;
if (mCancelled)
return eImportCancelled;
else
return eImportFailed;
}
*outNumTracks = 0;
for (int s = 0; s < mNumStreams; s++)
{
*outNumTracks += mScs[s]->m_stream->codec->channels;
}
// Create new tracks
*outTracks = new Track *[*outNumTracks];
// Copy audio from mChannels to newly created tracks (destroying mChannels elements in process)
int trackindex = 0;
for (int s = 0; s < mNumStreams; s++)
{
for(int c = 0; c < mScs[s]->m_stream->codec->channels; c++)
{
mChannels[s][c]->Flush();
(*outTracks)[trackindex++] = mChannels[s][c];
}
delete[] mChannels[s];
}
delete[] mChannels;
// Save metadata
WriteMetadata(mFormatContext,tags);
return eImportSuccess;
}
int FFmpegImportFileHandle::Import(TrackFactory *trackFactory,
Track ***outTracks,
int *outNumTracks,
Tags *tags)
{
CreateProgress();
// Remove stream contexts which are not marked for importing and adjust mScs and mNumStreams accordingly
for (int i = 0; i < mNumStreams;)
{
if (!mScs[i]->m_use)
{
delete mScs[i];
for (int j = i; j < mNumStreams - 1; j++)
{
mScs[j] = mScs[j+1];
}
mNumStreams--;
}
else i++;
}
mChannels = new WaveTrack **[mNumStreams];
for (int s = 0; s < mNumStreams; s++)
{
switch (mScs[s]->m_stream->codec->sample_fmt)
{
case SAMPLE_FMT_U8:
case SAMPLE_FMT_S16:
mScs[s]->m_osamplesize = sizeof(int16_t);
mScs[s]->m_osamplefmt = int16Sample;
break;
default:
mScs[s]->m_osamplesize = sizeof(float);
mScs[s]->m_osamplefmt = floatSample;
break;
}
// There is a possibility that number of channels will change over time, but we do not have WaveTracks for new channels. Remember the number of channels and stick to it.
mScs[s]->m_initialchannels = mScs[s]->m_stream->codec->channels;
mChannels[s] = new WaveTrack *[mScs[s]->m_stream->codec->channels];
int c;
for (c = 0; c < mScs[s]->m_stream->codec->channels; c++)
{
mChannels[s][c] = trackFactory->NewWaveTrack(mScs[s]->m_osamplefmt, mScs[s]->m_stream->codec->sample_rate);
if (mScs[s]->m_stream->codec->channels == 2)
{
switch (c)
{
case 0:
mChannels[s][c]->SetChannel(Track::LeftChannel);
mChannels[s][c]->SetLinked(true);
break;
case 1:
mChannels[s][c]->SetChannel(Track::RightChannel);
break;
}
}
else
{
mChannels[s][c]->SetChannel(Track::MonoChannel);
}
}
}
// Handles the start_time by creating silence. This may or may not be correct.
// There is a possibility that we should ignore first N milliseconds of audio instead. I do not know.
/// TODO: Nag FFmpeg devs about start_time until they finally say WHAT is this and HOW to handle it.
for (int s = 0; s < mNumStreams; s++)
{
int64_t stream_delay = 0;
if (mScs[s]->m_stream->start_time != int64_t(AV_NOPTS_VALUE) && mScs[s]->m_stream->start_time > 0)
{
stream_delay = mScs[s]->m_stream->start_time;
wxLogDebug(wxT("Stream %d start_time = %d, that would be %f milliseconds."), s, mScs[s]->m_stream->start_time, double(mScs[s]->m_stream->start_time)/AV_TIME_BASE*1000);
}
if (stream_delay != 0)
{
for (int c = 0; c < mScs[s]->m_stream->codec->channels; c++)
{
WaveTrack *t = mChannels[s][c];
t->InsertSilence(0,double(stream_delay)/AV_TIME_BASE);
}
}
}
// This is the heart of the importing process
// The result of Import() to be returend. It will be something other than zero if user canceled or some error appears.
int res = eProgressSuccess;
#ifdef EXPERIMENTAL_OD_FFMPEG
mUsingOD = false;
gPrefs->Read(wxT("/Library/FFmpegOnDemand"), &mUsingOD);
//at this point we know the file is good and that we have to load the number of channels in mScs[s]->m_stream->codec->channels;
//so for OD loading we create the tracks and releasee the modal lock after starting the ODTask.
if (mUsingOD) {
std::vector<ODDecodeFFmpegTask*> tasks;
//append blockfiles to each stream and add an individual ODDecodeTask for each one.
for (int s = 0; s < mNumStreams; s++) {
ODDecodeFFmpegTask* odTask=new ODDecodeFFmpegTask(mScs,mNumStreams,mChannels,mFormatContext, s);
odTask->CreateFileDecoder(mFilename);
//each stream has different duration. We need to know it if seeking is to be allowed.
sampleCount sampleDuration = 0;
if (mScs[s]->m_stream->duration > 0)
sampleDuration = ((sampleCount)mScs[s]->m_stream->duration * mScs[s]->m_stream->time_base.num) *mScs[s]->m_stream->codec->sample_rate / mScs[s]->m_stream->time_base.den;
else
sampleDuration = ((sampleCount)mFormatContext->duration *mScs[s]->m_stream->codec->sample_rate) / AV_TIME_BASE;
// printf(" OD duration samples %qi, sr %d, secs %d\n",sampleDuration, (int)mScs[s]->m_stream->codec->sample_rate,(int)sampleDuration/mScs[s]->m_stream->codec->sample_rate);
//for each wavetrack within the stream add coded blockfiles
for (int c = 0; c < mScs[s]->m_stream->codec->channels; c++) {
WaveTrack *t = mChannels[s][c];
odTask->AddWaveTrack(t);
sampleCount maxBlockSize = t->GetMaxBlockSize();
//use the maximum blockfile size to divide the sections (about 11secs per blockfile at 44.1khz)
for (sampleCount i = 0; i < sampleDuration; i += maxBlockSize) {
sampleCount blockLen = maxBlockSize;
if (i + blockLen > sampleDuration)
blockLen = sampleDuration - i;
t->AppendCoded(mFilename, i, blockLen, c,ODTask::eODFFMPEG);
// This only works well for single streams since we assume
// each stream is of the same duration and channels
res = mProgress->Update(i+sampleDuration*c+ sampleDuration*mScs[s]->m_stream->codec->channels*s,
sampleDuration*mScs[s]->m_stream->codec->channels*mNumStreams);
if (res != eProgressSuccess)
break;
}
}
tasks.push_back(odTask);
}
//Now we add the tasks and let them run, or delete them if the user cancelled
for(int i=0; i < (int)tasks.size(); i++) {
if(res==eProgressSuccess)
ODManager::Instance()->AddNewTask(tasks[i]);
else
{
delete tasks[i];
}
}
} else {
#endif
streamContext *sc = NULL;
// Read next frame.
while ((sc = ReadNextFrame()) != NULL && (res == eProgressSuccess))
{
// ReadNextFrame returns 1 if stream is not to be imported
if (sc != (streamContext*)1)
{
// Decode frame until it is not possible to decode any further
while (sc->m_pktRemainingSiz > 0 && (res == eProgressSuccess || res == eProgressStopped))
{
if (DecodeFrame(sc,false) < 0)
break;
// If something useable was decoded - write it to mChannels
if (sc->m_frameValid)
res = WriteData(sc);
}
// Cleanup after frame decoding
if (sc->m_pktValid)
{
av_free_packet(&sc->m_pkt);
sc->m_pktValid = 0;
}
}
}
// Flush the decoders.
if ((mNumStreams != 0) && (res == eProgressSuccess || res == eProgressStopped))
{
for (int i = 0; i < mNumStreams; i++)
{
if (DecodeFrame(mScs[i], true) == 0)
{
WriteData(mScs[i]);
if (mScs[i]->m_pktValid)
{
av_free_packet(&mScs[i]->m_pkt);
mScs[i]->m_pktValid = 0;
}
}
}
}
#ifdef EXPERIMENTAL_OD_FFMPEG
} // else -- !mUsingOD == true
#endif //EXPERIMENTAL_OD_FFMPEG
// Something bad happened - destroy everything!
if (res == eProgressCancelled || res == eProgressFailed)
{
for (int s = 0; s < mNumStreams; s++)
{
delete[] mChannels[s];
}
delete[] mChannels;
return res;
}
//else if (res == 2), we just stop the decoding as if the file has ended
*outNumTracks = 0;
for (int s = 0; s < mNumStreams; s++)
{
*outNumTracks += mScs[s]->m_initialchannels;
}
// Create new tracks
*outTracks = new Track *[*outNumTracks];
// Copy audio from mChannels to newly created tracks (destroying mChannels elements in process)
int trackindex = 0;
for (int s = 0; s < mNumStreams; s++)
{
for(int c = 0; c < mScs[s]->m_initialchannels; c++)
{
mChannels[s][c]->Flush();
(*outTracks)[trackindex++] = mChannels[s][c];
}
delete[] mChannels[s];
}
delete[] mChannels;
// Save metadata
WriteMetadata(tags);
return res;
}
