void
DecodedStream::SendVideo(bool aIsSameOrigin, const PrincipalHandle& aPrincipalHandle)
{
AssertOwnerThread();
if (!mInfo.HasVideo()) {
return;
}
VideoSegment output;
TrackID videoTrackId = mInfo.mVideo.mTrackId;
AutoTArray<RefPtr<MediaData>, 10> video;
SourceMediaStream* sourceStream = mData->mStream;
// It's OK to hold references to the VideoData because VideoData
// is ref-counted.
mVideoQueue.GetElementsAfter(mData->mNextVideoTime, &video);
// tracksStartTimeStamp might be null when the SourceMediaStream not yet
// be added to MediaStreamGraph.
TimeStamp tracksStartTimeStamp = sourceStream->GetStreamTracksStrartTimeStamp();
if (tracksStartTimeStamp.IsNull()) {
tracksStartTimeStamp = TimeStamp::Now();
}
for (uint32_t i = 0; i < video.Length(); ++i) {
VideoData* v = video[i]->As<VideoData>();
if (mData->mNextVideoTime < v->mTime) {
// Write last video frame to catch up. mLastVideoImage can be null here
// which is fine, it just means there's no video.
// TODO: |mLastVideoImage| should come from the last image rendered
// by the state machine. This will avoid the black frame when capture
// happens in the middle of playback (especially in th middle of a
// video frame). E.g. if we have a video frame that is 30 sec long
// and capture happens at 15 sec, we'll have to append a black frame
// that is 15 sec long.
WriteVideoToMediaStream(sourceStream, mData->mLastVideoImage, v->mTime,
mData->mNextVideoTime, mData->mLastVideoImageDisplaySize,
tracksStartTimeStamp + TimeDuration::FromMicroseconds(v->mTime),
&output, aPrincipalHandle);
mData->mNextVideoTime = v->mTime;
}
if (mData->mNextVideoTime < v->GetEndTime()) {
WriteVideoToMediaStream(sourceStream, v->mImage, v->GetEndTime(),
mData->mNextVideoTime, v->mDisplay,
tracksStartTimeStamp + TimeDuration::FromMicroseconds(v->GetEndTime()),
&output, aPrincipalHandle);
mData->mNextVideoTime = v->GetEndTime();
mData->mLastVideoImage = v->mImage;
mData->mLastVideoImageDisplaySize = v->mDisplay;
}
}
// Check the output is not empty.
if (output.GetLastFrame()) {
mData->mEOSVideoCompensation = ZeroDurationAtLastChunk(output);
}
if (!aIsSameOrigin) {
output.ReplaceWithDisabled();
}
if (output.GetDuration() > 0) {
sourceStream->AppendToTrack(videoTrackId, &output);
}
if (mVideoQueue.IsFinished() && !mData->mHaveSentFinishVideo) {
if (mData->mEOSVideoCompensation) {
VideoSegment endSegment;
// Calculate the deviation clock time from DecodedStream.
int64_t deviation_usec = sourceStream->StreamTimeToMicroseconds(1);
WriteVideoToMediaStream(sourceStream, mData->mLastVideoImage,
mData->mNextVideoTime + deviation_usec, mData->mNextVideoTime,
mData->mLastVideoImageDisplaySize,
tracksStartTimeStamp + TimeDuration::FromMicroseconds(mData->mNextVideoTime + deviation_usec),
&endSegment, aPrincipalHandle);
mData->mNextVideoTime += deviation_usec;
MOZ_ASSERT(endSegment.GetDuration() > 0);
if (!aIsSameOrigin) {
endSegment.ReplaceWithDisabled();
}
sourceStream->AppendToTrack(videoTrackId, &endSegment);
}
sourceStream->EndTrack(videoTrackId);
mData->mHaveSentFinishVideo = true;
}
}
nsresult
OmxVideoTrackEncoder::GetEncodedTrack(EncodedFrameContainer& aData)
{
VideoSegment segment;
{
// Move all the samples from mRawSegment to segment. We only hold the
// monitor in this block.
ReentrantMonitorAutoEnter mon(mReentrantMonitor);
// Wait if mEncoder is not initialized nor is being canceled.
while (!mCanceled && (!mInitialized ||
(mRawSegment.GetDuration() == 0 && !mEndOfStream))) {
mReentrantMonitor.Wait();
}
if (mCanceled || mEncodingComplete) {
return NS_ERROR_FAILURE;
}
segment.AppendFrom(&mRawSegment);
}
nsresult rv;
// Start queuing raw frames to the input buffers of OMXCodecWrapper.
VideoSegment::ChunkIterator iter(segment);
while (!iter.IsEnded()) {
VideoChunk chunk = *iter;
// Send only the unique video frames to OMXCodecWrapper.
if (mLastFrame != chunk.mFrame) {
uint64_t totalDurationUs = mTotalFrameDuration * USECS_PER_S / mTrackRate;
layers::Image* img = (chunk.IsNull() || chunk.mFrame.GetForceBlack()) ?
nullptr : chunk.mFrame.GetImage();
rv = mEncoder->Encode(img, mFrameWidth, mFrameHeight, totalDurationUs);
NS_ENSURE_SUCCESS(rv, rv);
}
mLastFrame.TakeFrom(&chunk.mFrame);
mTotalFrameDuration += chunk.GetDuration();
iter.Next();
}
// Send the EOS signal to OMXCodecWrapper.
if (mEndOfStream && iter.IsEnded() && !mEosSetInEncoder) {
uint64_t totalDurationUs = mTotalFrameDuration * USECS_PER_S / mTrackRate;
layers::Image* img = (!mLastFrame.GetImage() || mLastFrame.GetForceBlack())
? nullptr : mLastFrame.GetImage();
rv = mEncoder->Encode(img, mFrameWidth, mFrameHeight, totalDurationUs,
OMXCodecWrapper::BUFFER_EOS);
NS_ENSURE_SUCCESS(rv, rv);
// Keep sending EOS signal until OMXVideoEncoder gets it.
mEosSetInEncoder = true;
}
// Dequeue an encoded frame from the output buffers of OMXCodecWrapper.
nsTArray<uint8_t> buffer;
int outFlags = 0;
int64_t outTimeStampUs = 0;
rv = mEncoder->GetNextEncodedFrame(&buffer, &outTimeStampUs, &outFlags,
GET_ENCODED_VIDEO_FRAME_TIMEOUT);
NS_ENSURE_SUCCESS(rv, rv);
if (!buffer.IsEmpty()) {
nsRefPtr<EncodedFrame> videoData = new EncodedFrame();
if (outFlags & OMXCodecWrapper::BUFFER_CODEC_CONFIG) {
videoData->SetFrameType(EncodedFrame::AVC_CSD);
} else {
videoData->SetFrameType((outFlags & OMXCodecWrapper::BUFFER_SYNC_FRAME) ?
EncodedFrame::AVC_I_FRAME : EncodedFrame::AVC_P_FRAME);
}
videoData->SwapInFrameData(buffer);
videoData->SetTimeStamp(outTimeStampUs);
aData.AppendEncodedFrame(videoData);
}
if (outFlags & OMXCodecWrapper::BUFFER_EOS) {
mEncodingComplete = true;
OMX_LOG("Done encoding video.");
}
return NS_OK;
}
nsresult
MediaEngineDefaultVideoSource::Start(SourceMediaStream* aStream, TrackID aID)
{
if (mState != kAllocated) {
return NS_ERROR_FAILURE;
}
mTimer = do_CreateInstance(NS_TIMER_CONTRACTID);
if (!mTimer) {
return NS_ERROR_FAILURE;
}
mSource = aStream;
// Allocate a single blank Image
layers::Image::Format format = layers::Image::PLANAR_YCBCR;
mImageContainer = layers::LayerManager::CreateImageContainer();
nsRefPtr<layers::Image> image = mImageContainer->CreateImage(&format, 1);
int len = ((WIDTH * HEIGHT) * 3 / 2);
mImage = static_cast<layers::PlanarYCbCrImage*>(image.get());
PRUint8* frame = (PRUint8*) PR_Malloc(len);
memset(frame, 0x80, len); // Gray
const PRUint8 lumaBpp = 8;
const PRUint8 chromaBpp = 4;
layers::PlanarYCbCrImage::Data data;
data.mYChannel = frame;
data.mYSize = gfxIntSize(WIDTH, HEIGHT);
data.mYStride = WIDTH * lumaBpp / 8.0;
data.mCbCrStride = WIDTH * chromaBpp / 8.0;
data.mCbChannel = frame + HEIGHT * data.mYStride;
data.mCrChannel = data.mCbChannel + HEIGHT * data.mCbCrStride / 2;
data.mCbCrSize = gfxIntSize(WIDTH / 2, HEIGHT / 2);
data.mPicX = 0;
data.mPicY = 0;
data.mPicSize = gfxIntSize(WIDTH, HEIGHT);
data.mStereoMode = layers::STEREO_MODE_MONO;
// SetData copies data, so we can free the frame
mImage->SetData(data);
PR_Free(frame);
// AddTrack takes ownership of segment
VideoSegment *segment = new VideoSegment();
segment->AppendFrame(image.forget(), USECS_PER_S / FPS, gfxIntSize(WIDTH, HEIGHT));
mSource->AddTrack(aID, RATE, 0, segment);
// We aren't going to add any more tracks
mSource->AdvanceKnownTracksTime(STREAM_TIME_MAX);
// Remember TrackID so we can end it later
mTrackID = aID;
// Start timer for subsequent frames
mTimer->InitWithCallback(this, 1000 / FPS, nsITimer::TYPE_REPEATING_SLACK);
mState = kStarted;
return NS_OK;
}
void VideoFrameContainer::SetCurrentFrames(const VideoSegment& aSegment)
{
if (aSegment.IsEmpty()) {
return;
}
MutexAutoLock lock(mMutex);
// Collect any new frames produced in this iteration.
AutoTArray<ImageContainer::NonOwningImage,4> newImages;
PrincipalHandle lastPrincipalHandle = PRINCIPAL_HANDLE_NONE;
VideoSegment::ConstChunkIterator iter(aSegment);
while (!iter.IsEnded()) {
VideoChunk chunk = *iter;
const VideoFrame* frame = &chunk.mFrame;
if (*frame == mLastPlayedVideoFrame) {
iter.Next();
continue;
}
Image* image = frame->GetImage();
CONTAINER_LOG(LogLevel::Verbose,
("VideoFrameContainer %p writing video frame %p (%d x %d)",
this, image, frame->GetIntrinsicSize().width,
frame->GetIntrinsicSize().height));
if (frame->GetForceBlack()) {
if (!mBlackImage) {
mBlackImage = GetImageContainer()->CreatePlanarYCbCrImage();
if (mBlackImage) {
// Sets the image to a single black pixel, which will be scaled to
// fill the rendered size.
SetImageToBlackPixel(mBlackImage->AsPlanarYCbCrImage());
}
}
if (mBlackImage) {
image = mBlackImage;
}
}
// Don't append null image to the newImages.
if (!image) {
iter.Next();
continue;
}
newImages.AppendElement(ImageContainer::NonOwningImage(image, chunk.mTimeStamp));
lastPrincipalHandle = chunk.GetPrincipalHandle();
mLastPlayedVideoFrame = *frame;
iter.Next();
}
// Don't update if there are no changes.
if (newImages.IsEmpty()) {
return;
}
AutoTArray<ImageContainer::NonOwningImage,4> images;
bool principalHandleChanged =
lastPrincipalHandle != PRINCIPAL_HANDLE_NONE &&
lastPrincipalHandle != GetLastPrincipalHandleLocked();
// Add the frames from this iteration.
for (auto& image : newImages) {
image.mFrameID = NewFrameID();
images.AppendElement(image);
}
if (principalHandleChanged) {
UpdatePrincipalHandleForFrameIDLocked(lastPrincipalHandle,
newImages.LastElement().mFrameID);
}
SetCurrentFramesLocked(mLastPlayedVideoFrame.GetIntrinsicSize(), images);
nsCOMPtr<nsIRunnable> event =
new VideoFrameContainerInvalidateRunnable(this);
mMainThread->Dispatch(event.forget());
images.ClearAndRetainStorage();
}