HRESULT WMFVideoMFTManager::CreateD3DVideoFrame(IMFSample* aSample, int64_t aStreamOffset, VideoData** aOutVideoData) { NS_ENSURE_TRUE(aSample, E_POINTER); NS_ENSURE_TRUE(aOutVideoData, E_POINTER); NS_ENSURE_TRUE(mDXVA2Manager, E_ABORT); NS_ENSURE_TRUE(mUseHwAccel, E_ABORT); *aOutVideoData = nullptr; HRESULT hr; nsRefPtr<Image> image; hr = mDXVA2Manager->CopyToImage(aSample, mPictureRegion, mImageContainer, getter_AddRefs(image)); NS_ENSURE_TRUE(SUCCEEDED(hr), hr); NS_ENSURE_TRUE(image, E_FAIL); media::TimeUnit pts = GetSampleTime(aSample); NS_ENSURE_TRUE(pts.IsValid(), E_FAIL); media::TimeUnit duration = GetSampleDuration(aSample); NS_ENSURE_TRUE(duration.IsValid(), E_FAIL); nsRefPtr<VideoData> v = VideoData::CreateFromImage(mVideoInfo, mImageContainer, aStreamOffset, pts.ToMicroseconds(), duration.ToMicroseconds(), image.forget(), false, -1, mPictureRegion); NS_ENSURE_TRUE(v, E_FAIL); v.forget(aOutVideoData); return S_OK; }
HRESULT WMFVideoOutputSource::CreateD3DVideoFrame(IMFSample* aSample, int64_t aStreamOffset, VideoData** aOutVideoData) { NS_ENSURE_TRUE(aSample, E_POINTER); NS_ENSURE_TRUE(aOutVideoData, E_POINTER); NS_ENSURE_TRUE(mDXVA2Manager, E_ABORT); NS_ENSURE_TRUE(mUseHwAccel, E_ABORT); *aOutVideoData = nullptr; HRESULT hr; nsRefPtr<Image> image; hr = mDXVA2Manager->CopyToImage(aSample, mPictureRegion, mImageContainer, getter_AddRefs(image)); NS_ENSURE_TRUE(SUCCEEDED(hr), hr); NS_ENSURE_TRUE(image, E_FAIL); Microseconds pts = GetSampleTime(aSample); Microseconds duration = GetSampleDuration(aSample); VideoData *v = VideoData::CreateFromImage(mVideoInfo, mImageContainer, aStreamOffset, pts, duration, image.forget(), false, -1, ToIntRect(mPictureRegion)); NS_ENSURE_TRUE(v, E_FAIL); *aOutVideoData = v; return S_OK; }
HRESULT WMFVideoMFTManager::CreateBasicVideoFrame(IMFSample* aSample, int64_t aStreamOffset, VideoData** aOutVideoData) { NS_ENSURE_TRUE(aSample, E_POINTER); NS_ENSURE_TRUE(aOutVideoData, E_POINTER); *aOutVideoData = nullptr; HRESULT hr; RefPtr<IMFMediaBuffer> buffer; // Must convert to contiguous buffer to use IMD2DBuffer interface. hr = aSample->ConvertToContiguousBuffer(byRef(buffer)); NS_ENSURE_TRUE(SUCCEEDED(hr), hr); // Try and use the IMF2DBuffer interface if available, otherwise fallback // to the IMFMediaBuffer interface. Apparently IMF2DBuffer is more efficient, // but only some systems (Windows 8?) support it. BYTE* data = nullptr; LONG stride = 0; RefPtr<IMF2DBuffer> twoDBuffer; hr = buffer->QueryInterface(static_cast<IMF2DBuffer**>(byRef(twoDBuffer))); if (SUCCEEDED(hr)) { hr = twoDBuffer->Lock2D(&data, &stride); NS_ENSURE_TRUE(SUCCEEDED(hr), hr); } else { hr = buffer->Lock(&data, nullptr, nullptr); NS_ENSURE_TRUE(SUCCEEDED(hr), hr); stride = mVideoStride; } // YV12, planar format: [YYYY....][VVVV....][UUUU....] // i.e., Y, then V, then U. VideoData::YCbCrBuffer b; // Y (Y') plane b.mPlanes[0].mData = data; b.mPlanes[0].mStride = stride; b.mPlanes[0].mHeight = mVideoHeight; b.mPlanes[0].mWidth = mVideoWidth; b.mPlanes[0].mOffset = 0; b.mPlanes[0].mSkip = 0; // The V and U planes are stored 16-row-aligned, so we need to add padding // to the row heights to ensure the Y'CbCr planes are referenced properly. uint32_t padding = 0; if (mVideoHeight % 16 != 0) { padding = 16 - (mVideoHeight % 16); } uint32_t y_size = stride * (mVideoHeight + padding); uint32_t v_size = stride * (mVideoHeight + padding) / 4; uint32_t halfStride = (stride + 1) / 2; uint32_t halfHeight = (mVideoHeight + 1) / 2; uint32_t halfWidth = (mVideoWidth + 1) / 2; // U plane (Cb) b.mPlanes[1].mData = data + y_size + v_size; b.mPlanes[1].mStride = halfStride; b.mPlanes[1].mHeight = halfHeight; b.mPlanes[1].mWidth = halfWidth; b.mPlanes[1].mOffset = 0; b.mPlanes[1].mSkip = 0; // V plane (Cr) b.mPlanes[2].mData = data + y_size; b.mPlanes[2].mStride = halfStride; b.mPlanes[2].mHeight = halfHeight; b.mPlanes[2].mWidth = halfWidth; b.mPlanes[2].mOffset = 0; b.mPlanes[2].mSkip = 0; Microseconds pts = GetSampleTime(aSample); Microseconds duration = GetSampleDuration(aSample); nsRefPtr<VideoData> v = VideoData::Create(mVideoInfo, mImageContainer, aStreamOffset, std::max(0LL, pts), duration, b, false, -1, ToIntRect(mPictureRegion)); if (twoDBuffer) { twoDBuffer->Unlock2D(); } else { buffer->Unlock(); } v.forget(aOutVideoData); return S_OK; }
// Blocks until decoded sample is produced by the deoder. HRESULT WMFVideoMFTManager::Output(int64_t aStreamOffset, RefPtr<MediaData>& aOutData) { RefPtr<IMFSample> sample; HRESULT hr; aOutData = nullptr; int typeChangeCount = 0; // Loop until we decode a sample, or an unexpected error that we can't // handle occurs. while (true) { hr = mDecoder->Output(&sample); if (hr == MF_E_TRANSFORM_NEED_MORE_INPUT) { return MF_E_TRANSFORM_NEED_MORE_INPUT; } if (hr == MF_E_TRANSFORM_STREAM_CHANGE) { // Video stream output type change. Probably a geometric apperature // change. Reconfigure the video geometry, so that we output the // correct size frames. MOZ_ASSERT(!sample); hr = ConfigureVideoFrameGeometry(); NS_ENSURE_TRUE(SUCCEEDED(hr), hr); // Catch infinite loops, but some decoders perform at least 2 stream // changes on consecutive calls, so be permissive. // 100 is arbitrarily > 2. NS_ENSURE_TRUE(typeChangeCount < 100, MF_E_TRANSFORM_STREAM_CHANGE); // Loop back and try decoding again... ++typeChangeCount; continue; } if (SUCCEEDED(hr)) { if (!sample) { LOG("Video MFTDecoder returned success but no output!"); // On some machines/input the MFT returns success but doesn't output // a video frame. If we detect this, try again, but only up to a // point; after 250 failures, give up. Note we count all failures // over the life of the decoder, as we may end up exiting with a // NEED_MORE_INPUT and coming back to hit the same error. So just // counting with a local variable (like typeChangeCount does) may // not work in this situation. ++mNullOutputCount; if (mNullOutputCount > 250) { LOG("Excessive Video MFTDecoder returning success but no output; giving up"); mGotExcessiveNullOutput = true; return E_FAIL; } continue; } if (mSeekTargetThreshold.isSome()) { media::TimeUnit pts = GetSampleTime(sample); if (!pts.IsValid()) { return E_FAIL; } if (pts < mSeekTargetThreshold.ref()) { LOG("Dropping video frame which pts is smaller than seek target."); // It is necessary to clear the pointer to release the previous output // buffer. sample = nullptr; continue; } mSeekTargetThreshold.reset(); } break; } // Else unexpected error, assert, and bail. NS_WARNING("WMFVideoMFTManager::Output() unexpected error"); return hr; } RefPtr<VideoData> frame; if (mUseHwAccel) { hr = CreateD3DVideoFrame(sample, aStreamOffset, getter_AddRefs(frame)); } else { hr = CreateBasicVideoFrame(sample, aStreamOffset, getter_AddRefs(frame)); } // Frame should be non null only when we succeeded. MOZ_ASSERT((frame != nullptr) == SUCCEEDED(hr)); NS_ENSURE_TRUE(SUCCEEDED(hr), hr); NS_ENSURE_TRUE(frame, E_FAIL); aOutData = frame; if (mNullOutputCount) { mGotValidOutputAfterNullOutput = true; } return S_OK; }
HRESULT WMFVideoMFTManager::CreateBasicVideoFrame(IMFSample* aSample, int64_t aStreamOffset, VideoData** aOutVideoData) { NS_ENSURE_TRUE(aSample, E_POINTER); NS_ENSURE_TRUE(aOutVideoData, E_POINTER); *aOutVideoData = nullptr; HRESULT hr; RefPtr<IMFMediaBuffer> buffer; // Must convert to contiguous buffer to use IMD2DBuffer interface. hr = aSample->ConvertToContiguousBuffer(getter_AddRefs(buffer)); NS_ENSURE_TRUE(SUCCEEDED(hr), hr); // Try and use the IMF2DBuffer interface if available, otherwise fallback // to the IMFMediaBuffer interface. Apparently IMF2DBuffer is more efficient, // but only some systems (Windows 8?) support it. BYTE* data = nullptr; LONG stride = 0; RefPtr<IMF2DBuffer> twoDBuffer; hr = buffer->QueryInterface(static_cast<IMF2DBuffer**>(getter_AddRefs(twoDBuffer))); if (SUCCEEDED(hr)) { hr = twoDBuffer->Lock2D(&data, &stride); NS_ENSURE_TRUE(SUCCEEDED(hr), hr); } else { hr = buffer->Lock(&data, nullptr, nullptr); NS_ENSURE_TRUE(SUCCEEDED(hr), hr); stride = mVideoStride; } // YV12, planar format: [YYYY....][VVVV....][UUUU....] // i.e., Y, then V, then U. VideoData::YCbCrBuffer b; uint32_t videoWidth = mImageSize.width; uint32_t videoHeight = mImageSize.height; // Y (Y') plane b.mPlanes[0].mData = data; b.mPlanes[0].mStride = stride; b.mPlanes[0].mHeight = videoHeight; b.mPlanes[0].mWidth = videoWidth; b.mPlanes[0].mOffset = 0; b.mPlanes[0].mSkip = 0; // The V and U planes are stored 16-row-aligned, so we need to add padding // to the row heights to ensure the Y'CbCr planes are referenced properly. uint32_t padding = 0; if (videoHeight % 16 != 0) { padding = 16 - (videoHeight % 16); } uint32_t y_size = stride * (videoHeight + padding); uint32_t v_size = stride * (videoHeight + padding) / 4; uint32_t halfStride = (stride + 1) / 2; uint32_t halfHeight = (videoHeight + 1) / 2; uint32_t halfWidth = (videoWidth + 1) / 2; // U plane (Cb) b.mPlanes[1].mData = data + y_size + v_size; b.mPlanes[1].mStride = halfStride; b.mPlanes[1].mHeight = halfHeight; b.mPlanes[1].mWidth = halfWidth; b.mPlanes[1].mOffset = 0; b.mPlanes[1].mSkip = 0; // V plane (Cr) b.mPlanes[2].mData = data + y_size; b.mPlanes[2].mStride = halfStride; b.mPlanes[2].mHeight = halfHeight; b.mPlanes[2].mWidth = halfWidth; b.mPlanes[2].mOffset = 0; b.mPlanes[2].mSkip = 0; media::TimeUnit pts = GetSampleTime(aSample); NS_ENSURE_TRUE(pts.IsValid(), E_FAIL); media::TimeUnit duration = GetSampleDuration(aSample); NS_ENSURE_TRUE(duration.IsValid(), E_FAIL); RefPtr<layers::PlanarYCbCrImage> image = new IMFYCbCrImage(buffer, twoDBuffer); nsIntRect pictureRegion = mVideoInfo.ScaledImageRect(videoWidth, videoHeight); VideoData::SetVideoDataToImage(image, mVideoInfo, b, pictureRegion, false); RefPtr<VideoData> v = VideoData::CreateFromImage(mVideoInfo, mImageContainer, aStreamOffset, pts.ToMicroseconds(), duration.ToMicroseconds(), image.forget(), false, -1, pictureRegion); v.forget(aOutVideoData); return S_OK; }