/* static */ already_AddRefed<VideoData> VideoData::Create(const VideoInfo& aInfo, ImageContainer* aContainer, Image* aImage, int64_t aOffset, int64_t aTime, int64_t aDuration, const YCbCrBuffer& aBuffer, bool aKeyframe, int64_t aTimecode, const IntRect& aPicture) { if (!aImage && !aContainer) { // Create a dummy VideoData with no image. This gives us something to // send to media streams if necessary. RefPtr<VideoData> v(new VideoData(aOffset, aTime, aDuration, aKeyframe, aTimecode, aInfo.mDisplay, 0)); return v.forget(); } // The following situation should never happen unless there is a bug // in the decoder if (aBuffer.mPlanes[1].mWidth != aBuffer.mPlanes[2].mWidth || aBuffer.mPlanes[1].mHeight != aBuffer.mPlanes[2].mHeight) { NS_ERROR("C planes with different sizes"); return nullptr; } // The following situations could be triggered by invalid input if (aPicture.width <= 0 || aPicture.height <= 0) { // In debug mode, makes the error more noticeable MOZ_ASSERT(false, "Empty picture rect"); return nullptr; } if (!ValidatePlane(aBuffer.mPlanes[0]) || !ValidatePlane(aBuffer.mPlanes[1]) || !ValidatePlane(aBuffer.mPlanes[2])) { NS_WARNING("Invalid plane size"); return nullptr; } // Ensure the picture size specified in the headers can be extracted out of // the frame we've been supplied without indexing out of bounds. CheckedUint32 xLimit = aPicture.x + CheckedUint32(aPicture.width); CheckedUint32 yLimit = aPicture.y + CheckedUint32(aPicture.height); if (!xLimit.isValid() || xLimit.value() > aBuffer.mPlanes[0].mStride || !yLimit.isValid() || yLimit.value() > aBuffer.mPlanes[0].mHeight) { // The specified picture dimensions can't be contained inside the video // frame, we'll stomp memory if we try to copy it. Fail. NS_WARNING("Overflowing picture rect"); return nullptr; } RefPtr<VideoData> v(new VideoData(aOffset, aTime, aDuration, aKeyframe, aTimecode, aInfo.mDisplay, 0)); #ifdef MOZ_WIDGET_GONK const YCbCrBuffer::Plane &Y = aBuffer.mPlanes[0]; const YCbCrBuffer::Plane &Cb = aBuffer.mPlanes[1]; const YCbCrBuffer::Plane &Cr = aBuffer.mPlanes[2]; #endif if (!aImage) { // Currently our decoder only knows how to output to ImageFormat::PLANAR_YCBCR // format. #ifdef MOZ_WIDGET_GONK if (IsYV12Format(Y, Cb, Cr) && !IsInEmulator()) { v->mImage = aContainer->CreateImage(ImageFormat::GRALLOC_PLANAR_YCBCR); } #endif if (!v->mImage) { v->mImage = aContainer->CreateImage(ImageFormat::PLANAR_YCBCR); } } else { v->mImage = aImage; } if (!v->mImage) { return nullptr; } NS_ASSERTION(v->mImage->GetFormat() == ImageFormat::PLANAR_YCBCR || v->mImage->GetFormat() == ImageFormat::GRALLOC_PLANAR_YCBCR, "Wrong format?"); PlanarYCbCrImage* videoImage = static_cast<PlanarYCbCrImage*>(v->mImage.get()); bool shouldCopyData = (aImage == nullptr); if (!VideoData::SetVideoDataToImage(videoImage, aInfo, aBuffer, aPicture, shouldCopyData)) { return nullptr; } #ifdef MOZ_WIDGET_GONK if (!videoImage->IsValid() && !aImage && IsYV12Format(Y, Cb, Cr)) { // Failed to allocate gralloc. Try fallback. v->mImage = aContainer->CreateImage(ImageFormat::PLANAR_YCBCR); if (!v->mImage) { return nullptr; } videoImage = static_cast<PlanarYCbCrImage*>(v->mImage.get()); if(!VideoData::SetVideoDataToImage(videoImage, aInfo, aBuffer, aPicture, true /* aCopyData */)) { return nullptr; } } #endif return v.forget(); }
void ImageLayerD3D9::RenderLayer() { ImageContainer *container = GetContainer(); if (!container || mD3DManager->CompositingDisabled()) { return; } AutoLockImage autoLock(container); Image *image = autoLock.GetImage(); if (!image) { return; } SetShaderTransformAndOpacity(); gfx::IntSize size = image->GetSize(); if (image->GetFormat() == ImageFormat::CAIRO_SURFACE || image->GetFormat() == ImageFormat::REMOTE_IMAGE_BITMAP || image->GetFormat() == ImageFormat::D3D9_RGB32_TEXTURE) { NS_ASSERTION(image->GetFormat() != ImageFormat::CAIRO_SURFACE || !static_cast<CairoImage*>(image)->mSourceSurface || static_cast<CairoImage*>(image)->mSourceSurface->GetFormat() != SurfaceFormat::A8, "Image layer has alpha image"); bool hasAlpha = false; nsRefPtr<IDirect3DTexture9> texture = GetTexture(image, hasAlpha); device()->SetVertexShaderConstantF(CBvLayerQuad, ShaderConstantRect(0, 0, size.width, size.height), 1); if (hasAlpha) { mD3DManager->SetShaderMode(DeviceManagerD3D9::RGBALAYER, GetMaskLayer()); } else { mD3DManager->SetShaderMode(DeviceManagerD3D9::RGBLAYER, GetMaskLayer()); } if (mFilter == GraphicsFilter::FILTER_NEAREST) { device()->SetSamplerState(0, D3DSAMP_MAGFILTER, D3DTEXF_POINT); device()->SetSamplerState(0, D3DSAMP_MINFILTER, D3DTEXF_POINT); } device()->SetTexture(0, texture); image = nullptr; autoLock.Unlock(); device()->DrawPrimitive(D3DPT_TRIANGLESTRIP, 0, 2); if (mFilter == GraphicsFilter::FILTER_NEAREST) { device()->SetSamplerState(0, D3DSAMP_MAGFILTER, D3DTEXF_LINEAR); device()->SetSamplerState(0, D3DSAMP_MINFILTER, D3DTEXF_LINEAR); } } else { PlanarYCbCrImage *yuvImage = static_cast<PlanarYCbCrImage*>(image); if (!yuvImage->IsValid()) { return; } if (!yuvImage->GetBackendData(mozilla::layers::LayersBackend::LAYERS_D3D9)) { AllocateTexturesYCbCr(yuvImage, device(), mD3DManager); } PlanarYCbCrD3D9BackendData *data = static_cast<PlanarYCbCrD3D9BackendData*>(yuvImage->GetBackendData(mozilla::layers::LayersBackend::LAYERS_D3D9)); if (!data) { return; } nsRefPtr<IDirect3DDevice9> dev; data->mYTexture->GetDevice(getter_AddRefs(dev)); if (dev != device()) { return; } device()->SetVertexShaderConstantF(CBvLayerQuad, ShaderConstantRect(0, 0, size.width, size.height), 1); device()->SetVertexShaderConstantF(CBvTextureCoords, ShaderConstantRect( (float)yuvImage->GetData()->mPicX / yuvImage->GetData()->mYSize.width, (float)yuvImage->GetData()->mPicY / yuvImage->GetData()->mYSize.height, (float)yuvImage->GetData()->mPicSize.width / yuvImage->GetData()->mYSize.width, (float)yuvImage->GetData()->mPicSize.height / yuvImage->GetData()->mYSize.height ), 1); mD3DManager->SetShaderMode(DeviceManagerD3D9::YCBCRLAYER, GetMaskLayer()); /* * Send 3d control data and metadata */ if (mD3DManager->GetNv3DVUtils()) { Nv_Stereo_Mode mode; switch (yuvImage->GetData()->mStereoMode) { case StereoMode::LEFT_RIGHT: mode = NV_STEREO_MODE_LEFT_RIGHT; break; case StereoMode::RIGHT_LEFT: mode = NV_STEREO_MODE_RIGHT_LEFT; break; case StereoMode::BOTTOM_TOP: mode = NV_STEREO_MODE_BOTTOM_TOP; break; case StereoMode::TOP_BOTTOM: mode = NV_STEREO_MODE_TOP_BOTTOM; break; case StereoMode::MONO: mode = NV_STEREO_MODE_MONO; break; } // Send control data even in mono case so driver knows to leave stereo mode. mD3DManager->GetNv3DVUtils()->SendNv3DVControl(mode, true, FIREFOX_3DV_APP_HANDLE); if (yuvImage->GetData()->mStereoMode != StereoMode::MONO) { mD3DManager->GetNv3DVUtils()->SendNv3DVControl(mode, true, FIREFOX_3DV_APP_HANDLE); nsRefPtr<IDirect3DSurface9> renderTarget; device()->GetRenderTarget(0, getter_AddRefs(renderTarget)); mD3DManager->GetNv3DVUtils()->SendNv3DVMetaData((unsigned int)yuvImage->GetSize().width, (unsigned int)yuvImage->GetSize().height, (HANDLE)(data->mYTexture), (HANDLE)(renderTarget)); } } // Linear scaling is default here, adhering to mFilter is difficult since // presumably even with point filtering we'll still want chroma upsampling // to be linear. In the current approach we can't. device()->SetTexture(0, data->mYTexture); device()->SetTexture(1, data->mCbTexture); device()->SetTexture(2, data->mCrTexture); image = nullptr; data = nullptr; autoLock.Unlock(); device()->DrawPrimitive(D3DPT_TRIANGLESTRIP, 0, 2); device()->SetVertexShaderConstantF(CBvTextureCoords, ShaderConstantRect(0, 0, 1.0f, 1.0f), 1); } GetContainer()->NotifyPaintedImage(image); }
void ImageLayerOGL::RenderLayer(int, const nsIntPoint& aOffset) { nsRefPtr<ImageContainer> container = GetContainer(); if (!container || mOGLManager->CompositingDisabled()) return; mOGLManager->MakeCurrent(); AutoLockImage autoLock(container); Image *image = autoLock.GetImage(); if (!image) { return; } NS_ASSERTION(image->GetFormat() != REMOTE_IMAGE_BITMAP, "Remote images aren't handled yet in OGL layers!"); if (image->GetFormat() == PLANAR_YCBCR) { PlanarYCbCrImage *yuvImage = static_cast<PlanarYCbCrImage*>(image); if (!yuvImage->IsValid()) { return; } PlanarYCbCrOGLBackendData *data = static_cast<PlanarYCbCrOGLBackendData*>(yuvImage->GetBackendData(LAYERS_OPENGL)); if (data && data->mTextures->GetGLContext() != gl()) { // If these textures were allocated by another layer manager, // clear them out and re-allocate below. data = nullptr; yuvImage->SetBackendData(LAYERS_OPENGL, nullptr); } if (!data) { AllocateTexturesYCbCr(yuvImage); data = static_cast<PlanarYCbCrOGLBackendData*>(yuvImage->GetBackendData(LAYERS_OPENGL)); } if (!data || data->mTextures->GetGLContext() != gl()) { // XXX - Can this ever happen? If so I need to fix this! return; } gl()->MakeCurrent(); gl()->fActiveTexture(LOCAL_GL_TEXTURE2); gl()->fBindTexture(LOCAL_GL_TEXTURE_2D, data->mTextures[2].GetTextureID()); gl()->ApplyFilterToBoundTexture(mFilter); gl()->fActiveTexture(LOCAL_GL_TEXTURE1); gl()->fBindTexture(LOCAL_GL_TEXTURE_2D, data->mTextures[1].GetTextureID()); gl()->ApplyFilterToBoundTexture(mFilter); gl()->fActiveTexture(LOCAL_GL_TEXTURE0); gl()->fBindTexture(LOCAL_GL_TEXTURE_2D, data->mTextures[0].GetTextureID()); gl()->ApplyFilterToBoundTexture(mFilter); ShaderProgramOGL *program = mOGLManager->GetProgram(YCbCrLayerProgramType, GetMaskLayer()); program->Activate(); program->SetLayerQuadRect(nsIntRect(0, 0, yuvImage->GetSize().width, yuvImage->GetSize().height)); program->SetLayerTransform(GetEffectiveTransform()); program->SetLayerOpacity(GetEffectiveOpacity()); program->SetRenderOffset(aOffset); program->SetYCbCrTextureUnits(0, 1, 2); program->LoadMask(GetMaskLayer()); mOGLManager->BindAndDrawQuadWithTextureRect(program, yuvImage->GetData()->GetPictureRect(), nsIntSize(yuvImage->GetData()->mYSize.width, yuvImage->GetData()->mYSize.height)); // We shouldn't need to do this, but do it anyway just in case // someone else forgets. gl()->fActiveTexture(LOCAL_GL_TEXTURE0); } else if (image->GetFormat() == CAIRO_SURFACE) { CairoImage *cairoImage = static_cast<CairoImage*>(image); if (!cairoImage->mSurface) { return; } NS_ASSERTION(cairoImage->mSurface->GetContentType() != gfxASurface::CONTENT_ALPHA, "Image layer has alpha image"); CairoOGLBackendData *data = static_cast<CairoOGLBackendData*>(cairoImage->GetBackendData(LAYERS_OPENGL)); if (data && data->mTexture.GetGLContext() != gl()) { // If this texture was allocated by another layer manager, clear // it out and re-allocate below. data = nullptr; cairoImage->SetBackendData(LAYERS_OPENGL, nullptr); } if (!data) { AllocateTexturesCairo(cairoImage); data = static_cast<CairoOGLBackendData*>(cairoImage->GetBackendData(LAYERS_OPENGL)); } if (!data || data->mTexture.GetGLContext() != gl()) { // XXX - Can this ever happen? If so I need to fix this! return; } gl()->MakeCurrent(); gl()->fActiveTexture(LOCAL_GL_TEXTURE0); gl()->fBindTexture(LOCAL_GL_TEXTURE_2D, data->mTexture.GetTextureID()); ShaderProgramOGL *program = mOGLManager->GetProgram(data->mLayerProgram, GetMaskLayer()); gl()->ApplyFilterToBoundTexture(mFilter); program->Activate(); program->SetLayerQuadRect(nsIntRect(0, 0, cairoImage->GetSize().width, cairoImage->GetSize().height)); program->SetLayerTransform(GetEffectiveTransform()); program->SetLayerOpacity(GetEffectiveOpacity()); program->SetRenderOffset(aOffset); program->SetTextureUnit(0); program->LoadMask(GetMaskLayer()); mOGLManager->BindAndDrawQuad(program); #ifdef XP_MACOSX } else if (image->GetFormat() == MAC_IO_SURFACE) { MacIOSurfaceImage *ioImage = static_cast<MacIOSurfaceImage*>(image); if (!mOGLManager->GetThebesLayerCallback()) { // If its an empty transaction we still need to update // the plugin IO Surface and make sure we grab the // new image ioImage->Update(GetContainer()); image = nullptr; autoLock.Refresh(); image = autoLock.GetImage(); gl()->MakeCurrent(); ioImage = static_cast<MacIOSurfaceImage*>(image); } if (!ioImage) { return; } gl()->fActiveTexture(LOCAL_GL_TEXTURE0); if (!ioImage->GetBackendData(LAYERS_OPENGL)) { AllocateTextureIOSurface(ioImage, gl()); } MacIOSurfaceImageOGLBackendData *data = static_cast<MacIOSurfaceImageOGLBackendData*>(ioImage->GetBackendData(LAYERS_OPENGL)); gl()->fActiveTexture(LOCAL_GL_TEXTURE0); gl()->fBindTexture(LOCAL_GL_TEXTURE_RECTANGLE_ARB, data->mTexture.GetTextureID()); ShaderProgramOGL *program = mOGLManager->GetProgram(gl::RGBARectLayerProgramType, GetMaskLayer()); program->Activate(); if (program->GetTexCoordMultiplierUniformLocation() != -1) { // 2DRect case, get the multiplier right for a sampler2DRect program->SetTexCoordMultiplier(ioImage->GetSize().width, ioImage->GetSize().height); } else { NS_ASSERTION(0, "no rects?"); } program->SetLayerQuadRect(nsIntRect(0, 0, ioImage->GetSize().width, ioImage->GetSize().height)); program->SetLayerTransform(GetEffectiveTransform()); program->SetLayerOpacity(GetEffectiveOpacity()); program->SetRenderOffset(aOffset); program->SetTextureUnit(0); program->LoadMask(GetMaskLayer()); mOGLManager->BindAndDrawQuad(program); gl()->fBindTexture(LOCAL_GL_TEXTURE_RECTANGLE_ARB, 0); #endif #ifdef MOZ_WIDGET_GONK } else if (image->GetFormat() == GONK_IO_SURFACE) { GonkIOSurfaceImage *ioImage = static_cast<GonkIOSurfaceImage*>(image); if (!ioImage) { return; } gl()->MakeCurrent(); gl()->fActiveTexture(LOCAL_GL_TEXTURE0); if (!ioImage->GetBackendData(LAYERS_OPENGL)) { AllocateTextureIOSurface(ioImage, gl()); } GonkIOSurfaceImageOGLBackendData *data = static_cast<GonkIOSurfaceImageOGLBackendData*>(ioImage->GetBackendData(LAYERS_OPENGL)); gl()->fActiveTexture(LOCAL_GL_TEXTURE0); gl()->BindExternalBuffer(data->mTexture.GetTextureID(), ioImage->GetNativeBuffer()); ShaderProgramOGL *program = mOGLManager->GetProgram(RGBAExternalLayerProgramType, GetMaskLayer()); gl()->ApplyFilterToBoundTexture(mFilter); program->Activate(); program->SetLayerQuadRect(nsIntRect(0, 0, ioImage->GetSize().width, ioImage->GetSize().height)); program->SetLayerTransform(GetEffectiveTransform()); program->SetLayerOpacity(GetEffectiveOpacity()); program->SetRenderOffset(aOffset); program->SetTextureUnit(0); program->LoadMask(GetMaskLayer()); mOGLManager->BindAndDrawQuadWithTextureRect(program, GetVisibleRegion().GetBounds(), nsIntSize(ioImage->GetSize().width, ioImage->GetSize().height)); #endif } GetContainer()->NotifyPaintedImage(image); }
nsresult VP8TrackEncoder::PrepareRawFrame(VideoChunk &aChunk) { nsRefPtr<Image> img; if (aChunk.mFrame.GetForceBlack() || aChunk.IsNull()) { if (!mMuteFrame) { mMuteFrame = VideoFrame::CreateBlackImage(gfxIntSize(mFrameWidth, mFrameHeight)); MOZ_ASSERT(mMuteFrame); } img = mMuteFrame; } else { img = aChunk.mFrame.GetImage(); } ImageFormat format = img->GetFormat(); if (format != ImageFormat::PLANAR_YCBCR) { VP8LOG("Unsupported video format\n"); return NS_ERROR_FAILURE; } // Cast away constness b/c some of the accessors are non-const PlanarYCbCrImage* yuv = const_cast<PlanarYCbCrImage *>(static_cast<const PlanarYCbCrImage *>(img.get())); // Big-time assumption here that this is all contiguous data coming // from getUserMedia or other sources. MOZ_ASSERT(yuv); if (!yuv->IsValid()) { NS_WARNING("PlanarYCbCrImage is not valid"); return NS_ERROR_FAILURE; } const PlanarYCbCrImage::Data *data = yuv->GetData(); if (isYUV420(data) && !data->mCbSkip) { // 420 planar mVPXImageWrapper->planes[VPX_PLANE_Y] = data->mYChannel; mVPXImageWrapper->planes[VPX_PLANE_U] = data->mCbChannel; mVPXImageWrapper->planes[VPX_PLANE_V] = data->mCrChannel; mVPXImageWrapper->stride[VPX_PLANE_Y] = data->mYStride; mVPXImageWrapper->stride[VPX_PLANE_U] = data->mCbCrStride; mVPXImageWrapper->stride[VPX_PLANE_V] = data->mCbCrStride; } else { uint32_t yPlaneSize = mFrameWidth * mFrameHeight; uint32_t halfWidth = (mFrameWidth + 1) / 2; uint32_t halfHeight = (mFrameHeight + 1) / 2; uint32_t uvPlaneSize = halfWidth * halfHeight; if (mI420Frame.IsEmpty()) { mI420Frame.SetLength(yPlaneSize + uvPlaneSize * 2); } MOZ_ASSERT(mI420Frame.Length() >= (yPlaneSize + uvPlaneSize * 2)); uint8_t *y = mI420Frame.Elements(); uint8_t *cb = mI420Frame.Elements() + yPlaneSize; uint8_t *cr = mI420Frame.Elements() + yPlaneSize + uvPlaneSize; if (isYUV420(data) && data->mCbSkip) { // If mCbSkip is set, we assume it's nv12 or nv21. if (data->mCbChannel < data->mCrChannel) { // nv12 libyuv::NV12ToI420(data->mYChannel, data->mYStride, data->mCbChannel, data->mCbCrStride, y, mFrameWidth, cb, halfWidth, cr, halfWidth, mFrameWidth, mFrameHeight); } else { // nv21 libyuv::NV21ToI420(data->mYChannel, data->mYStride, data->mCrChannel, data->mCbCrStride, y, mFrameWidth, cb, halfWidth, cr, halfWidth, mFrameWidth, mFrameHeight); } } else if (isYUV444(data) && !data->mCbSkip) { libyuv::I444ToI420(data->mYChannel, data->mYStride, data->mCbChannel, data->mCbCrStride, data->mCrChannel, data->mCbCrStride, y, mFrameWidth, cb, halfWidth, cr, halfWidth, mFrameWidth, mFrameHeight); } else if (isYUV422(data) && !data->mCbSkip) { libyuv::I422ToI420(data->mYChannel, data->mYStride, data->mCbChannel, data->mCbCrStride, data->mCrChannel, data->mCbCrStride, y, mFrameWidth, cb, halfWidth, cr, halfWidth, mFrameWidth, mFrameHeight); } else { VP8LOG("Unsupported planar format\n"); return NS_ERROR_NOT_IMPLEMENTED; } mVPXImageWrapper->planes[VPX_PLANE_Y] = y; mVPXImageWrapper->planes[VPX_PLANE_U] = cb; mVPXImageWrapper->planes[VPX_PLANE_V] = cr; mVPXImageWrapper->stride[VPX_PLANE_Y] = mFrameWidth; mVPXImageWrapper->stride[VPX_PLANE_U] = halfWidth; mVPXImageWrapper->stride[VPX_PLANE_V] = halfWidth; } return NS_OK; }
void ImageLayerD3D10::RenderLayer() { ImageContainer *container = GetContainer(); if (!container) { return; } AutoLockImage autoLock(container); Image *image = autoLock.GetImage(); if (!image) { return; } gfxIntSize size = image->GetSize(); SetEffectTransformAndOpacity(); ID3D10EffectTechnique *technique; nsRefPtr<IDXGIKeyedMutex> keyedMutex; if (image->GetFormat() == ImageFormat::CAIRO_SURFACE || image->GetFormat() == ImageFormat::REMOTE_IMAGE_BITMAP || image->GetFormat() == ImageFormat::REMOTE_IMAGE_DXGI_TEXTURE || image->GetFormat() == ImageFormat::D3D9_RGB32_TEXTURE) { NS_ASSERTION(image->GetFormat() != ImageFormat::CAIRO_SURFACE || !static_cast<CairoImage*>(image)->mSurface || static_cast<CairoImage*>(image)->mSurface->GetContentType() != gfxASurface::CONTENT_ALPHA, "Image layer has alpha image"); bool hasAlpha = false; nsRefPtr<ID3D10ShaderResourceView> srView = GetImageSRView(image, hasAlpha, getter_AddRefs(keyedMutex)); if (!srView) { return; } uint8_t shaderFlags = SHADER_PREMUL; shaderFlags |= LoadMaskTexture(); shaderFlags |= hasAlpha ? SHADER_RGBA : SHADER_RGB; shaderFlags |= mFilter == gfxPattern::FILTER_NEAREST ? SHADER_POINT : SHADER_LINEAR; technique = SelectShader(shaderFlags); effect()->GetVariableByName("tRGB")->AsShaderResource()->SetResource(srView); effect()->GetVariableByName("vLayerQuad")->AsVector()->SetFloatVector( ShaderConstantRectD3D10( (float)0, (float)0, (float)size.width, (float)size.height) ); } else if (image->GetFormat() == ImageFormat::PLANAR_YCBCR) { PlanarYCbCrImage *yuvImage = static_cast<PlanarYCbCrImage*>(image); if (!yuvImage->IsValid()) { return; } if (!yuvImage->GetBackendData(mozilla::layers::LAYERS_D3D10)) { AllocateTexturesYCbCr(yuvImage); } PlanarYCbCrD3D10BackendData *data = static_cast<PlanarYCbCrD3D10BackendData*>(yuvImage->GetBackendData(mozilla::layers::LAYERS_D3D10)); if (!data) { return; } nsRefPtr<ID3D10Device> dev; data->mYTexture->GetDevice(getter_AddRefs(dev)); if (dev != device()) { return; } // TODO: At some point we should try to deal with mFilter here, you don't // really want to use point filtering in the case of NEAREST, since that // would also use point filtering for Chroma upsampling. Where most likely // the user would only want point filtering for final RGB image upsampling. technique = SelectShader(SHADER_YCBCR | LoadMaskTexture()); effect()->GetVariableByName("tY")->AsShaderResource()->SetResource(data->mYView); effect()->GetVariableByName("tCb")->AsShaderResource()->SetResource(data->mCbView); effect()->GetVariableByName("tCr")->AsShaderResource()->SetResource(data->mCrView); effect()->GetVariableByName("vLayerQuad")->AsVector()->SetFloatVector( ShaderConstantRectD3D10( (float)0, (float)0, (float)size.width, (float)size.height) ); effect()->GetVariableByName("vTextureCoords")->AsVector()->SetFloatVector( ShaderConstantRectD3D10( (float)yuvImage->GetData()->mPicX / yuvImage->GetData()->mYSize.width, (float)yuvImage->GetData()->mPicY / yuvImage->GetData()->mYSize.height, (float)yuvImage->GetData()->mPicSize.width / yuvImage->GetData()->mYSize.width, (float)yuvImage->GetData()->mPicSize.height / yuvImage->GetData()->mYSize.height) ); } bool resetTexCoords = image->GetFormat() == ImageFormat::PLANAR_YCBCR; image = nullptr; autoLock.Unlock(); technique->GetPassByIndex(0)->Apply(0); device()->Draw(4, 0); if (keyedMutex) { keyedMutex->ReleaseSync(0); } if (resetTexCoords) { effect()->GetVariableByName("vTextureCoords")->AsVector()-> SetFloatVector(ShaderConstantRectD3D10(0, 0, 1.0f, 1.0f)); } GetContainer()->NotifyPaintedImage(image); }
nsresult VP8TrackEncoder::PrepareRawFrame(VideoChunk &aChunk) { RefPtr<Image> img; if (aChunk.mFrame.GetForceBlack() || aChunk.IsNull()) { if (!mMuteFrame) { mMuteFrame = VideoFrame::CreateBlackImage(gfx::IntSize(mFrameWidth, mFrameHeight)); MOZ_ASSERT(mMuteFrame); } img = mMuteFrame; } else { img = aChunk.mFrame.GetImage(); } if (img->GetSize() != IntSize(mFrameWidth, mFrameHeight)) { VP8LOG("Dynamic resolution changes (was %dx%d, now %dx%d) are unsupported\n", mFrameWidth, mFrameHeight, img->GetSize().width, img->GetSize().height); return NS_ERROR_FAILURE; } ImageFormat format = img->GetFormat(); if (format == ImageFormat::PLANAR_YCBCR) { PlanarYCbCrImage* yuv = static_cast<PlanarYCbCrImage *>(img.get()); MOZ_RELEASE_ASSERT(yuv); if (!yuv->IsValid()) { NS_WARNING("PlanarYCbCrImage is not valid"); return NS_ERROR_FAILURE; } const PlanarYCbCrImage::Data *data = yuv->GetData(); if (isYUV420(data) && !data->mCbSkip) { // 420 planar, no need for conversions mVPXImageWrapper->planes[VPX_PLANE_Y] = data->mYChannel; mVPXImageWrapper->planes[VPX_PLANE_U] = data->mCbChannel; mVPXImageWrapper->planes[VPX_PLANE_V] = data->mCrChannel; mVPXImageWrapper->stride[VPX_PLANE_Y] = data->mYStride; mVPXImageWrapper->stride[VPX_PLANE_U] = data->mCbCrStride; mVPXImageWrapper->stride[VPX_PLANE_V] = data->mCbCrStride; return NS_OK; } } // Not 420 planar, have to convert uint32_t yPlaneSize = mFrameWidth * mFrameHeight; uint32_t halfWidth = (mFrameWidth + 1) / 2; uint32_t halfHeight = (mFrameHeight + 1) / 2; uint32_t uvPlaneSize = halfWidth * halfHeight; if (mI420Frame.IsEmpty()) { mI420Frame.SetLength(yPlaneSize + uvPlaneSize * 2); } uint8_t *y = mI420Frame.Elements(); uint8_t *cb = mI420Frame.Elements() + yPlaneSize; uint8_t *cr = mI420Frame.Elements() + yPlaneSize + uvPlaneSize; if (format == ImageFormat::PLANAR_YCBCR) { PlanarYCbCrImage* yuv = static_cast<PlanarYCbCrImage *>(img.get()); MOZ_RELEASE_ASSERT(yuv); if (!yuv->IsValid()) { NS_WARNING("PlanarYCbCrImage is not valid"); return NS_ERROR_FAILURE; } const PlanarYCbCrImage::Data *data = yuv->GetData(); int rv; std::string yuvFormat; if (isYUV420(data) && data->mCbSkip) { // If mCbSkip is set, we assume it's nv12 or nv21. if (data->mCbChannel < data->mCrChannel) { // nv12 rv = libyuv::NV12ToI420(data->mYChannel, data->mYStride, data->mCbChannel, data->mCbCrStride, y, mFrameWidth, cb, halfWidth, cr, halfWidth, mFrameWidth, mFrameHeight); yuvFormat = "NV12"; } else { // nv21 rv = libyuv::NV21ToI420(data->mYChannel, data->mYStride, data->mCrChannel, data->mCbCrStride, y, mFrameWidth, cb, halfWidth, cr, halfWidth, mFrameWidth, mFrameHeight); yuvFormat = "NV21"; } } else if (isYUV444(data) && !data->mCbSkip) { rv = libyuv::I444ToI420(data->mYChannel, data->mYStride, data->mCbChannel, data->mCbCrStride, data->mCrChannel, data->mCbCrStride, y, mFrameWidth, cb, halfWidth, cr, halfWidth, mFrameWidth, mFrameHeight); yuvFormat = "I444"; } else if (isYUV422(data) && !data->mCbSkip) { rv = libyuv::I422ToI420(data->mYChannel, data->mYStride, data->mCbChannel, data->mCbCrStride, data->mCrChannel, data->mCbCrStride, y, mFrameWidth, cb, halfWidth, cr, halfWidth, mFrameWidth, mFrameHeight); yuvFormat = "I422"; } else { VP8LOG("Unsupported planar format\n"); NS_ASSERTION(false, "Unsupported planar format"); return NS_ERROR_NOT_IMPLEMENTED; } if (rv != 0) { VP8LOG("Converting an %s frame to I420 failed\n", yuvFormat.c_str()); return NS_ERROR_FAILURE; } VP8LOG("Converted an %s frame to I420\n"); } else { // Not YCbCr at all. Try to get access to the raw data and convert. RefPtr<SourceSurface> surf = img->GetAsSourceSurface(); if (!surf) { VP8LOG("Getting surface from %s image failed\n", Stringify(format).c_str()); return NS_ERROR_FAILURE; } RefPtr<DataSourceSurface> data = surf->GetDataSurface(); if (!data) { VP8LOG("Getting data surface from %s image with %s (%s) surface failed\n", Stringify(format).c_str(), Stringify(surf->GetType()).c_str(), Stringify(surf->GetFormat()).c_str()); return NS_ERROR_FAILURE; } DataSourceSurface::ScopedMap map(data, DataSourceSurface::READ); if (!map.IsMapped()) { VP8LOG("Reading DataSourceSurface from %s image with %s (%s) surface failed\n", Stringify(format).c_str(), Stringify(surf->GetType()).c_str(), Stringify(surf->GetFormat()).c_str()); return NS_ERROR_FAILURE; } int rv; switch (surf->GetFormat()) { case SurfaceFormat::B8G8R8A8: case SurfaceFormat::B8G8R8X8: rv = libyuv::ARGBToI420(static_cast<uint8*>(map.GetData()), map.GetStride(), y, mFrameWidth, cb, halfWidth, cr, halfWidth, mFrameWidth, mFrameHeight); break; case SurfaceFormat::R5G6B5_UINT16: rv = libyuv::RGB565ToI420(static_cast<uint8*>(map.GetData()), map.GetStride(), y, mFrameWidth, cb, halfWidth, cr, halfWidth, mFrameWidth, mFrameHeight); break; default: VP8LOG("Unsupported SourceSurface format %s\n", Stringify(surf->GetFormat()).c_str()); NS_ASSERTION(false, "Unsupported SourceSurface format"); return NS_ERROR_NOT_IMPLEMENTED; } if (rv != 0) { VP8LOG("%s to I420 conversion failed\n", Stringify(surf->GetFormat()).c_str()); return NS_ERROR_FAILURE; } VP8LOG("Converted a %s frame to I420\n", Stringify(surf->GetFormat()).c_str()); } mVPXImageWrapper->planes[VPX_PLANE_Y] = y; mVPXImageWrapper->planes[VPX_PLANE_U] = cb; mVPXImageWrapper->planes[VPX_PLANE_V] = cr; mVPXImageWrapper->stride[VPX_PLANE_Y] = mFrameWidth; mVPXImageWrapper->stride[VPX_PLANE_U] = halfWidth; mVPXImageWrapper->stride[VPX_PLANE_V] = halfWidth; return NS_OK; }
void ImageLayerOGL::RenderLayer(int, const nsIntPoint& aOffset) { nsRefPtr<ImageContainer> container = GetContainer(); if (!container || mOGLManager->CompositingDisabled()) return; mOGLManager->MakeCurrent(); AutoLockImage autoLock(container); Image *image = autoLock.GetImage(); if (!image) { return; } NS_ASSERTION(image->GetFormat() != REMOTE_IMAGE_BITMAP, "Remote images aren't handled yet in OGL layers!"); if (image->GetFormat() == PLANAR_YCBCR) { PlanarYCbCrImage *yuvImage = static_cast<PlanarYCbCrImage*>(image); if (!yuvImage->IsValid()) { return; } PlanarYCbCrOGLBackendData *data = static_cast<PlanarYCbCrOGLBackendData*>(yuvImage->GetBackendData(LAYERS_OPENGL)); if (data && data->mTextures->GetGLContext() != gl()) { // If these textures were allocated by another layer manager, // clear them out and re-allocate below. data = nullptr; yuvImage->SetBackendData(LAYERS_OPENGL, nullptr); } if (!data) { AllocateTexturesYCbCr(yuvImage); data = static_cast<PlanarYCbCrOGLBackendData*>(yuvImage->GetBackendData(LAYERS_OPENGL)); } if (!data || data->mTextures->GetGLContext() != gl()) { // XXX - Can this ever happen? If so I need to fix this! return; } gl()->MakeCurrent(); gl()->fActiveTexture(LOCAL_GL_TEXTURE2); gl()->fBindTexture(LOCAL_GL_TEXTURE_2D, data->mTextures[2].GetTextureID()); gl()->ApplyFilterToBoundTexture(mFilter); gl()->fActiveTexture(LOCAL_GL_TEXTURE1); gl()->fBindTexture(LOCAL_GL_TEXTURE_2D, data->mTextures[1].GetTextureID()); gl()->ApplyFilterToBoundTexture(mFilter); gl()->fActiveTexture(LOCAL_GL_TEXTURE0); gl()->fBindTexture(LOCAL_GL_TEXTURE_2D, data->mTextures[0].GetTextureID()); gl()->ApplyFilterToBoundTexture(mFilter); ShaderProgramOGL *program = mOGLManager->GetProgram(YCbCrLayerProgramType, GetMaskLayer()); program->Activate(); program->SetLayerQuadRect(nsIntRect(0, 0, yuvImage->GetSize().width, yuvImage->GetSize().height)); program->SetLayerTransform(GetEffectiveTransform()); program->SetLayerOpacity(GetEffectiveOpacity()); program->SetRenderOffset(aOffset); program->SetYCbCrTextureUnits(0, 1, 2); program->LoadMask(GetMaskLayer()); mOGLManager->BindAndDrawQuadWithTextureRect(program, yuvImage->GetData()->GetPictureRect(), nsIntSize(yuvImage->GetData()->mYSize.width, yuvImage->GetData()->mYSize.height)); // We shouldn't need to do this, but do it anyway just in case // someone else forgets. gl()->fActiveTexture(LOCAL_GL_TEXTURE0); } else if (image->GetFormat() == CAIRO_SURFACE) { CairoImage *cairoImage = static_cast<CairoImage*>(image); if (!cairoImage->mSurface) { return; } NS_ASSERTION(cairoImage->mSurface->GetContentType() != gfxASurface::CONTENT_ALPHA, "Image layer has alpha image"); CairoOGLBackendData *data = static_cast<CairoOGLBackendData*>(cairoImage->GetBackendData(LAYERS_OPENGL)); if (data && data->mTexture.GetGLContext() != gl()) { // If this texture was allocated by another layer manager, clear // it out and re-allocate below. data = nullptr; cairoImage->SetBackendData(LAYERS_OPENGL, nullptr); } if (!data) { AllocateTexturesCairo(cairoImage); data = static_cast<CairoOGLBackendData*>(cairoImage->GetBackendData(LAYERS_OPENGL)); } if (!data || data->mTexture.GetGLContext() != gl()) { // XXX - Can this ever happen? If so I need to fix this! return; } gl()->MakeCurrent(); gl()->fActiveTexture(LOCAL_GL_TEXTURE0); gl()->fBindTexture(LOCAL_GL_TEXTURE_2D, data->mTexture.GetTextureID()); ShaderProgramOGL *program = mOGLManager->GetProgram(data->mLayerProgram, GetMaskLayer()); gl()->ApplyFilterToBoundTexture(mFilter); program->Activate(); program->SetLayerQuadRect(nsIntRect(0, 0, cairoImage->GetSize().width, cairoImage->GetSize().height)); program->SetLayerTransform(GetEffectiveTransform()); program->SetLayerOpacity(GetEffectiveOpacity()); program->SetRenderOffset(aOffset); program->SetTextureUnit(0); program->LoadMask(GetMaskLayer()); mOGLManager->BindAndDrawQuad(program); } else if (image->GetFormat() == SHARED_TEXTURE) { SharedTextureImage* texImage = static_cast<SharedTextureImage*>(image); const SharedTextureImage::Data* data = texImage->GetData(); GLContext::SharedHandleDetails handleDetails; if (!gl()->GetSharedHandleDetails(data->mShareType, data->mHandle, handleDetails)) { NS_ERROR("Failed to get shared handle details"); return; } ShaderProgramOGL* program = mOGLManager->GetProgram(handleDetails.mProgramType, GetMaskLayer()); program->Activate(); if (handleDetails.mProgramType == gl::RGBARectLayerProgramType) { // 2DRect case, get the multiplier right for a sampler2DRect program->SetTexCoordMultiplier(data->mSize.width, data->mSize.height); } program->SetLayerTransform(GetEffectiveTransform()); program->SetLayerOpacity(GetEffectiveOpacity()); program->SetRenderOffset(aOffset); program->SetTextureUnit(0); program->SetTextureTransform(handleDetails.mTextureTransform); program->LoadMask(GetMaskLayer()); if (!texImage->GetBackendData(LAYERS_OPENGL)) { AllocateTextureSharedTexture(texImage, gl(), handleDetails.mTarget); } ImageOGLBackendData *backendData = static_cast<ImageOGLBackendData*>(texImage->GetBackendData(LAYERS_OPENGL)); gl()->fActiveTexture(LOCAL_GL_TEXTURE0); gl()->fBindTexture(handleDetails.mTarget, backendData->mTexture.GetTextureID()); if (!gl()->AttachSharedHandle(data->mShareType, data->mHandle)) { NS_ERROR("Failed to bind shared texture handle"); return; } gl()->ApplyFilterToBoundTexture(handleDetails.mTarget, mFilter); program->SetLayerQuadRect(nsIntRect(nsIntPoint(0, 0), data->mSize)); mOGLManager->BindAndDrawQuad(program, data->mInverted); gl()->fBindTexture(handleDetails.mTarget, 0); gl()->DetachSharedHandle(data->mShareType, data->mHandle); #ifdef MOZ_WIDGET_GONK } else if (image->GetFormat() == GONK_IO_SURFACE) { GonkIOSurfaceImage *ioImage = static_cast<GonkIOSurfaceImage*>(image); if (!ioImage) { return; } gl()->MakeCurrent(); gl()->fActiveTexture(LOCAL_GL_TEXTURE0); if (!ioImage->GetBackendData(LAYERS_OPENGL)) { AllocateTextureIOSurface(ioImage, gl()); } GonkIOSurfaceImageOGLBackendData *data = static_cast<GonkIOSurfaceImageOGLBackendData*>(ioImage->GetBackendData(LAYERS_OPENGL)); gl()->fActiveTexture(LOCAL_GL_TEXTURE0); gl()->BindExternalBuffer(data->mTexture.GetTextureID(), ioImage->GetNativeBuffer()); ShaderProgramOGL *program = mOGLManager->GetProgram(RGBAExternalLayerProgramType, GetMaskLayer()); gl()->ApplyFilterToBoundTexture(mFilter); program->Activate(); program->SetLayerQuadRect(nsIntRect(0, 0, ioImage->GetSize().width, ioImage->GetSize().height)); program->SetLayerTransform(GetEffectiveTransform()); program->SetLayerOpacity(GetEffectiveOpacity()); program->SetRenderOffset(aOffset); program->SetTextureUnit(0); program->LoadMask(GetMaskLayer()); mOGLManager->BindAndDrawQuadWithTextureRect(program, GetVisibleRegion().GetBounds(), nsIntSize(ioImage->GetSize().width, ioImage->GetSize().height)); #endif } GetContainer()->NotifyPaintedImage(image); }