void RasterImage::RecoverFromInvalidFrames(const IntSize& aSize, uint32_t aFlags) { if (!mHasSize) { return; } NS_WARNING("A RasterImage's frames became invalid. Attempting to recover..."); // Discard all existing frames, since they're probably all now invalid. SurfaceCache::RemoveImage(ImageKey(this)); // Relock the image if it's supposed to be locked. if (mLockCount > 0) { SurfaceCache::LockImage(ImageKey(this)); } // Animated images require some special handling, because we normally require // that they never be discarded. if (mAnimationState) { Decode(mSize, aFlags | FLAG_SYNC_DECODE, PlaybackType::eAnimated); ResetAnimation(); return; } // For non-animated images, it's fine to recover using an async decode. Decode(aSize, aFlags, PlaybackType::eStatic); }
LookupResult RasterImage::LookupFrameInternal(const IntSize& aSize, uint32_t aFlags, PlaybackType aPlaybackType) { if (mAnimationState && aPlaybackType == PlaybackType::eAnimated) { MOZ_ASSERT(mFrameAnimator); MOZ_ASSERT(ToSurfaceFlags(aFlags) == DefaultSurfaceFlags(), "Can't composite frames with non-default surface flags"); const size_t index = mAnimationState->GetCurrentAnimationFrameIndex(); return mFrameAnimator->GetCompositedFrame(index); } SurfaceFlags surfaceFlags = ToSurfaceFlags(aFlags); // We don't want any substitution for sync decodes, and substitution would be // illegal when high quality downscaling is disabled, so we use // SurfaceCache::Lookup in this case. if ((aFlags & FLAG_SYNC_DECODE) || !(aFlags & FLAG_HIGH_QUALITY_SCALING)) { return SurfaceCache::Lookup(ImageKey(this), RasterSurfaceKey(aSize, surfaceFlags, PlaybackType::eStatic)); } // We'll return the best match we can find to the requested frame. return SurfaceCache::LookupBestMatch(ImageKey(this), RasterSurfaceKey(aSize, surfaceFlags, PlaybackType::eStatic)); }
NS_IMETHODIMP RasterImage::UnlockImage() { MOZ_ASSERT(NS_IsMainThread(), "Main thread to encourage serialization with LockImage"); if (mError) { return NS_ERROR_FAILURE; } // It's an error to call this function if the lock count is 0 MOZ_ASSERT(mLockCount > 0, "Calling UnlockImage with mLockCount == 0!"); if (mLockCount == 0) { return NS_ERROR_ABORT; } // Decrement our lock count mLockCount--; // Unlock this image's surfaces in the SurfaceCache. if (mLockCount == 0 ) { SurfaceCache::UnlockImage(ImageKey(this)); } return NS_OK; }
RasterImage::WillDrawOpaqueNow() { if (!IsOpaque()) { return false; } if (mAnimationState) { // We never discard frames of animated images. return true; } // If we are not locked our decoded data could get discard at any time (ie // between the call to this function and when we are asked to draw), so we // have to return false if we are unlocked. if (IsUnlocked()) { return false; } LookupResult result = SurfaceCache::LookupBestMatch(ImageKey(this), RasterSurfaceKey(mSize, DefaultSurfaceFlags(), PlaybackType::eStatic)); MatchType matchType = result.Type(); if (matchType == MatchType::NOT_FOUND || matchType == MatchType::PENDING || !result.Surface()->IsFinished()) { return false; } return true; }
already_AddRefed<gfxDrawable> VectorImage::LookupCachedSurface(const SVGDrawingParameters& aParams) { // If we're not allowed to use a cached surface, don't attempt a lookup. if (aParams.flags & FLAG_BYPASS_SURFACE_CACHE) { return nullptr; } // We don't do any caching if we have animation, so don't bother with a lookup // in this case either. if (mHaveAnimations) { return nullptr; } LookupResult result = SurfaceCache::Lookup(ImageKey(this), VectorSurfaceKey(aParams.size, aParams.svgContext)); if (!result) { return nullptr; // No matching surface, or the OS freed the volatile buffer. } RefPtr<SourceSurface> sourceSurface = result.Surface()->GetSourceSurface(); if (!sourceSurface) { // Something went wrong. (Probably a GPU driver crash or device reset.) // Attempt to recover. RecoverFromLossOfSurfaces(); return nullptr; } RefPtr<gfxDrawable> svgDrawable = new gfxSurfaceDrawable(sourceSurface, result.Surface()->GetSize()); return svgDrawable.forget(); }
LookupResult FrameAnimator::GetCompositedFrame(AnimationState& aState) { // If we have a composited version of this frame, return that. if (mLastCompositedFrameIndex >= 0 && (uint32_t(mLastCompositedFrameIndex) == aState.mCurrentAnimationFrameIndex)) { return LookupResult(DrawableSurface(mCompositingFrame->DrawableRef()), MatchType::EXACT); } // Otherwise return the raw frame. DoBlend is required to ensure that we only // hit this case if the frame is not paletted and doesn't require compositing. LookupResult result = SurfaceCache::Lookup(ImageKey(mImage), RasterSurfaceKey(mSize, DefaultSurfaceFlags(), PlaybackType::eAnimated)); if (!result) { return result; } // Seek to the appropriate frame. If seeking fails, it means that we couldn't // get the frame we're looking for; treat this as if the lookup failed. if (NS_FAILED(result.Surface().Seek(aState.mCurrentAnimationFrameIndex))) { return LookupResult(MatchType::NOT_FOUND); } MOZ_ASSERT(!result.Surface()->GetIsPaletted(), "About to return a paletted frame"); return result; }
void VectorImage::RecoverFromLossOfSurfaces() { NS_WARNING("An imgFrame became invalid. Attempting to recover..."); // Discard all existing frames, since they're probably all now invalid. SurfaceCache::RemoveImage(ImageKey(this)); }
void RasterImage::CollectSizeOfSurfaces(nsTArray<SurfaceMemoryCounter>& aCounters, MallocSizeOf aMallocSizeOf) const { SurfaceCache::CollectSizeOfSurfaces(ImageKey(this), aCounters, aMallocSizeOf); if (mFrameAnimator) { mFrameAnimator->CollectSizeOfCompositingSurfaces(aCounters, aMallocSizeOf); } }
RawAccessFrameRef FrameAnimator::GetRawFrame(uint32_t aFrameNum) const { LookupResult result = SurfaceCache::Lookup(ImageKey(mImage), RasterSurfaceKey(mSize, DefaultSurfaceFlags(), aFrameNum)); return result ? result.DrawableRef()->RawAccessRef() : RawAccessFrameRef(); }
NS_IMETHODIMP VectorImage::RequestDiscard() { MOZ_ASSERT(NS_IsMainThread()); if (mDiscardable && mLockCount == 0) { SurfaceCache::RemoveImage(ImageKey(this)); mProgressTracker->OnDiscard(); } return NS_OK; }
// Indempotent error flagging routine. If a decoder is open, shuts it down. void RasterImage::DoError() { // If we've flagged an error before, we have nothing to do if (mError) { return; } // We can't safely handle errors off-main-thread, so dispatch a worker to // do it. if (!NS_IsMainThread()) { HandleErrorWorker::DispatchIfNeeded(this); return; } // Put the container in an error state. mError = true; // Stop animation and release our FrameAnimator. if (mAnimating) { StopAnimation(); } mAnimationState = Nothing(); mFrameAnimator = nullptr; // Release all locks. mLockCount = 0; SurfaceCache::UnlockImage(ImageKey(this)); // Release all frames from the surface cache. SurfaceCache::RemoveImage(ImageKey(this)); // Invalidate to get rid of any partially-drawn image content. NotifyProgress(NoProgress, IntRect(0, 0, mSize.width, mSize.height)); MOZ_LOG(gImgLog, LogLevel::Error, ("RasterImage: [this=%p] Error detected for image\n", this)); }
DecodedSurfaceProvider::DecodedSurfaceProvider(NotNull<RasterImage*> aImage, const SurfaceKey& aSurfaceKey, NotNull<Decoder*> aDecoder) : ISurfaceProvider(ImageKey(aImage.get()), aSurfaceKey, AvailabilityState::StartAsPlaceholder()) , mImage(aImage.get()) , mMutex("mozilla::image::DecodedSurfaceProvider") , mDecoder(aDecoder.get()) { MOZ_ASSERT(!mDecoder->IsMetadataDecode(), "Use MetadataDecodingTask for metadata decodes"); MOZ_ASSERT(mDecoder->IsFirstFrameDecode(), "Use AnimationSurfaceProvider for animation decodes"); }
void RasterImage::Discard() { MOZ_ASSERT(NS_IsMainThread()); MOZ_ASSERT(CanDiscard(), "Asked to discard but can't"); MOZ_ASSERT(!mAnimationState, "Asked to discard for animated image"); // Delete all the decoded frames. SurfaceCache::RemoveImage(ImageKey(this)); // Notify that we discarded. if (mProgressTracker) { mProgressTracker->OnDiscard(); } }
NS_IMETHODIMP VectorImage::LockImage() { MOZ_ASSERT(NS_IsMainThread()); if (mError) { return NS_ERROR_FAILURE; } mLockCount++; if (mLockCount == 1) { // Lock this image's surfaces in the SurfaceCache. SurfaceCache::LockImage(ImageKey(this)); } return NS_OK; }
NS_IMETHODIMP RasterImage::LockImage() { MOZ_ASSERT(NS_IsMainThread(), "Main thread to encourage serialization with UnlockImage"); if (mError) { return NS_ERROR_FAILURE; } // Increment the lock count mLockCount++; // Lock this image's surfaces in the SurfaceCache. if (mLockCount == 1) { SurfaceCache::LockImage(ImageKey(this)); } return NS_OK; }
LookupResult FrameAnimator::GetCompositedFrame(uint32_t aFrameNum) { MOZ_ASSERT(aFrameNum != 0, "First frame is never composited"); // If we have a composited version of this frame, return that. if (mLastCompositedFrameIndex == int32_t(aFrameNum)) { return LookupResult(mCompositingFrame->DrawableRef(), MatchType::EXACT); } // Otherwise return the raw frame. DoBlend is required to ensure that we only // hit this case if the frame is not paletted and doesn't require compositing. LookupResult result = SurfaceCache::Lookup(ImageKey(mImage), RasterSurfaceKey(mSize, DefaultSurfaceFlags(), aFrameNum)); MOZ_ASSERT(!result || !result.DrawableRef()->GetIsPaletted(), "About to return a paletted frame"); return result; }
RawAccessFrameRef FrameAnimator::GetRawFrame(uint32_t aFrameNum) const { LookupResult result = SurfaceCache::Lookup(ImageKey(mImage), RasterSurfaceKey(mSize, DefaultSurfaceFlags(), PlaybackType::eAnimated)); if (!result) { return RawAccessFrameRef(); } // Seek to the frame we want. If seeking fails, it means we couldn't get the // frame we're looking for, so we bail here to avoid returning the wrong frame // to the caller. if (NS_FAILED(result.Surface().Seek(aFrameNum))) { return RawAccessFrameRef(); // Not available yet. } return result.Surface()->RawAccessRef(); }
NS_IMETHODIMP VectorImage::UnlockImage() { MOZ_ASSERT(NS_IsMainThread()); if (mError) { return NS_ERROR_FAILURE; } if (mLockCount == 0) { MOZ_ASSERT_UNREACHABLE("Calling UnlockImage with a zero lock count"); return NS_ERROR_ABORT; } mLockCount--; if (mLockCount == 0) { // Unlock this image's surfaces in the SurfaceCache. SurfaceCache::UnlockImage(ImageKey(this)); } return NS_OK; }
void VectorImage::SendInvalidationNotifications() { // Animated images don't send out invalidation notifications as soon as // they're generated. Instead, InvalidateObserversOnNextRefreshDriverTick // records that there are pending invalidations and then returns immediately. // The notifications are actually sent from RequestRefresh(). We send these // notifications there to ensure that there is actually a document observing // us. Otherwise, the notifications are just wasted effort. // // Non-animated images call this method directly from // InvalidateObserversOnNextRefreshDriverTick, because RequestRefresh is never // called for them. Ordinarily this isn't needed, since we send out // invalidation notifications in OnSVGDocumentLoaded, but in rare cases the // SVG document may not be 100% ready to render at that time. In those cases // we would miss the subsequent invalidations if we didn't send out the // notifications directly in |InvalidateObservers...|. if (mProgressTracker) { SurfaceCache::RemoveImage(ImageKey(this)); mProgressTracker->SyncNotifyProgress(FLAG_FRAME_COMPLETE, GetMaxSizedIntRect()); } }
/* Dump an output file containing information about the current * state of the world */ void BaseApp::DumpOutputFile(const char *output_dir, const char *filename, int num_images, int num_cameras, int num_points, int *added_order, camera_params_t *cameras, v3_t *points, v3_t *colors, std::vector<ImageKeyVector> &pt_views /*bool output_radial_distortion*/) { clock_t start = clock(); int num_visible_points = 0; for (int i = 0; i < num_points; i++) { if (pt_views[i].size() > 0) num_visible_points++; } char buf[256]; sprintf(buf, "%s/%s", output_dir, filename); FILE *f = fopen(buf, "w"); if (f == NULL) { printf("Error opening file %s for writing\n", buf); return; } // if (output_radial_distortion) { /* Print version number */ // fprintf(f, "# Bundle file v0.4\n"); fprintf(f, "# Bundle file v0.3\n"); // } fprintf(f, "%d %d\n", num_images, num_visible_points); /* Dump cameras */ for (int i = 0; i < num_images; i++) { #if 0 /* Print the name of the file */ fprintf(f, "%s %d %d\n", m_image_data[i].m_name, m_image_data[i].GetWidth(), m_image_data[i].GetHeight()); #endif int idx = -1; for (int j = 0; j < num_cameras; j++) { if (added_order[j] == i) { idx = j; break; } } if (idx == -1) { // if (!output_radial_distortion) // fprintf(f, "0\n"); // else fprintf(f, "0 0 0\n"); fprintf(f, "0 0 0\n0 0 0\n0 0 0\n0 0 0\n"); } else { // if (!output_radial_distortion) // fprintf(f, "%0.10e\n", cameras[idx].f); // else fprintf(f, "%0.10e %0.10e %0.10e\n", cameras[idx].f, cameras[idx].k[0], cameras[idx].k[1]); fprintf(f, "%0.10e %0.10e %0.10e\n", cameras[idx].R[0], cameras[idx].R[1], cameras[idx].R[2]); fprintf(f, "%0.10e %0.10e %0.10e\n", cameras[idx].R[3], cameras[idx].R[4], cameras[idx].R[5]); fprintf(f, "%0.10e %0.10e %0.10e\n", cameras[idx].R[6], cameras[idx].R[7], cameras[idx].R[8]); double t[3]; matrix_product(3, 3, 3, 1, cameras[idx].R, cameras[idx].t, t); matrix_scale(3, 1, t, -1.0, t); fprintf(f, "%0.10e %0.10e %0.10e\n", t[0], t[1], t[2]); } } /* Dump points */ for (int i = 0; i < num_points; i++) { int num_visible = (int) pt_views[i].size(); if (num_visible > 0) { /* Position */ fprintf(f, "%0.10e %0.10e %0.10e\n", Vx(points[i]), Vy(points[i]), Vz(points[i])); // Vx(points[idx]), Vy(points[idx]), Vz(points[idx])); /* Color */ fprintf(f, "%d %d %d\n", iround(Vx(colors[i])), iround(Vy(colors[i])), iround(Vz(colors[i]))); int num_visible = (int) pt_views[i].size(); fprintf(f, "%d", num_visible); for (int j = 0; j < num_visible; j++) { int img = added_order[pt_views[i][j].first]; int key = pt_views[i][j].second; double x = m_image_data[img].m_keys[key].m_x; double y = m_image_data[img].m_keys[key].m_y; fprintf(f, " %d %d %0.4f %0.4f", img, key, x, y); } fprintf(f, "\n"); } } #if 0 /* Finally, dump all outliers */ ImageKeyVector outliers; for (int i = 0; i < num_images; i++) { /* Find the index of this camera in the ordering */ int idx = -1; for (int j = 0; j < num_cameras; j++) { if (added_order[j] == i) { idx = j; break; } } if (idx == -1) continue; int num_keys = GetNumKeys(i); for (int j = 0; j < num_keys; j++) { if (GetKey(i,j).m_extra == -2) { outliers.push_back(ImageKey(i,j)); } } } int num_outliers = (int) outliers.size(); fprintf(f, "%d\n", num_outliers); for (int i = 0; i < num_outliers; i++) { fprintf(f, "%d %d\n", outliers[i].first, outliers[i].second); } #endif fclose(f); clock_t end = clock(); printf("[DumpOutputFile] Wrote file in %0.3fs\n", (double) (end - start) / (double) CLOCKS_PER_SEC); }
void VectorImage::CreateSurfaceAndShow(const SVGDrawingParameters& aParams) { mSVGDocumentWrapper->UpdateViewportBounds(aParams.viewportSize); mSVGDocumentWrapper->FlushImageTransformInvalidation(); RefPtr<gfxDrawingCallback> cb = new SVGDrawingCallback(mSVGDocumentWrapper, IntRect(IntPoint(0, 0), aParams.viewportSize), aParams.size, aParams.flags); RefPtr<gfxDrawable> svgDrawable = new gfxCallbackDrawable(cb, aParams.size); bool bypassCache = bool(aParams.flags & FLAG_BYPASS_SURFACE_CACHE) || // Refuse to cache animated images: // XXX(seth): We may remove this restriction in bug 922893. mHaveAnimations || // The image is too big to fit in the cache: !SurfaceCache::CanHold(aParams.size); if (bypassCache) { return Show(svgDrawable, aParams); } // We're about to rerasterize, which may mean that some of the previous // surfaces we've rasterized aren't useful anymore. We can allow them to // expire from the cache by unlocking them here, and then sending out an // invalidation. If this image is locked, any surfaces that are still useful // will become locked again when Draw touches them, and the remainder will // eventually expire. SurfaceCache::UnlockSurfaces(ImageKey(this)); // Try to create an imgFrame, initializing the surface it contains by drawing // our gfxDrawable into it. (We use FILTER_NEAREST since we never scale here.) RefPtr<imgFrame> frame = new imgFrame; nsresult rv = frame->InitWithDrawable(svgDrawable, aParams.size, SurfaceFormat::B8G8R8A8, Filter::POINT, aParams.flags); // If we couldn't create the frame, it was probably because it would end // up way too big. Generally it also wouldn't fit in the cache, but the prefs // could be set such that the cache isn't the limiting factor. if (NS_FAILED(rv)) { return Show(svgDrawable, aParams); } // Take a strong reference to the frame's surface and make sure it hasn't // already been purged by the operating system. RefPtr<SourceSurface> surface = frame->GetSurface(); if (!surface) { return Show(svgDrawable, aParams); } // Attempt to cache the frame. SurfaceCache::Insert(frame, ImageKey(this), VectorSurfaceKey(aParams.size, aParams.svgContext, aParams.animationTime)); // Draw. RefPtr<gfxDrawable> drawable = new gfxSurfaceDrawable(surface, aParams.size); Show(drawable, aParams); // Send out an invalidation so that surfaces that are still in use get // re-locked. See the discussion of the UnlockSurfaces call above. mProgressTracker->SyncNotifyProgress(FLAG_FRAME_COMPLETE, GetMaxSizedIntRect()); }
VectorImage::Draw(gfxContext* aContext, const nsIntSize& aSize, const ImageRegion& aRegion, uint32_t aWhichFrame, Filter aFilter, const Maybe<SVGImageContext>& aSVGContext, uint32_t aFlags) { if (aWhichFrame > FRAME_MAX_VALUE) { return DrawResult::BAD_ARGS; } if (!aContext) { return DrawResult::BAD_ARGS; } if (mError) { return DrawResult::BAD_IMAGE; } if (!mIsFullyLoaded) { return DrawResult::NOT_READY; } if (mIsDrawing) { NS_WARNING("Refusing to make re-entrant call to VectorImage::Draw"); return DrawResult::TEMPORARY_ERROR; } if (mAnimationConsumers == 0 && mProgressTracker) { mProgressTracker->OnUnlockedDraw(); } AutoRestore<bool> autoRestoreIsDrawing(mIsDrawing); mIsDrawing = true; Maybe<SVGImageContext> svgContext; // If FLAG_FORCE_PRESERVEASPECTRATIO_NONE bit is set, that mean we should // overwrite SVG preserveAspectRatio attibute of this image with none, and // always stretch this image to viewport non-uniformly. // And we can do this only if the caller pass in the the SVG viewport, via // aSVGContext. if ((aFlags & FLAG_FORCE_PRESERVEASPECTRATIO_NONE) && aSVGContext.isSome()) { Maybe<SVGPreserveAspectRatio> aspectRatio = Some(SVGPreserveAspectRatio(SVG_PRESERVEASPECTRATIO_NONE, SVG_MEETORSLICE_UNKNOWN)); svgContext = Some(SVGImageContext(aSVGContext->GetViewportSize(), aspectRatio)); } else { svgContext = aSVGContext; } float animTime = (aWhichFrame == FRAME_FIRST) ? 0.0f : mSVGDocumentWrapper->GetCurrentTime(); AutoSVGRenderingState autoSVGState(svgContext, animTime, mSVGDocumentWrapper->GetRootSVGElem()); SVGDrawingParameters params(aContext, aSize, aRegion, aFilter, svgContext, animTime, aFlags); if (aFlags & FLAG_BYPASS_SURFACE_CACHE) { CreateSurfaceAndShow(params); return DrawResult::SUCCESS; } LookupResult result = SurfaceCache::Lookup(ImageKey(this), VectorSurfaceKey(params.size, params.svgContext, params.animationTime)); // Draw. if (result) { RefPtr<SourceSurface> surface = result.DrawableRef()->GetSurface(); if (surface) { RefPtr<gfxDrawable> svgDrawable = new gfxSurfaceDrawable(surface, result.DrawableRef()->GetSize()); Show(svgDrawable, params); return DrawResult::SUCCESS; } // We lost our surface due to some catastrophic event. RecoverFromLossOfSurfaces(); } CreateSurfaceAndShow(params); return DrawResult::SUCCESS; }
void VectorImage::CollectSizeOfSurfaces(nsTArray<SurfaceMemoryCounter>& aCounters, MallocSizeOf aMallocSizeOf) const { SurfaceCache::CollectSizeOfSurfaces(ImageKey(this), aCounters, aMallocSizeOf); }
void VectorImage::CreateSurfaceAndShow(const SVGDrawingParameters& aParams) { mSVGDocumentWrapper->UpdateViewportBounds(aParams.viewportSize); mSVGDocumentWrapper->FlushImageTransformInvalidation(); nsRefPtr<gfxDrawingCallback> cb = new SVGDrawingCallback(mSVGDocumentWrapper, nsIntRect(nsIntPoint(0, 0), aParams.viewportSize), aParams.size, aParams.flags); nsRefPtr<gfxDrawable> svgDrawable = new gfxCallbackDrawable(cb, ThebesIntSize(aParams.size)); // We take an early exit without using the surface cache if too large, // because for vector images this can cause bad perf issues if large sizes // are scaled repeatedly (a rather common scenario) that can quickly exhaust // the cache. // Similar to max image size calculations, this has a max cap and size check. // max cap = 8000 (pixels); size check = 5% of cache int32_t maxDimension = 8000; int32_t maxCacheElemSize = (gfxPrefs::ImageMemSurfaceCacheMaxSizeKB() * 1024) / 20; bool bypassCache = bool(aParams.flags & FLAG_BYPASS_SURFACE_CACHE) || // Refuse to cache animated images: // XXX(seth): We may remove this restriction in bug 922893. mHaveAnimations || // The image is too big to fit in the cache: !SurfaceCache::CanHold(aParams.size) || // Image x or y is larger than our cache cap: aParams.size.width > maxDimension || aParams.size.height > maxDimension; if (!bypassCache) { // This is separated out to make sure width and height are sane at this point // and the result can't overflow. Note: keep maxDimension low enough so that // (maxDimension)^2 x 4 < INT32_MAX. // Assuming surface size for any rendered vector image is RGBA, so 4Bpp. bypassCache = (aParams.size.width * aParams.size.height * 4) > maxCacheElemSize; } if (bypassCache) return Show(svgDrawable, aParams); // Try to create an imgFrame, initializing the surface it contains by drawing // our gfxDrawable into it. (We use FILTER_NEAREST since we never scale here.) nsRefPtr<imgFrame> frame = new imgFrame; nsresult rv = frame->InitWithDrawable(svgDrawable, ThebesIntSize(aParams.size), SurfaceFormat::B8G8R8A8, GraphicsFilter::FILTER_NEAREST, aParams.flags); // If we couldn't create the frame, it was probably because it would end // up way too big. Generally it also wouldn't fit in the cache, but the prefs // could be set such that the cache isn't the limiting factor. if (NS_FAILED(rv)) return Show(svgDrawable, aParams); // Take a strong reference to the frame's surface and make sure it hasn't // already been purged by the operating system. RefPtr<SourceSurface> surface = frame->GetSurface(); if (!surface) return Show(svgDrawable, aParams); // Attempt to cache the frame. SurfaceCache::Insert(frame, ImageKey(this), VectorSurfaceKey(aParams.size, aParams.svgContext, aParams.animationTime), Lifetime::Transient); // Draw. nsRefPtr<gfxDrawable> drawable = new gfxSurfaceDrawable(surface, ThebesIntSize(aParams.size)); Show(drawable, aParams); }
/* Read in information about the world */ void BaseApp::ReadBundleFile(const char *filename) { printf("[ReadBundleFile] Reading file...\n"); FILE *f = fopen(filename, "r"); if (f == NULL) { printf("Error opening file %s for reading\n", filename); return; } int num_images, num_points; char first_line[256]; fgets(first_line, 256, f); if (first_line[0] == '#') { double version; sscanf(first_line, "# Bundle file v%lf", &version); m_bundle_version = version; printf("[ReadBundleFile] Bundle version: %0.3f\n", version); fscanf(f, "%d %d\n", &num_images, &num_points); } else if (first_line[0] == 'v') { double version; sscanf(first_line, "v%lf", &version); m_bundle_version = version; printf("[ReadBundleFile] Bundle version: %0.3f\n", version); fscanf(f, "%d %d\n", &num_images, &num_points); } else { m_bundle_version = 0.1; sscanf(first_line, "%d %d\n", &num_images, &num_points); } printf("[ReadBundleFile] Reading %d images and %d points...\n", num_images, num_points); if (num_images != GetNumImages()) { printf("Error: number of images doesn't match file!\n"); return; } /* Read cameras */ for (int i = 0; i < num_images; i++) { double focal_length; double R[9]; double t[3]; double k[2] = { 0.0, 0.0 }; if (m_bundle_version >= 0.4) { char name[512]; int w, h; fscanf(f, "%s %d %d\n", name, &w, &h); } /* Focal length */ if (m_bundle_version > 0.1) { fscanf(f, "%lf %lf %lf\n", &focal_length, k+0, k+1); } else { fscanf(f, "%lf\n", &focal_length); } /* Rotation */ fscanf(f, "%lf %lf %lf\n%lf %lf %lf\n%lf %lf %lf\n", R+0, R+1, R+2, R+3, R+4, R+5, R+6, R+7, R+8); /* Translation */ fscanf(f, "%lf %lf %lf\n", t+0, t+1, t+2); #if 0 if (m_bundle_version < 0.3) { R[2] = -R[2]; R[5] = -R[5]; R[6] = -R[6]; R[7] = -R[7]; t[2] = -t[2]; } #endif if (focal_length <= 100.0 || m_image_data[i].m_ignore_in_bundle) { /* No (or bad) information about this camera */ m_image_data[i].m_camera.m_adjusted = false; } else { CameraInfo cd; cd.m_adjusted = true; cd.m_width = m_image_data[i].GetWidth(); cd.m_height = m_image_data[i].GetHeight(); cd.m_focal = focal_length; cd.m_k[0] = k[0]; cd.m_k[1] = k[1]; memcpy(cd.m_R, R, sizeof(double) * 9); memcpy(cd.m_t, t, sizeof(double) * 3); cd.Finalize(); m_image_data[i].m_camera = cd; } } /* Read points */ m_point_data.clear(); m_point_data.resize(num_points); int num_min_views_points = 0; for (int i = 0; i < num_points; i++) { PointData &pt = m_point_data[i]; /* Position */ fscanf(f, "%lf %lf %lf\n", pt.m_pos + 0, pt.m_pos + 1, pt.m_pos + 2); // if (m_bundle_version < 0.3) // pt.m_pos[2] = -pt.m_pos[2]; /* Color */ fscanf(f, "%f %f %f\n", pt.m_color + 0, pt.m_color + 1, pt.m_color + 2); int num_visible; fscanf(f, "%d", &num_visible); pt.m_num_vis=num_visible; if (num_visible >=3) num_min_views_points++; // pt.m_views.resize(num_visible); for (int j = 0; j < num_visible; j++) { int view, key; fscanf(f, "%d %d", &view, &key); if (!m_image_data[view].m_camera.m_adjusted) { // printf("[ReadBundleFile] " // "Removing view %d from point %d\n", view, i); } else { /* Check cheirality */ bool val = (m_bundle_version >= 0.3); double proj_test[2]; if (m_image_data[view].m_camera. Project(pt.m_pos, proj_test) == val) { pt.m_views.push_back(ImageKey(view, key)); } else { printf("[ReadBundleFile] " "Removing view %d from point %d [cheirality]\n", view, i); // pt.m_views.push_back(ImageKey(view, key)); } } // pt.m_views.push_back(ImageKey(view, key)); if (m_bundle_version >= 0.3) { double x, y; fscanf(f, "%lf %lf", &x, &y); } } // #define CROP_POINT_CLOUD #ifdef CROP_POINT_CLOUD const double x_min = 1.327; const double x_max = 3.556; const double y_min = -1.414; const double y_max = 1.074; const double z_min = -5.502; const double z_max = -3.288; if (pt.m_pos[0] < x_min || pt.m_pos[0] > x_max || pt.m_pos[1] < y_min || pt.m_pos[1] > y_max || pt.m_pos[2] < z_min || pt.m_pos[2] > z_max) { pt.m_views.clear(); } #endif /* CROP_POINT_CLOUD */ } #if 0 /* Read outliers */ int num_outliers; fscanf(f, "%d", &num_outliers); for (int i = 0; i < num_outliers; i++) { ImageKey ik; fscanf(f, "%d %d", &(ik.first), &(ik.second)); m_outliers.push_back(ik); } #endif fclose(f); printf("[ReadBundleFile] %d / %d points visible to more than 2 cameras!\n", num_min_views_points, num_points); }
NS_IMETHODIMP RasterImage::Decode(const IntSize& aSize, uint32_t aFlags, PlaybackType aPlaybackType) { MOZ_ASSERT(NS_IsMainThread()); if (mError) { return NS_ERROR_FAILURE; } // If we don't have a size yet, we can't do any other decoding. if (!mHasSize) { mWantFullDecode = true; return NS_OK; } // We're about to decode again, which may mean that some of the previous sizes // we've decoded at aren't useful anymore. We can allow them to expire from // the cache by unlocking them here. When the decode finishes, it will send an // invalidation that will cause all instances of this image to redraw. If this // image is locked, any surfaces that are still useful will become locked // again when LookupFrame touches them, and the remainder will eventually // expire. SurfaceCache::UnlockEntries(ImageKey(this)); // Determine which flags we need to decode this image with. DecoderFlags decoderFlags = DefaultDecoderFlags(); if (aFlags & FLAG_ASYNC_NOTIFY) { decoderFlags |= DecoderFlags::ASYNC_NOTIFY; } if (mTransient) { decoderFlags |= DecoderFlags::IMAGE_IS_TRANSIENT; } if (mHasBeenDecoded) { decoderFlags |= DecoderFlags::IS_REDECODE; } SurfaceFlags surfaceFlags = ToSurfaceFlags(aFlags); if (IsOpaque()) { // If there's no transparency, it doesn't matter whether we premultiply // alpha or not. surfaceFlags &= ~SurfaceFlags::NO_PREMULTIPLY_ALPHA; } // Create a decoder. RefPtr<IDecodingTask> task; if (mAnimationState && aPlaybackType == PlaybackType::eAnimated) { task = DecoderFactory::CreateAnimationDecoder(mDecoderType, WrapNotNull(this), mSourceBuffer, mSize, decoderFlags, surfaceFlags); } else { task = DecoderFactory::CreateDecoder(mDecoderType, WrapNotNull(this), mSourceBuffer, mSize, aSize, decoderFlags, surfaceFlags, mRequestedSampleSize); } // Make sure DecoderFactory was able to create a decoder successfully. if (!task) { return NS_ERROR_FAILURE; } mDecodeCount++; // We're ready to decode; start the decoder. LaunchDecodingTask(task, this, aFlags, mHasSourceData); return NS_OK; }
/* void requestDiscard() */ NS_IMETHODIMP VectorImage::RequestDiscard() { SurfaceCache::RemoveImage(ImageKey(this)); return NS_OK; }
/* Add new points to the bundle adjustment */ int BundlerApp::BundleAdjustAddNewPoints(int camera_idx, int num_points, int num_cameras, int *added_order, camera_params_t *cameras, v3_t *points, v3_t *colors, double reference_baseline, std::vector<ImageKeyVector> &pt_views) { int pt_count = num_points; int image_idx = added_order[camera_idx]; /* Recompute the locations of the new points given the initial * pose estimate */ for (int i = 0; i < num_cameras; i++) { int other = added_order[i]; if (other == image_idx) continue; int first = MIN(image_idx, other); int second = MAX(image_idx, other); MatchIndex idx = GetMatchIndex(first, second); SetMatchesFromTracks(first, second); printf(" Matches[%d,%d] = %d\n", image_idx, other, (int) m_matches.GetNumMatches(idx)); // (int) m_match_lists[idx].size()); double disti = GetCameraDistance(cameras + i, cameras + camera_idx); printf(" dist0, disti = %0.3e, %0.3e\n", reference_baseline, disti); if (disti < m_min_camera_distance_ratio * reference_baseline) { printf(" Distance too low (possible panorama?)\n"); // m_match_lists[idx].clear(); m_matches.ClearMatch(idx); continue; } std::vector<KeypointMatch> &list = m_matches.GetMatchList(idx); for (int j = 0; j < (int) list.size(); j++) { int idx1 = list[j].m_idx1; int idx2 = list[j].m_idx2; int this_idx, other_idx; if (image_idx == first) { this_idx = idx1; other_idx = idx2; } else { other_idx = idx1; this_idx = idx2; } if (GetKey(other,other_idx).m_extra == -2) { /* The other key was already marked as an outlier */ continue; } else if (GetKey(image_idx,this_idx).m_extra == -2) { /* This key was already marked as an outlier */ continue; } if (GetKey(other,other_idx).m_extra == -1 && GetKey(image_idx,this_idx).m_extra >= 0) { /**** Connecting an existing point *** */ /* Connect up the other point to this one */ int pt_idx = GetKey(image_idx,this_idx).m_extra; /* Check reprojection error */ v2_t pr = sfm_project_final(cameras + i, points[pt_idx], true, m_estimate_distortion); double dx = GetKey(other,other_idx).m_x - Vx(pr); double dy = GetKey(other,other_idx).m_y - Vy(pr); double proj_error = sqrt(dx * dx + dy * dy); if (proj_error >= 32.0) { printf(" Would have connected existing match " "%d ==> %d [%d] (cam: %d), \n" " but reprojection error (%0.3f) " "is too high.\n", this_idx, other_idx, pt_idx, other, proj_error); } else { printf(" Connecting existing match " "%d ==> %d [%d] (cam: %d) [%0.3f]\n", this_idx, other_idx, pt_idx, other, proj_error); GetKey(other,other_idx).m_extra = pt_idx; pt_views[pt_idx].push_back(ImageKey(i, other_idx)); } } else if (GetKey(other,other_idx).m_extra == -1) { if (GetKey(image_idx,this_idx).m_extra != -1) { printf("Error! Key (%d,%d) shouldn't be seen yet!\n", image_idx, this_idx); printf("Point index is %d\n", GetKey(image_idx,this_idx).m_extra); } /* This is a new point */ GetKey(other,other_idx).m_extra = pt_count; GetKey(image_idx,this_idx).m_extra = pt_count; /* Set up the 3D point */ v2_t p = v2_new(GetKey(other,other_idx).m_x, GetKey(other,other_idx).m_y); v2_t q = v2_new(GetKey(image_idx,this_idx).m_x, GetKey(image_idx,this_idx).m_y); if (m_optimize_for_fisheye) { double p_x = Vx(p), p_y = Vy(p); double q_x = Vx(q), q_y = Vy(q); m_image_data[other]. UndistortPoint(p_x, p_y, Vx(p), Vy(p)); m_image_data[image_idx]. UndistortPoint(q_x, q_y, Vx(q), Vy(q)); } double proj_error = 0.0; bool in_front = false; double angle = 0.0; points[pt_count] = Triangulate(p, q, cameras[i], cameras[camera_idx], proj_error, in_front, angle, true); /* Check that the angle between the rays is large * enough */ if (RAD2DEG(angle) < m_ray_angle_threshold) { printf(" Ray angle %d => %d is too small (%0.3f)\n", this_idx, other_idx, RAD2DEG(angle)); /* Remove point */ GetKey(other,other_idx).m_extra = -1; GetKey(image_idx,this_idx).m_extra = -1; continue; } /* Check the reprojection error */ if (proj_error >= ADD_REPROJECTION_ERROR) { printf(" Projection error for %d => %d is %0.3e, " "removing\n", this_idx, other_idx, proj_error); /* Remove point */ GetKey(other,other_idx).m_extra = -2; GetKey(image_idx,this_idx).m_extra = -2; continue; } /* Check cheirality */ if (!in_front) { printf(" Cheirality violated!\n"); /* Remove point */ GetKey(other,other_idx).m_extra = -2; GetKey(image_idx,this_idx).m_extra = -2; continue; } printf(" Adding match %d ==> %d [%d] (cam: %d ==> %d) " "[%0.3f, %0.3f]\n", other_idx, this_idx, pt_count, image_idx, other, RAD2DEG(angle), proj_error); /* Finally, add the point */ unsigned char r = GetKey(other,other_idx).m_r; unsigned char g = GetKey(other,other_idx).m_g; unsigned char b = GetKey(other,other_idx).m_b; colors[pt_count] = v3_new((double) r, (double) g, (double) b); ImageKeyVector views; views.push_back(ImageKey(i, other_idx)); views.push_back(ImageKey(camera_idx, this_idx)); pt_views.push_back(views); pt_count++; } else if (GetKey(other,other_idx).m_extra >= 0 && GetKey(image_idx,this_idx).m_extra == -1) { /* We didn't connect this point originally -- * check if it's now a good idea to add it in */ /* Connect up the other point to this one */ int pt_idx = GetKey(other,other_idx).m_extra; /* Check reprojection error */ v2_t pr = sfm_project_final(cameras + camera_idx, points[pt_idx], true, m_estimate_distortion); double dx = GetKey(image_idx,this_idx).m_x - Vx(pr); double dy = GetKey(image_idx,this_idx).m_y - Vy(pr); double proj_error = sqrt(dx * dx + dy * dy); if (proj_error <= INIT_REPROJECTION_ERROR) { printf(" Reconnecting point [%d] (%d) (error: %0.3f)\n", pt_idx, this_idx, proj_error); GetKey(image_idx,this_idx).m_extra = pt_idx; pt_views[pt_idx].push_back(ImageKey(camera_idx,this_idx)); } else { /* Throw out this point as an outlier */ GetKey(image_idx,this_idx).m_extra = -2; } } } // m_match_lists[idx].clear(); m_matches.ClearMatch(idx); } return pt_count; }
/* Add new points to the bundle adjustment */ int BundlerApp::BundleAdjustAddAllNewPoints(int num_points, int num_cameras, int *added_order, camera_params_t *cameras, v3_t *points, v3_t *colors, double reference_baseline, std::vector<ImageKeyVector> &pt_views, double max_reprojection_error, int min_views) { std::vector<int> track_idxs; std::vector<ImageKeyVector> new_tracks; int num_tracks_total = (int) m_track_data.size(); int *tracks_seen = new int[num_tracks_total]; for (int i = 0; i < num_tracks_total; i++) { tracks_seen[i] = -1; } /* Gather up the projections of all the new tracks */ for (int i = 0; i < num_cameras; i++) { int image_idx1 = added_order[i]; int num_keys = GetNumKeys(image_idx1); for (int j = 0; j < num_keys; j++) { Keypoint &key = GetKey(image_idx1, j); if (key.m_track == -1) continue; /* Key belongs to no track */ if (key.m_extra != -1) continue; /* Key is outlier or has already been added */ int track_idx = key.m_track; /* Check if this track is already associated with a point */ if (m_track_data[track_idx].m_extra != -1) continue; /* Check if we've seen this track */ int seen = tracks_seen[track_idx]; if (seen == -1) { /* We haven't yet seen this track, create a new track */ tracks_seen[track_idx] = (int) new_tracks.size(); ImageKeyVector track; track.push_back(ImageKey(i, j)); new_tracks.push_back(track); track_idxs.push_back(track_idx); } else { new_tracks[seen].push_back(ImageKey(i, j)); } } } delete [] tracks_seen; /* Now for each (sub) track, triangulate to see if the track is * consistent */ int pt_count = num_points; int num_ill_conditioned = 0; int num_high_reprojection = 0; int num_cheirality_failed = 0; int num_added = 0; int num_tracks = (int) new_tracks.size(); for (int i = 0; i < num_tracks; i++) { int num_views = (int) new_tracks[i].size(); if (num_views < min_views) continue; /* Not enough views */ #if 0 printf("Triangulating track "); PrintTrack(new_tracks[i]); printf("\n"); #endif /* Check if at least two cameras fix the position of the point */ bool conditioned = false; bool good_distance = false; double max_angle = 0.0; for (int j = 0; j < num_views; j++) { for (int k = j+1; k < num_views; k++) { int camera_idx1 = new_tracks[i][j].first; int image_idx1 = added_order[camera_idx1]; int key_idx1 = new_tracks[i][j].second; int camera_idx2 = new_tracks[i][k].first; int image_idx2 = added_order[camera_idx2]; int key_idx2 = new_tracks[i][k].second; Keypoint &key1 = GetKey(image_idx1, key_idx1); Keypoint &key2 = GetKey(image_idx2, key_idx2); v2_t p = v2_new(key1.m_x, key1.m_y); v2_t q = v2_new(key2.m_x, key2.m_y); if (m_optimize_for_fisheye) { double p_x = Vx(p), p_y = Vy(p); double q_x = Vx(q), q_y = Vy(q); m_image_data[image_idx1]. UndistortPoint(p_x, p_y, Vx(p), Vy(p)); m_image_data[image_idx2]. UndistortPoint(q_x, q_y, Vx(q), Vy(q)); } double angle = ComputeRayAngle(p, q, cameras[camera_idx1], cameras[camera_idx2]); if (angle > max_angle) max_angle = angle; /* Check that the angle between the rays is large * enough */ if (RAD2DEG(angle) >= m_ray_angle_threshold) { conditioned = true; } #if 0 double dist_jk = GetCameraDistance(cameras + j, cameras + k, m_explicit_camera_centers); if (dist_jk > m_min_camera_distance_ratio * reference_baseline) good_distance = true; #else good_distance = true; #endif } } if (!conditioned || !good_distance) { num_ill_conditioned++; #if 0 printf(">> Track is ill-conditioned [max_angle = %0.3f]\n", RAD2DEG(max_angle)); fflush(stdout); #endif continue; } double error; v3_t pt; if (!m_panorama_mode) { pt = TriangulateNViews(new_tracks[i], added_order, cameras, error, true); } else { pt = GeneratePointAtInfinity(new_tracks[i], added_order, cameras, error, true); } // Changed by Wan, Yi if (::isnan(error) || error > max_reprojection_error) { num_high_reprojection++; #if 0 printf(">> Reprojection error [%0.3f] is too large\n", error); fflush(stdout); #endif continue; } bool all_in_front = true; for (int j = 0; j < num_views; j++) { int camera_idx = new_tracks[i][j].first; bool in_front = CheckCheirality(pt, cameras[camera_idx]); if (!in_front) { all_in_front = false; break; } } if (!all_in_front) { num_cheirality_failed++; #if 0 printf(">> Cheirality check failed\n"); fflush(stdout); #endif continue; } /* All tests succeeded, so let's add the point */ #if 0 printf("Triangulating track "); PrintTrack(new_tracks[i]); printf("\n"); printf(">> All tests succeeded [%0.3f, %0.3f] for point [%d]\n", RAD2DEG(max_angle), error, pt_count); #endif fflush(stdout); points[pt_count] = pt; int camera_idx = new_tracks[i][0].first; int image_idx = added_order[camera_idx]; int key_idx = new_tracks[i][0].second; unsigned char r = GetKey(image_idx, key_idx).m_r; unsigned char g = GetKey(image_idx, key_idx).m_g; unsigned char b = GetKey(image_idx, key_idx).m_b; colors[pt_count] = v3_new((double) r, (double) g, (double) b); pt_views.push_back(new_tracks[i]); /* Set the point index on the keys */ for (int j = 0; j < num_views; j++) { int camera_idx = new_tracks[i][j].first; int image_idx = added_order[camera_idx]; int key_idx = new_tracks[i][j].second; GetKey(image_idx, key_idx).m_extra = pt_count; } int track_idx = track_idxs[i]; m_track_data[track_idx].m_extra = pt_count; pt_count++; num_added++; } printf("[AddAllNewPoints] Added %d new points\n", num_added); printf("[AddAllNewPoints] Ill-conditioned tracks: %d\n", num_ill_conditioned); printf("[AddAllNewPoints] Bad reprojections: %d\n", num_high_reprojection); printf("[AddAllNewPoints] Failed cheirality checks: %d\n", num_cheirality_failed); return pt_count; }
VectorImage::Draw(gfxContext* aContext, const nsIntSize& aSize, const ImageRegion& aRegion, uint32_t aWhichFrame, GraphicsFilter aFilter, const Maybe<SVGImageContext>& aSVGContext, uint32_t aFlags) { if (aWhichFrame > FRAME_MAX_VALUE) { return DrawResult::BAD_ARGS; } if (!aContext) { return DrawResult::BAD_ARGS; } if (mError) { return DrawResult::BAD_IMAGE; } if (!mIsFullyLoaded) { return DrawResult::NOT_READY; } if (mIsDrawing) { NS_WARNING("Refusing to make re-entrant call to VectorImage::Draw"); return DrawResult::TEMPORARY_ERROR; } if (mAnimationConsumers == 0 && mProgressTracker) { mProgressTracker->OnUnlockedDraw(); } AutoRestore<bool> autoRestoreIsDrawing(mIsDrawing); mIsDrawing = true; float animTime = (aWhichFrame == FRAME_FIRST) ? 0.0f : mSVGDocumentWrapper->GetCurrentTime(); AutoSVGRenderingState autoSVGState(aSVGContext, animTime, mSVGDocumentWrapper->GetRootSVGElem()); // Pack up the drawing parameters. SVGDrawingParameters params(aContext, aSize, aRegion, aFilter, aSVGContext, animTime, aFlags); if (aFlags & FLAG_BYPASS_SURFACE_CACHE) { CreateSurfaceAndShow(params); return DrawResult::SUCCESS; } DrawableFrameRef frameRef = SurfaceCache::Lookup(ImageKey(this), VectorSurfaceKey(params.size, params.svgContext, params.animationTime)); // Draw. if (frameRef) { RefPtr<SourceSurface> surface = frameRef->GetSurface(); if (surface) { nsRefPtr<gfxDrawable> svgDrawable = new gfxSurfaceDrawable(surface, frameRef->GetSize()); Show(svgDrawable, params); return DrawResult::SUCCESS; } // We lost our surface due to some catastrophic event. RecoverFromLossOfSurfaces(); } CreateSurfaceAndShow(params); return DrawResult::SUCCESS; }