Mosaic::Status Mosaic::composePanorama(InputArrayOfArrays images, OutputArray pano) { LOGLN("Warping images (auxiliary)... "); std::vector<UMat> imgs; images.getUMatVector(imgs); if (!imgs.empty()) { CV_Assert(imgs.size() == imgs_.size()); UMat img; seam_est_imgs_.resize(imgs.size()); for (size_t i = 0; i < imgs.size(); ++i) { imgs_[i] = imgs[i]; resize(imgs[i], img, Size(), seam_scale_, seam_scale_); seam_est_imgs_[i] = img.clone(); } std::vector<UMat> seam_est_imgs_subset; std::vector<UMat> imgs_subset; for (size_t i = 0; i < indices_.size(); ++i) { imgs_subset.push_back(imgs_[indices_[i]]); seam_est_imgs_subset.push_back(seam_est_imgs_[indices_[i]]); } seam_est_imgs_ = seam_est_imgs_subset; imgs_ = imgs_subset; } UMat pano_; #if ENABLE_LOG int64 t = getTickCount(); #endif std::vector<Point> corners(imgs_.size()); std::vector<UMat> masks_warped(imgs_.size()); std::vector<UMat> images_warped(imgs_.size()); std::vector<Size> sizes(imgs_.size()); std::vector<UMat> masks(imgs_.size()); // Prepare image masks for (size_t i = 0; i < imgs_.size(); ++i) { masks[i].create(seam_est_imgs_[i].size(), CV_8U); masks[i].setTo(Scalar::all(255)); } // Warp images and their masks Ptr<detail::TranslationWarper> w = warper_->create(float(warped_image_scale_ * seam_work_aspect_)); for (size_t i = 0; i < imgs_.size(); ++i) { Mat_<float> K; cameras_[i].K().convertTo(K, CV_32F); K(0, 0) *= (float) seam_work_aspect_; K(0, 2) *= (float) seam_work_aspect_; K(1, 1) *= (float) seam_work_aspect_; K(1, 2) *= (float) seam_work_aspect_; corners[i] = w->warp(seam_est_imgs_[i], K, cameras_[i].R, cameras_[i].t, INTER_LANCZOS4, BORDER_REFLECT, images_warped[i]); sizes[i] = images_warped[i].size(); w->warp(masks[i], K, cameras_[i].R, cameras_[i].t, INTER_NEAREST, BORDER_CONSTANT, masks_warped[i]); } std::vector<UMat> images_warped_f(imgs_.size()); for (size_t i = 0; i < imgs_.size(); ++i) images_warped[i].convertTo(images_warped_f[i], CV_32F); LOGLN("Warping images, time: " << ((getTickCount() - t) / getTickFrequency()) << " sec"); // Find seams exposure_comp_->feed(corners, images_warped, masks_warped); seam_finder_->find(images_warped_f, corners, masks_warped); // Release unused memory seam_est_imgs_.clear(); images_warped.clear(); images_warped_f.clear(); masks.clear(); LOGLN("Compositing..."); #if ENABLE_LOG t = getTickCount(); #endif UMat img_warped, img_warped_s; UMat dilated_mask, seam_mask, mask, mask_warped; //double compose_seam_aspect = 1; double compose_work_aspect = 1; bool is_blender_prepared = false; double compose_scale = 1; bool is_compose_scale_set = false; UMat full_img, img; for (size_t img_idx = 0; img_idx < imgs_.size(); ++img_idx) { LOGLN("Compositing image #" << indices_[img_idx] + 1); #if ENABLE_LOG int64 compositing_t = getTickCount(); #endif // Read image and resize it if necessary full_img = imgs_[img_idx]; if (!is_compose_scale_set) { if (compose_resol_ > 0) compose_scale = std::min(1.0, std::sqrt(compose_resol_ * 1e6 / full_img.size().area())); is_compose_scale_set = true; // Compute relative scales //compose_seam_aspect = compose_scale / seam_scale_; compose_work_aspect = compose_scale / work_scale_; // Update warped image scale warped_image_scale_ *= static_cast<float>(compose_work_aspect); w = warper_->create((float) warped_image_scale_); // Update corners and sizes for (size_t i = 0; i < imgs_.size(); ++i) { // Update intrinsics cameras_[i].focal *= compose_work_aspect; cameras_[i].ppx *= compose_work_aspect; cameras_[i].ppy *= compose_work_aspect; // Update corner and size Size sz = full_img_sizes_[i]; if (std::abs(compose_scale - 1) > 1e-1) { sz.width = cvRound(full_img_sizes_[i].width * compose_scale); sz.height = cvRound(full_img_sizes_[i].height * compose_scale); } Mat K; cameras_[i].K().convertTo(K, CV_32F); Rect roi = w->warpRoi(sz, K, cameras_[i].R, cameras_[i].t); corners[i] = roi.tl(); sizes[i] = roi.size(); } } if (std::abs(compose_scale - 1) > 1e-1) { #if ENABLE_LOG int64 resize_t = getTickCount(); #endif resize(full_img, img, Size(), compose_scale, compose_scale); LOGLN(" resize time: " << ((getTickCount() - resize_t) / getTickFrequency()) << " sec"); } else img = full_img; full_img.release(); Size img_size = img.size(); LOGLN(" after resize time: " << ((getTickCount() - compositing_t) / getTickFrequency()) << " sec"); Mat K; cameras_[img_idx].K().convertTo(K, CV_32F); #if ENABLE_LOG int64 pt = getTickCount(); #endif // Warp the current image w->warp(img, K, cameras_[img_idx].R, cameras_[img_idx].t, INTER_LANCZOS4, BORDER_REFLECT, img_warped); LOGLN(" warp the current image: " << ((getTickCount() - pt) / getTickFrequency()) << " sec"); #if ENABLE_LOG pt = getTickCount(); #endif // Warp the current image mask mask.create(img_size, CV_8U); mask.setTo(Scalar::all(255)); w->warp(mask, K, cameras_[img_idx].R, cameras_[img_idx].t, INTER_NEAREST, BORDER_CONSTANT, mask_warped); LOGLN(" warp the current image mask: " << ((getTickCount() - pt) / getTickFrequency()) << " sec"); #if ENABLE_LOG pt = getTickCount(); #endif // Compensate exposure exposure_comp_->apply((int) img_idx, corners[img_idx], img_warped, mask_warped); LOGLN(" compensate exposure: " << ((getTickCount() - pt) / getTickFrequency()) << " sec"); #if ENABLE_LOG pt = getTickCount(); #endif img_warped.convertTo(img_warped_s, CV_16S); img_warped.release(); img.release(); mask.release(); // Make sure seam mask has proper size dilate(masks_warped[img_idx], dilated_mask, Mat()); resize(dilated_mask, seam_mask, mask_warped.size()); bitwise_and(seam_mask, mask_warped, mask_warped); LOGLN(" other: " << ((getTickCount() - pt) / getTickFrequency()) << " sec"); #if ENABLE_LOG pt = getTickCount(); #endif if (!is_blender_prepared) { blender_->prepare(corners, sizes); is_blender_prepared = true; } LOGLN(" other2: " << ((getTickCount() - pt) / getTickFrequency()) << " sec"); LOGLN(" feed..."); #if ENABLE_LOG int64 feed_t = getTickCount(); #endif // Blend the current image blender_->feed(img_warped_s, mask_warped, corners[img_idx]); LOGLN(" feed time: " << ((getTickCount() - feed_t) / getTickFrequency()) << " sec"); LOGLN("Compositing ## time: " << ((getTickCount() - compositing_t) / getTickFrequency()) << " sec"); } #if ENABLE_LOG int64 blend_t = getTickCount(); #endif UMat result, result_mask; blender_->blend(result, result_mask); LOGLN("blend time: " << ((getTickCount() - blend_t) / getTickFrequency()) << " sec"); LOGLN("Compositing, time: " << ((getTickCount() - t) / getTickFrequency()) << " sec"); // Preliminary result is in CV_16SC3 format, but all values are in [0,255] range, // so convert it to avoid user confusing result.convertTo(pano, CV_8U); return OK; }