Stitcher::Status Stitcher::estimateTransform(InputArrayOfArrays images, InputArrayOfArrays masks) { CV_INSTRUMENT_REGION(); images.getUMatVector(imgs_); masks.getUMatVector(masks_); Status status; if ((status = matchImages()) != OK) return status; if ((status = estimateCameraParams()) != OK) return status; return OK; }
Mosaic::Status Mosaic::estimateTransform(InputArrayOfArrays images) { images.getUMatVector(imgs_); Status status; if ((status = matchImages()) != OK) return status; if ((status = estimateCameraParams()) != OK) return status; return OK; }
bool forward_ocl(InputArrayOfArrays inps, OutputArrayOfArrays outs, OutputArrayOfArrays internals) { std::vector<UMat> inputs; std::vector<UMat> outputs; inps.getUMatVector(inputs); outs.getUMatVector(outputs); int _layerWidth = inputs[0].size[3]; int _layerHeight = inputs[0].size[2]; int _imageWidth = inputs[1].size[3]; int _imageHeight = inputs[1].size[2]; float stepX, stepY; if (_stepX == 0 || _stepY == 0) { stepX = static_cast<float>(_imageWidth) / _layerWidth; stepY = static_cast<float>(_imageHeight) / _layerHeight; } else { stepX = _stepX; stepY = _stepY; } if (umat_offsetsX.empty()) { Mat offsetsX(1, _offsetsX.size(), CV_32FC1, &_offsetsX[0]); Mat offsetsY(1, _offsetsX.size(), CV_32FC1, &_offsetsY[0]); Mat aspectRatios(1, _aspectRatios.size(), CV_32FC1, &_aspectRatios[0]); Mat variance(1, _variance.size(), CV_32FC1, &_variance[0]); offsetsX.copyTo(umat_offsetsX); offsetsY.copyTo(umat_offsetsY); aspectRatios.copyTo(umat_aspectRatios); variance.copyTo(umat_variance); int real_numPriors = _numPriors >> (_offsetsX.size() - 1); umat_scales = UMat(1, &real_numPriors, CV_32F, 1.0f); }
bool forward_ocl(InputArrayOfArrays inps, OutputArrayOfArrays outs, OutputArrayOfArrays internals) { std::vector<UMat> inputs; std::vector<UMat> outputs; inps.getUMatVector(inputs); outs.getUMatVector(outputs); for (size_t i = 0; i < inputs.size(); i++) { UMat srcBlob = inputs[i]; void *src_handle = inputs[i].handle(ACCESS_READ); void *dst_handle = outputs[i].handle(ACCESS_WRITE); if (src_handle != dst_handle) { MatShape outShape = shape(outputs[i]); UMat umat = srcBlob.reshape(1, (int)outShape.size(), &outShape[0]); umat.copyTo(outputs[i]); } } outs.assign(outputs); return true; }
bool forward_ocl(InputArrayOfArrays inps, OutputArrayOfArrays outs, OutputArrayOfArrays internals) { std::vector<UMat> inputs; std::vector<UMat> outputs; inps.getUMatVector(inputs); outs.getUMatVector(outputs); if (useSoftmaxTree) { // Yolo 9000 CV_Error(cv::Error::StsNotImplemented, "Yolo9000 is not implemented"); return false; } CV_Assert(inputs.size() >= 1); int const cell_size = classes + coords + 1; UMat blob_umat = blobs[0].getUMat(ACCESS_READ); for (size_t ii = 0; ii < outputs.size(); ii++) { UMat& inpBlob = inputs[ii]; UMat& outBlob = outputs[ii]; int rows = inpBlob.size[1]; int cols = inpBlob.size[2]; ocl::Kernel logistic_kernel("logistic_activ", ocl::dnn::region_oclsrc); size_t global = rows*cols*anchors; logistic_kernel.set(0, (int)global); logistic_kernel.set(1, ocl::KernelArg::PtrReadOnly(inpBlob)); logistic_kernel.set(2, (int)cell_size); logistic_kernel.set(3, ocl::KernelArg::PtrWriteOnly(outBlob)); logistic_kernel.run(1, &global, NULL, false); if (useSoftmax) { // Yolo v2 // softmax activation for Probability, for each grid cell (X x Y x Anchor-index) ocl::Kernel softmax_kernel("softmax_activ", ocl::dnn::region_oclsrc); size_t nthreads = rows*cols*anchors; softmax_kernel.set(0, (int)nthreads); softmax_kernel.set(1, ocl::KernelArg::PtrReadOnly(inpBlob)); softmax_kernel.set(2, ocl::KernelArg::PtrReadOnly(blob_umat)); softmax_kernel.set(3, (int)cell_size); softmax_kernel.set(4, (int)classes); softmax_kernel.set(5, (int)classfix); softmax_kernel.set(6, (int)rows); softmax_kernel.set(7, (int)cols); softmax_kernel.set(8, (int)anchors); softmax_kernel.set(9, (float)thresh); softmax_kernel.set(10, ocl::KernelArg::PtrWriteOnly(outBlob)); if (!softmax_kernel.run(1, &nthreads, NULL, false)) return false; } if (nmsThreshold > 0) { Mat mat = outBlob.getMat(ACCESS_WRITE); float *dstData = mat.ptr<float>(); do_nms_sort(dstData, rows*cols*anchors, thresh, nmsThreshold); } } return true; }
bool forward_ocl(InputArrayOfArrays inps, OutputArrayOfArrays outs, OutputArrayOfArrays internals) { std::vector<UMat> inputs; std::vector<UMat> outputs; bool use_half = (inps.depth() == CV_16S); inps.getUMatVector(inputs); outs.getUMatVector(outputs); int _layerWidth = inputs[0].size[3]; int _layerHeight = inputs[0].size[2]; int _imageWidth = inputs[1].size[3]; int _imageHeight = inputs[1].size[2]; if (umat_offsetsX.empty()) { Mat offsetsX(1, _offsetsX.size(), CV_32FC1, &_offsetsX[0]); Mat offsetsY(1, _offsetsY.size(), CV_32FC1, &_offsetsY[0]); Mat variance(1, _variance.size(), CV_32FC1, &_variance[0]); Mat widths(1, _boxWidths.size(), CV_32FC1, &_boxWidths[0]); Mat heights(1, _boxHeights.size(), CV_32FC1, &_boxHeights[0]); offsetsX.copyTo(umat_offsetsX); offsetsY.copyTo(umat_offsetsY); variance.copyTo(umat_variance); widths.copyTo(umat_widths); heights.copyTo(umat_heights); } String opts; if (use_half) opts = "-DDtype=half -DDtype4=half4 -Dconvert_T=convert_half4"; else opts = "-DDtype=float -DDtype4=float4 -Dconvert_T=convert_float4"; size_t nthreads = _layerHeight * _layerWidth; ocl::Kernel kernel("prior_box", ocl::dnn::prior_box_oclsrc, opts); kernel.set(0, (int)nthreads); kernel.set(1, (float)_stepX); kernel.set(2, (float)_stepY); kernel.set(3, ocl::KernelArg::PtrReadOnly(umat_offsetsX)); kernel.set(4, ocl::KernelArg::PtrReadOnly(umat_offsetsY)); kernel.set(5, (int)_offsetsX.size()); kernel.set(6, ocl::KernelArg::PtrReadOnly(umat_widths)); kernel.set(7, ocl::KernelArg::PtrReadOnly(umat_heights)); kernel.set(8, (int)_boxWidths.size()); kernel.set(9, ocl::KernelArg::PtrWriteOnly(outputs[0])); kernel.set(10, (int)_layerHeight); kernel.set(11, (int)_layerWidth); kernel.set(12, (int)_imageHeight); kernel.set(13, (int)_imageWidth); kernel.run(1, &nthreads, NULL, false); // clip the prior's coordidate such that it is within [0, 1] if (_clip) { Mat mat = outputs[0].getMat(ACCESS_READ); int aspect_count = (_maxSize > 0) ? 1 : 0; int offset = nthreads * 4 * _offsetsX.size() * (1 + aspect_count + _aspectRatios.size()); float* outputPtr = mat.ptr<float>() + offset; int _outChannelSize = _layerHeight * _layerWidth * _numPriors * 4; for (size_t d = 0; d < _outChannelSize; ++d) { outputPtr[d] = std::min<float>(std::max<float>(outputPtr[d], 0.), 1.); } } // set the variance. { ocl::Kernel kernel("set_variance", ocl::dnn::prior_box_oclsrc, opts); int offset = total(shape(outputs[0]), 2); size_t nthreads = _layerHeight * _layerWidth * _numPriors; kernel.set(0, (int)nthreads); kernel.set(1, (int)offset); kernel.set(2, (int)_variance.size()); kernel.set(3, ocl::KernelArg::PtrReadOnly(umat_variance)); kernel.set(4, ocl::KernelArg::PtrWriteOnly(outputs[0])); if (!kernel.run(1, &nthreads, NULL, false)) return false; } return true; }
Mosaic::Status Mosaic::composePanorama(InputArrayOfArrays images, OutputArray pano) { LOGLN("Warping images (auxiliary)... "); std::vector<UMat> imgs; images.getUMatVector(imgs); if (!imgs.empty()) { CV_Assert(imgs.size() == imgs_.size()); UMat img; seam_est_imgs_.resize(imgs.size()); for (size_t i = 0; i < imgs.size(); ++i) { imgs_[i] = imgs[i]; resize(imgs[i], img, Size(), seam_scale_, seam_scale_); seam_est_imgs_[i] = img.clone(); } std::vector<UMat> seam_est_imgs_subset; std::vector<UMat> imgs_subset; for (size_t i = 0; i < indices_.size(); ++i) { imgs_subset.push_back(imgs_[indices_[i]]); seam_est_imgs_subset.push_back(seam_est_imgs_[indices_[i]]); } seam_est_imgs_ = seam_est_imgs_subset; imgs_ = imgs_subset; } UMat pano_; #if ENABLE_LOG int64 t = getTickCount(); #endif std::vector<Point> corners(imgs_.size()); std::vector<UMat> masks_warped(imgs_.size()); std::vector<UMat> images_warped(imgs_.size()); std::vector<Size> sizes(imgs_.size()); std::vector<UMat> masks(imgs_.size()); // Prepare image masks for (size_t i = 0; i < imgs_.size(); ++i) { masks[i].create(seam_est_imgs_[i].size(), CV_8U); masks[i].setTo(Scalar::all(255)); } // Warp images and their masks Ptr<detail::TranslationWarper> w = warper_->create(float(warped_image_scale_ * seam_work_aspect_)); for (size_t i = 0; i < imgs_.size(); ++i) { Mat_<float> K; cameras_[i].K().convertTo(K, CV_32F); K(0, 0) *= (float) seam_work_aspect_; K(0, 2) *= (float) seam_work_aspect_; K(1, 1) *= (float) seam_work_aspect_; K(1, 2) *= (float) seam_work_aspect_; corners[i] = w->warp(seam_est_imgs_[i], K, cameras_[i].R, cameras_[i].t, INTER_LANCZOS4, BORDER_REFLECT, images_warped[i]); sizes[i] = images_warped[i].size(); w->warp(masks[i], K, cameras_[i].R, cameras_[i].t, INTER_NEAREST, BORDER_CONSTANT, masks_warped[i]); } std::vector<UMat> images_warped_f(imgs_.size()); for (size_t i = 0; i < imgs_.size(); ++i) images_warped[i].convertTo(images_warped_f[i], CV_32F); LOGLN("Warping images, time: " << ((getTickCount() - t) / getTickFrequency()) << " sec"); // Find seams exposure_comp_->feed(corners, images_warped, masks_warped); seam_finder_->find(images_warped_f, corners, masks_warped); // Release unused memory seam_est_imgs_.clear(); images_warped.clear(); images_warped_f.clear(); masks.clear(); LOGLN("Compositing..."); #if ENABLE_LOG t = getTickCount(); #endif UMat img_warped, img_warped_s; UMat dilated_mask, seam_mask, mask, mask_warped; //double compose_seam_aspect = 1; double compose_work_aspect = 1; bool is_blender_prepared = false; double compose_scale = 1; bool is_compose_scale_set = false; UMat full_img, img; for (size_t img_idx = 0; img_idx < imgs_.size(); ++img_idx) { LOGLN("Compositing image #" << indices_[img_idx] + 1); #if ENABLE_LOG int64 compositing_t = getTickCount(); #endif // Read image and resize it if necessary full_img = imgs_[img_idx]; if (!is_compose_scale_set) { if (compose_resol_ > 0) compose_scale = std::min(1.0, std::sqrt(compose_resol_ * 1e6 / full_img.size().area())); is_compose_scale_set = true; // Compute relative scales //compose_seam_aspect = compose_scale / seam_scale_; compose_work_aspect = compose_scale / work_scale_; // Update warped image scale warped_image_scale_ *= static_cast<float>(compose_work_aspect); w = warper_->create((float) warped_image_scale_); // Update corners and sizes for (size_t i = 0; i < imgs_.size(); ++i) { // Update intrinsics cameras_[i].focal *= compose_work_aspect; cameras_[i].ppx *= compose_work_aspect; cameras_[i].ppy *= compose_work_aspect; // Update corner and size Size sz = full_img_sizes_[i]; if (std::abs(compose_scale - 1) > 1e-1) { sz.width = cvRound(full_img_sizes_[i].width * compose_scale); sz.height = cvRound(full_img_sizes_[i].height * compose_scale); } Mat K; cameras_[i].K().convertTo(K, CV_32F); Rect roi = w->warpRoi(sz, K, cameras_[i].R, cameras_[i].t); corners[i] = roi.tl(); sizes[i] = roi.size(); } } if (std::abs(compose_scale - 1) > 1e-1) { #if ENABLE_LOG int64 resize_t = getTickCount(); #endif resize(full_img, img, Size(), compose_scale, compose_scale); LOGLN(" resize time: " << ((getTickCount() - resize_t) / getTickFrequency()) << " sec"); } else img = full_img; full_img.release(); Size img_size = img.size(); LOGLN(" after resize time: " << ((getTickCount() - compositing_t) / getTickFrequency()) << " sec"); Mat K; cameras_[img_idx].K().convertTo(K, CV_32F); #if ENABLE_LOG int64 pt = getTickCount(); #endif // Warp the current image w->warp(img, K, cameras_[img_idx].R, cameras_[img_idx].t, INTER_LANCZOS4, BORDER_REFLECT, img_warped); LOGLN(" warp the current image: " << ((getTickCount() - pt) / getTickFrequency()) << " sec"); #if ENABLE_LOG pt = getTickCount(); #endif // Warp the current image mask mask.create(img_size, CV_8U); mask.setTo(Scalar::all(255)); w->warp(mask, K, cameras_[img_idx].R, cameras_[img_idx].t, INTER_NEAREST, BORDER_CONSTANT, mask_warped); LOGLN(" warp the current image mask: " << ((getTickCount() - pt) / getTickFrequency()) << " sec"); #if ENABLE_LOG pt = getTickCount(); #endif // Compensate exposure exposure_comp_->apply((int) img_idx, corners[img_idx], img_warped, mask_warped); LOGLN(" compensate exposure: " << ((getTickCount() - pt) / getTickFrequency()) << " sec"); #if ENABLE_LOG pt = getTickCount(); #endif img_warped.convertTo(img_warped_s, CV_16S); img_warped.release(); img.release(); mask.release(); // Make sure seam mask has proper size dilate(masks_warped[img_idx], dilated_mask, Mat()); resize(dilated_mask, seam_mask, mask_warped.size()); bitwise_and(seam_mask, mask_warped, mask_warped); LOGLN(" other: " << ((getTickCount() - pt) / getTickFrequency()) << " sec"); #if ENABLE_LOG pt = getTickCount(); #endif if (!is_blender_prepared) { blender_->prepare(corners, sizes); is_blender_prepared = true; } LOGLN(" other2: " << ((getTickCount() - pt) / getTickFrequency()) << " sec"); LOGLN(" feed..."); #if ENABLE_LOG int64 feed_t = getTickCount(); #endif // Blend the current image blender_->feed(img_warped_s, mask_warped, corners[img_idx]); LOGLN(" feed time: " << ((getTickCount() - feed_t) / getTickFrequency()) << " sec"); LOGLN("Compositing ## time: " << ((getTickCount() - compositing_t) / getTickFrequency()) << " sec"); } #if ENABLE_LOG int64 blend_t = getTickCount(); #endif UMat result, result_mask; blender_->blend(result, result_mask); LOGLN("blend time: " << ((getTickCount() - blend_t) / getTickFrequency()) << " sec"); LOGLN("Compositing, time: " << ((getTickCount() - t) / getTickFrequency()) << " sec"); // Preliminary result is in CV_16SC3 format, but all values are in [0,255] range, // so convert it to avoid user confusing result.convertTo(pano, CV_8U); return OK; }
bool forward_ocl(InputArrayOfArrays inputs_, OutputArrayOfArrays outputs_, OutputArrayOfArrays internals_) { std::vector<UMat> inputs; std::vector<UMat> outputs; std::vector<UMat> internals; inputs_.getUMatVector(inputs); outputs_.getUMatVector(outputs); internals_.getUMatVector(internals); CV_Assert(inputs.size() == 1 && outputs.size() == 1); CV_Assert(inputs[0].total() == outputs[0].total()); const UMat& inp0 = inputs[0]; UMat& buffer = internals[0]; size_t num = inp0.size[0]; size_t channels = inp0.size[1]; size_t channelSize = inp0.total() / (num * channels); for (size_t i = 0; i < num; ++i) { MatShape s = shape(channels, channelSize); UMat src = inputs[i].reshape(1, s.size(), &s[0]); UMat dst = outputs[i].reshape(1, s.size(), &s[0]); UMat abs_mat; absdiff(src, cv::Scalar::all(0), abs_mat); pow(abs_mat, pnorm, buffer); if (acrossSpatial) { // add eps to avoid overflow float absSum = sum(buffer)[0] + epsilon; float norm = pow(absSum, 1.0f / pnorm); multiply(src, 1.0f / norm, dst); } else { Mat norm; reduce(buffer, norm, 0, REDUCE_SUM); norm += epsilon; // compute inverted norm to call multiply instead divide cv::pow(norm, -1.0f / pnorm, norm); repeat(norm, channels, 1, buffer); multiply(src, buffer, dst); } if (!blobs.empty()) { // scale the output Mat scale = blobs[0]; if (scale.total() == 1) { // _scale: 1 x 1 multiply(dst, scale.at<float>(0, 0), dst); } else { // _scale: _channels x 1 CV_Assert(scale.total() == channels); repeat(scale, 1, dst.cols, buffer); multiply(dst, buffer, dst); } } } return true; }