Stitcher::Status Stitcher::estimateTransform(InputArrayOfArrays images, InputArrayOfArrays masks)
{
CV_INSTRUMENT_REGION();
images.getUMatVector(imgs_);
masks.getUMatVector(masks_);
Status status;
if ((status = matchImages()) != OK)
return status;
if ((status = estimateCameraParams()) != OK)
return status;
return OK;
}
Mosaic::Status Mosaic::estimateTransform(InputArrayOfArrays images) {
images.getUMatVector(imgs_);
Status status;
if ((status = matchImages()) != OK)
return status;
if ((status = estimateCameraParams()) != OK)
return status;
return OK;
}
bool forward_ocl(InputArrayOfArrays inps, OutputArrayOfArrays outs, OutputArrayOfArrays internals)
{
std::vector<UMat> inputs;
std::vector<UMat> outputs;
inps.getUMatVector(inputs);
outs.getUMatVector(outputs);
int _layerWidth = inputs[0].size[3];
int _layerHeight = inputs[0].size[2];
int _imageWidth = inputs[1].size[3];
int _imageHeight = inputs[1].size[2];
float stepX, stepY;
if (_stepX == 0 || _stepY == 0)
{
stepX = static_cast<float>(_imageWidth) / _layerWidth;
stepY = static_cast<float>(_imageHeight) / _layerHeight;
} else {
stepX = _stepX;
stepY = _stepY;
}
if (umat_offsetsX.empty())
{
Mat offsetsX(1, _offsetsX.size(), CV_32FC1, &_offsetsX[0]);
Mat offsetsY(1, _offsetsX.size(), CV_32FC1, &_offsetsY[0]);
Mat aspectRatios(1, _aspectRatios.size(), CV_32FC1, &_aspectRatios[0]);
Mat variance(1, _variance.size(), CV_32FC1, &_variance[0]);
offsetsX.copyTo(umat_offsetsX);
offsetsY.copyTo(umat_offsetsY);
aspectRatios.copyTo(umat_aspectRatios);
variance.copyTo(umat_variance);
int real_numPriors = _numPriors >> (_offsetsX.size() - 1);
umat_scales = UMat(1, &real_numPriors, CV_32F, 1.0f);
}
bool forward_ocl(InputArrayOfArrays inps, OutputArrayOfArrays outs, OutputArrayOfArrays internals)
{
std::vector<UMat> inputs;
std::vector<UMat> outputs;
inps.getUMatVector(inputs);
outs.getUMatVector(outputs);
for (size_t i = 0; i < inputs.size(); i++)
{
UMat srcBlob = inputs[i];
void *src_handle = inputs[i].handle(ACCESS_READ);
void *dst_handle = outputs[i].handle(ACCESS_WRITE);
if (src_handle != dst_handle)
{
MatShape outShape = shape(outputs[i]);
UMat umat = srcBlob.reshape(1, (int)outShape.size(), &outShape[0]);
umat.copyTo(outputs[i]);
}
}
outs.assign(outputs);
return true;
}
bool forward_ocl(InputArrayOfArrays inps, OutputArrayOfArrays outs, OutputArrayOfArrays internals)
{
std::vector<UMat> inputs;
std::vector<UMat> outputs;
inps.getUMatVector(inputs);
outs.getUMatVector(outputs);
if (useSoftmaxTree) { // Yolo 9000
CV_Error(cv::Error::StsNotImplemented, "Yolo9000 is not implemented");
return false;
}
CV_Assert(inputs.size() >= 1);
int const cell_size = classes + coords + 1;
UMat blob_umat = blobs[0].getUMat(ACCESS_READ);
for (size_t ii = 0; ii < outputs.size(); ii++)
{
UMat& inpBlob = inputs[ii];
UMat& outBlob = outputs[ii];
int rows = inpBlob.size[1];
int cols = inpBlob.size[2];
ocl::Kernel logistic_kernel("logistic_activ", ocl::dnn::region_oclsrc);
size_t global = rows*cols*anchors;
logistic_kernel.set(0, (int)global);
logistic_kernel.set(1, ocl::KernelArg::PtrReadOnly(inpBlob));
logistic_kernel.set(2, (int)cell_size);
logistic_kernel.set(3, ocl::KernelArg::PtrWriteOnly(outBlob));
logistic_kernel.run(1, &global, NULL, false);
if (useSoftmax)
{
// Yolo v2
// softmax activation for Probability, for each grid cell (X x Y x Anchor-index)
ocl::Kernel softmax_kernel("softmax_activ", ocl::dnn::region_oclsrc);
size_t nthreads = rows*cols*anchors;
softmax_kernel.set(0, (int)nthreads);
softmax_kernel.set(1, ocl::KernelArg::PtrReadOnly(inpBlob));
softmax_kernel.set(2, ocl::KernelArg::PtrReadOnly(blob_umat));
softmax_kernel.set(3, (int)cell_size);
softmax_kernel.set(4, (int)classes);
softmax_kernel.set(5, (int)classfix);
softmax_kernel.set(6, (int)rows);
softmax_kernel.set(7, (int)cols);
softmax_kernel.set(8, (int)anchors);
softmax_kernel.set(9, (float)thresh);
softmax_kernel.set(10, ocl::KernelArg::PtrWriteOnly(outBlob));
if (!softmax_kernel.run(1, &nthreads, NULL, false))
return false;
}
if (nmsThreshold > 0) {
Mat mat = outBlob.getMat(ACCESS_WRITE);
float *dstData = mat.ptr<float>();
do_nms_sort(dstData, rows*cols*anchors, thresh, nmsThreshold);
}
}
return true;
}
bool forward_ocl(InputArrayOfArrays inps, OutputArrayOfArrays outs, OutputArrayOfArrays internals)
{
std::vector<UMat> inputs;
std::vector<UMat> outputs;
bool use_half = (inps.depth() == CV_16S);
inps.getUMatVector(inputs);
outs.getUMatVector(outputs);
int _layerWidth = inputs[0].size[3];
int _layerHeight = inputs[0].size[2];
int _imageWidth = inputs[1].size[3];
int _imageHeight = inputs[1].size[2];
if (umat_offsetsX.empty())
{
Mat offsetsX(1, _offsetsX.size(), CV_32FC1, &_offsetsX[0]);
Mat offsetsY(1, _offsetsY.size(), CV_32FC1, &_offsetsY[0]);
Mat variance(1, _variance.size(), CV_32FC1, &_variance[0]);
Mat widths(1, _boxWidths.size(), CV_32FC1, &_boxWidths[0]);
Mat heights(1, _boxHeights.size(), CV_32FC1, &_boxHeights[0]);
offsetsX.copyTo(umat_offsetsX);
offsetsY.copyTo(umat_offsetsY);
variance.copyTo(umat_variance);
widths.copyTo(umat_widths);
heights.copyTo(umat_heights);
}
String opts;
if (use_half)
opts = "-DDtype=half -DDtype4=half4 -Dconvert_T=convert_half4";
else
opts = "-DDtype=float -DDtype4=float4 -Dconvert_T=convert_float4";
size_t nthreads = _layerHeight * _layerWidth;
ocl::Kernel kernel("prior_box", ocl::dnn::prior_box_oclsrc, opts);
kernel.set(0, (int)nthreads);
kernel.set(1, (float)_stepX);
kernel.set(2, (float)_stepY);
kernel.set(3, ocl::KernelArg::PtrReadOnly(umat_offsetsX));
kernel.set(4, ocl::KernelArg::PtrReadOnly(umat_offsetsY));
kernel.set(5, (int)_offsetsX.size());
kernel.set(6, ocl::KernelArg::PtrReadOnly(umat_widths));
kernel.set(7, ocl::KernelArg::PtrReadOnly(umat_heights));
kernel.set(8, (int)_boxWidths.size());
kernel.set(9, ocl::KernelArg::PtrWriteOnly(outputs[0]));
kernel.set(10, (int)_layerHeight);
kernel.set(11, (int)_layerWidth);
kernel.set(12, (int)_imageHeight);
kernel.set(13, (int)_imageWidth);
kernel.run(1, &nthreads, NULL, false);
// clip the prior's coordidate such that it is within [0, 1]
if (_clip)
{
Mat mat = outputs[0].getMat(ACCESS_READ);
int aspect_count = (_maxSize > 0) ? 1 : 0;
int offset = nthreads * 4 * _offsetsX.size() * (1 + aspect_count + _aspectRatios.size());
float* outputPtr = mat.ptr<float>() + offset;
int _outChannelSize = _layerHeight * _layerWidth * _numPriors * 4;
for (size_t d = 0; d < _outChannelSize; ++d)
{
outputPtr[d] = std::min<float>(std::max<float>(outputPtr[d], 0.), 1.);
}
}
// set the variance.
{
ocl::Kernel kernel("set_variance", ocl::dnn::prior_box_oclsrc, opts);
int offset = total(shape(outputs[0]), 2);
size_t nthreads = _layerHeight * _layerWidth * _numPriors;
kernel.set(0, (int)nthreads);
kernel.set(1, (int)offset);
kernel.set(2, (int)_variance.size());
kernel.set(3, ocl::KernelArg::PtrReadOnly(umat_variance));
kernel.set(4, ocl::KernelArg::PtrWriteOnly(outputs[0]));
if (!kernel.run(1, &nthreads, NULL, false))
return false;
}
return true;
}
Mosaic::Status Mosaic::composePanorama(InputArrayOfArrays images, OutputArray pano) {
LOGLN("Warping images (auxiliary)... ");
std::vector<UMat> imgs;
images.getUMatVector(imgs);
if (!imgs.empty()) {
CV_Assert(imgs.size() == imgs_.size());
UMat img;
seam_est_imgs_.resize(imgs.size());
for (size_t i = 0; i < imgs.size(); ++i) {
imgs_[i] = imgs[i];
resize(imgs[i], img, Size(), seam_scale_, seam_scale_);
seam_est_imgs_[i] = img.clone();
}
std::vector<UMat> seam_est_imgs_subset;
std::vector<UMat> imgs_subset;
for (size_t i = 0; i < indices_.size(); ++i) {
imgs_subset.push_back(imgs_[indices_[i]]);
seam_est_imgs_subset.push_back(seam_est_imgs_[indices_[i]]);
}
seam_est_imgs_ = seam_est_imgs_subset;
imgs_ = imgs_subset;
}
UMat pano_;
#if ENABLE_LOG
int64 t = getTickCount();
#endif
std::vector<Point> corners(imgs_.size());
std::vector<UMat> masks_warped(imgs_.size());
std::vector<UMat> images_warped(imgs_.size());
std::vector<Size> sizes(imgs_.size());
std::vector<UMat> masks(imgs_.size());
// Prepare image masks
for (size_t i = 0; i < imgs_.size(); ++i) {
masks[i].create(seam_est_imgs_[i].size(), CV_8U);
masks[i].setTo(Scalar::all(255));
}
// Warp images and their masks
Ptr<detail::TranslationWarper> w = warper_->create(float(warped_image_scale_ * seam_work_aspect_));
for (size_t i = 0; i < imgs_.size(); ++i) {
Mat_<float> K;
cameras_[i].K().convertTo(K, CV_32F);
K(0, 0) *= (float) seam_work_aspect_;
K(0, 2) *= (float) seam_work_aspect_;
K(1, 1) *= (float) seam_work_aspect_;
K(1, 2) *= (float) seam_work_aspect_;
corners[i] = w->warp(seam_est_imgs_[i], K, cameras_[i].R, cameras_[i].t, INTER_LANCZOS4, BORDER_REFLECT,
images_warped[i]);
sizes[i] = images_warped[i].size();
w->warp(masks[i], K, cameras_[i].R, cameras_[i].t, INTER_NEAREST, BORDER_CONSTANT, masks_warped[i]);
}
std::vector<UMat> images_warped_f(imgs_.size());
for (size_t i = 0; i < imgs_.size(); ++i)
images_warped[i].convertTo(images_warped_f[i], CV_32F);
LOGLN("Warping images, time: " << ((getTickCount() - t) / getTickFrequency()) << " sec");
// Find seams
exposure_comp_->feed(corners, images_warped, masks_warped);
seam_finder_->find(images_warped_f, corners, masks_warped);
// Release unused memory
seam_est_imgs_.clear();
images_warped.clear();
images_warped_f.clear();
masks.clear();
LOGLN("Compositing...");
#if ENABLE_LOG
t = getTickCount();
#endif
UMat img_warped, img_warped_s;
UMat dilated_mask, seam_mask, mask, mask_warped;
//double compose_seam_aspect = 1;
double compose_work_aspect = 1;
bool is_blender_prepared = false;
double compose_scale = 1;
bool is_compose_scale_set = false;
UMat full_img, img;
for (size_t img_idx = 0; img_idx < imgs_.size(); ++img_idx) {
LOGLN("Compositing image #" << indices_[img_idx] + 1);
#if ENABLE_LOG
int64 compositing_t = getTickCount();
#endif
// Read image and resize it if necessary
full_img = imgs_[img_idx];
if (!is_compose_scale_set) {
if (compose_resol_ > 0)
compose_scale = std::min(1.0, std::sqrt(compose_resol_ * 1e6 / full_img.size().area()));
is_compose_scale_set = true;
// Compute relative scales
//compose_seam_aspect = compose_scale / seam_scale_;
compose_work_aspect = compose_scale / work_scale_;
// Update warped image scale
warped_image_scale_ *= static_cast<float>(compose_work_aspect);
w = warper_->create((float) warped_image_scale_);
// Update corners and sizes
for (size_t i = 0; i < imgs_.size(); ++i) {
// Update intrinsics
cameras_[i].focal *= compose_work_aspect;
cameras_[i].ppx *= compose_work_aspect;
cameras_[i].ppy *= compose_work_aspect;
// Update corner and size
Size sz = full_img_sizes_[i];
if (std::abs(compose_scale - 1) > 1e-1) {
sz.width = cvRound(full_img_sizes_[i].width * compose_scale);
sz.height = cvRound(full_img_sizes_[i].height * compose_scale);
}
Mat K;
cameras_[i].K().convertTo(K, CV_32F);
Rect roi = w->warpRoi(sz, K, cameras_[i].R, cameras_[i].t);
corners[i] = roi.tl();
sizes[i] = roi.size();
}
}
if (std::abs(compose_scale - 1) > 1e-1) {
#if ENABLE_LOG
int64 resize_t = getTickCount();
#endif
resize(full_img, img, Size(), compose_scale, compose_scale);
LOGLN(" resize time: " << ((getTickCount() - resize_t) / getTickFrequency()) << " sec");
} else
img = full_img;
full_img.release();
Size img_size = img.size();
LOGLN(" after resize time: " << ((getTickCount() - compositing_t) / getTickFrequency()) << " sec");
Mat K;
cameras_[img_idx].K().convertTo(K, CV_32F);
#if ENABLE_LOG
int64 pt = getTickCount();
#endif
// Warp the current image
w->warp(img, K, cameras_[img_idx].R, cameras_[img_idx].t, INTER_LANCZOS4, BORDER_REFLECT, img_warped);
LOGLN(" warp the current image: " << ((getTickCount() - pt) / getTickFrequency()) << " sec");
#if ENABLE_LOG
pt = getTickCount();
#endif
// Warp the current image mask
mask.create(img_size, CV_8U);
mask.setTo(Scalar::all(255));
w->warp(mask, K, cameras_[img_idx].R, cameras_[img_idx].t, INTER_NEAREST, BORDER_CONSTANT, mask_warped);
LOGLN(" warp the current image mask: " << ((getTickCount() - pt) / getTickFrequency()) << " sec");
#if ENABLE_LOG
pt = getTickCount();
#endif
// Compensate exposure
exposure_comp_->apply((int) img_idx, corners[img_idx], img_warped, mask_warped);
LOGLN(" compensate exposure: " << ((getTickCount() - pt) / getTickFrequency()) << " sec");
#if ENABLE_LOG
pt = getTickCount();
#endif
img_warped.convertTo(img_warped_s, CV_16S);
img_warped.release();
img.release();
mask.release();
// Make sure seam mask has proper size
dilate(masks_warped[img_idx], dilated_mask, Mat());
resize(dilated_mask, seam_mask, mask_warped.size());
bitwise_and(seam_mask, mask_warped, mask_warped);
LOGLN(" other: " << ((getTickCount() - pt) / getTickFrequency()) << " sec");
#if ENABLE_LOG
pt = getTickCount();
#endif
if (!is_blender_prepared) {
blender_->prepare(corners, sizes);
is_blender_prepared = true;
}
LOGLN(" other2: " << ((getTickCount() - pt) / getTickFrequency()) << " sec");
LOGLN(" feed...");
#if ENABLE_LOG
int64 feed_t = getTickCount();
#endif
// Blend the current image
blender_->feed(img_warped_s, mask_warped, corners[img_idx]);
LOGLN(" feed time: " << ((getTickCount() - feed_t) / getTickFrequency()) << " sec");
LOGLN("Compositing ## time: " << ((getTickCount() - compositing_t) / getTickFrequency()) << " sec");
}
#if ENABLE_LOG
int64 blend_t = getTickCount();
#endif
UMat result, result_mask;
blender_->blend(result, result_mask);
LOGLN("blend time: " << ((getTickCount() - blend_t) / getTickFrequency()) << " sec");
LOGLN("Compositing, time: " << ((getTickCount() - t) / getTickFrequency()) << " sec");
// Preliminary result is in CV_16SC3 format, but all values are in [0,255] range,
// so convert it to avoid user confusing
result.convertTo(pano, CV_8U);
return OK;
}
bool forward_ocl(InputArrayOfArrays inputs_, OutputArrayOfArrays outputs_, OutputArrayOfArrays internals_)
{
std::vector<UMat> inputs;
std::vector<UMat> outputs;
std::vector<UMat> internals;
inputs_.getUMatVector(inputs);
outputs_.getUMatVector(outputs);
internals_.getUMatVector(internals);
CV_Assert(inputs.size() == 1 && outputs.size() == 1);
CV_Assert(inputs[0].total() == outputs[0].total());
const UMat& inp0 = inputs[0];
UMat& buffer = internals[0];
size_t num = inp0.size[0];
size_t channels = inp0.size[1];
size_t channelSize = inp0.total() / (num * channels);
for (size_t i = 0; i < num; ++i)
{
MatShape s = shape(channels, channelSize);
UMat src = inputs[i].reshape(1, s.size(), &s[0]);
UMat dst = outputs[i].reshape(1, s.size(), &s[0]);
UMat abs_mat;
absdiff(src, cv::Scalar::all(0), abs_mat);
pow(abs_mat, pnorm, buffer);
if (acrossSpatial)
{
// add eps to avoid overflow
float absSum = sum(buffer)[0] + epsilon;
float norm = pow(absSum, 1.0f / pnorm);
multiply(src, 1.0f / norm, dst);
}
else
{
Mat norm;
reduce(buffer, norm, 0, REDUCE_SUM);
norm += epsilon;
// compute inverted norm to call multiply instead divide
cv::pow(norm, -1.0f / pnorm, norm);
repeat(norm, channels, 1, buffer);
multiply(src, buffer, dst);
}
if (!blobs.empty())
{
// scale the output
Mat scale = blobs[0];
if (scale.total() == 1)
{
// _scale: 1 x 1
multiply(dst, scale.at<float>(0, 0), dst);
}
else
{
// _scale: _channels x 1
CV_Assert(scale.total() == channels);
repeat(scale, 1, dst.cols, buffer);
multiply(dst, buffer, dst);
}
}
}
return true;
}
