bool DatasetCIFAR10::loadBatch(std::string filename)
{
std::ifstream ifs(filename);
if (not ifs.good()) {
std::cerr << "(loadImages) ERROR cannot open file: " << filename << std::endl;
return false;
}
while (not ifs.eof()) {
char lbl = 0;
ifs.read(&lbl, 1);
char buffer[imgSize * 3];
ifs.read(buffer, imgSize * 3);
if (ifs.eof())
break;
// load channels, convert to float, normalize
cv::Size size(imgRows, imgCols);
std::vector<Mat> mats({Mat(size, CV_8UC1, buffer, Mat::AUTO_STEP),
Mat(size, CV_8UC1, buffer + imgSize, Mat::AUTO_STEP),
Mat(size, CV_8UC1, buffer + 2 * imgSize, Mat::AUTO_STEP)});
Mat img(size, CV_8UC3);
cv::merge(mats, img);
img.convertTo(img, CV_64FC3);
img = img / 255.0;
img = img.reshape(1, imgRows);
labels.push_back(lbl);
images.push_back(img);
}
size = images.size();
return true;
}
TEST(SupervisedDescentOptimiser, SinConvergence) {
// sin(x):
auto h = [](Mat value, size_t, int) { return std::sin(value.at<float>(0)); };
auto h_inv = [](float value) {
if (value >= 1.0f) // our upper border of y is 1.0f, but it can be a bit larger due to floating point representation. asin then returns NaN.
return std::asin(1.0f);
else
return std::asin(value);
};
float startInterval = -1.0f; float stepSize = 0.2f; int numValues = 11; Mat y_tr(numValues, 1, CV_32FC1); // sin: [-1:0.2:1]
{
vector<float> values(numValues);
strided_iota(std::begin(values), std::next(std::begin(values), numValues), startInterval, stepSize);
y_tr = Mat(values, true);
}
Mat x_tr(numValues, 1, CV_32FC1); // Will be the inverse of y_tr
{
vector<float> values(numValues);
std::transform(y_tr.begin<float>(), y_tr.end<float>(), begin(values), h_inv);
x_tr = Mat(values, true);
}
Mat x0 = 0.5f * Mat::ones(numValues, 1, CV_32FC1); // fixed initialization x0 = c = 0.5.
SupervisedDescentOptimiser<LinearRegressor<>> sdo({ LinearRegressor<>() });
// Test the callback mechanism as well: (better move to a separate unit test?)
auto checkResidual = [&](const Mat& currentX) {
double residual = cv::norm(currentX, x_tr, cv::NORM_L2) / cv::norm(x_tr, cv::NORM_L2);
EXPECT_DOUBLE_EQ(0.21369851877468238, residual);
};
sdo.train(x_tr, x0, y_tr, h, checkResidual);
// Make sure the training converges, i.e. the residual is correct on the training data:
Mat predictions = sdo.test(x0, y_tr, h);
double trainingResidual = normalisedLeastSquaresResidual(predictions, x_tr);
EXPECT_DOUBLE_EQ(0.21369851877468238, trainingResidual);
// Test the trained model:
// Test data with finer resolution:
float startIntervalTest = -1.0f; float stepSizeTest = 0.05f; int numValuesTest = 41; Mat y_ts(numValuesTest, 1, CV_32FC1); // sin: [-1:0.05:1]
{
vector<float> values(numValuesTest);
strided_iota(std::begin(values), std::next(std::begin(values), numValuesTest), startIntervalTest, stepSizeTest);
y_ts = Mat(values, true);
}
Mat x_ts_gt(numValuesTest, 1, CV_32FC1); // Will be the inverse of y_ts
{
vector<float> values(numValuesTest);
std::transform(y_ts.begin<float>(), y_ts.end<float>(), begin(values), h_inv);
x_ts_gt = Mat(values, true);
}
Mat x0_ts = 0.5f * Mat::ones(numValuesTest, 1, CV_32FC1); // fixed initialization x0 = c = 0.5.
predictions = sdo.test(x0_ts, y_ts, h);
double testResidual = normalisedLeastSquaresResidual(predictions, x_ts_gt);
ASSERT_NEAR(0.1800101229, testResidual, 0.0000000003);
}
TEST(SupervisedDescentOptimiser, SinConvergenceCascade) {
// sin(x):
auto h = [](Mat value, size_t, int) { return std::sin(value.at<float>(0)); };
auto h_inv = [](float value) {
if (value >= 1.0f) // our upper border of y is 1.0f, but it can be a bit larger due to floating point representation. asin then returns NaN.
return std::asin(1.0f);
else
return std::asin(value);
};
float startInterval = -1.0f; float stepSize = 0.2f; int numValues = 11; Mat y_tr(numValues, 1, CV_32FC1); // sin: [-1:0.2:1]
{
vector<float> values(numValues);
strided_iota(std::begin(values), std::next(std::begin(values), numValues), startInterval, stepSize);
y_tr = Mat(values, true);
}
Mat x_tr(numValues, 1, CV_32FC1); // Will be the inverse of y_tr
{
vector<float> values(numValues);
std::transform(y_tr.begin<float>(), y_tr.end<float>(), begin(values), h_inv);
x_tr = Mat(values, true);
}
Mat x0 = 0.5f * Mat::ones(numValues, 1, CV_32FC1); // fixed initialization x0 = c = 0.5.
vector<LinearRegressor<>> regressors(10);
SupervisedDescentOptimiser<LinearRegressor<>> sdo(regressors);
sdo.train(x_tr, x0, y_tr, h);
// Make sure the training converges, i.e. the residual is correct on the training data:
Mat predictions = sdo.test(x0, y_tr, h);
double trainingResidual = normalisedLeastSquaresResidual(predictions, x_tr);
EXPECT_NEAR(0.040279395, trainingResidual, 0.00000008);
// Test the trained model:
// Test data with finer resolution:
float startIntervalTest = -1.0f; float stepSizeTest = 0.05f; int numValuesTest = 41; Mat y_ts(numValuesTest, 1, CV_32FC1); // sin: [-1:0.05:1]
{
vector<float> values(numValuesTest);
strided_iota(std::begin(values), std::next(std::begin(values), numValuesTest), startIntervalTest, stepSizeTest);
y_ts = Mat(values, true);
}
Mat x_ts_gt(numValuesTest, 1, CV_32FC1); // Will be the inverse of y_ts
{
vector<float> values(numValuesTest);
std::transform(y_ts.begin<float>(), y_ts.end<float>(), begin(values), h_inv);
x_ts_gt = Mat(values, true);
}
Mat x0_ts = 0.5f * Mat::ones(numValuesTest, 1, CV_32FC1); // fixed initialization x0 = c = 0.5.
predictions = sdo.test(x0_ts, y_ts, h);
double testResidual = normalisedLeastSquaresResidual(predictions, x_ts_gt);
ASSERT_NEAR(0.026156775, testResidual, 0.00000005);
}
TEST(SupervisedDescentOptimiser, XCubeConvergenceCascade) {
// x^3:
auto h = [](Mat value, size_t, int) { return static_cast<float>(std::pow(value.at<float>(0), 3)); };
auto h_inv = [](float value) { return std::cbrt(value); }; // cubic root
float startInterval = -27.0f; float stepSize = 3.0f; int numValues = 19; Mat y_tr(numValues, 1, CV_32FC1); // cube: [-27:3:27]
{
vector<float> values(numValues);
strided_iota(std::begin(values), std::next(std::begin(values), numValues), startInterval, stepSize);
y_tr = Mat(values, true);
}
Mat x_tr(numValues, 1, CV_32FC1); // Will be the inverse of y_tr
{
vector<float> values(numValues);
std::transform(y_tr.begin<float>(), y_tr.end<float>(), begin(values), h_inv);
x_tr = Mat(values, true);
}
Mat x0 = 0.5f * Mat::ones(numValues, 1, CV_32FC1); // fixed initialization x0 = c = 0.5.
vector<LinearRegressor<>> regressors(10);
SupervisedDescentOptimiser<LinearRegressor<>> sdo(regressors);
sdo.train(x_tr, x0, y_tr, h);
// Make sure the training converges, i.e. the residual is correct on the training data:
Mat predictions = sdo.test(x0, y_tr, h);
double trainingResidual = normalisedLeastSquaresResidual(predictions, x_tr);
EXPECT_NEAR(0.04312725, trainingResidual, 0.00000002);
// Test the trained model:
// Test data with finer resolution:
float startIntervalTest = -27.0f; float stepSizeTest = 0.5f; int numValuesTest = 109; Mat y_ts(numValuesTest, 1, CV_32FC1); // cube: [-27:0.5:27]
{
vector<float> values(numValuesTest);
strided_iota(std::begin(values), std::next(std::begin(values), numValuesTest), startIntervalTest, stepSizeTest);
y_ts = Mat(values, true);
}
Mat x_ts_gt(numValuesTest, 1, CV_32FC1); // Will be the inverse of y_ts
{
vector<float> values(numValuesTest);
std::transform(y_ts.begin<float>(), y_ts.end<float>(), begin(values), h_inv);
x_ts_gt = Mat(values, true);
}
Mat x0_ts = 0.5f * Mat::ones(numValuesTest, 1, CV_32FC1); // fixed initialization x0 = c = 0.5.
predictions = sdo.test(x0_ts, y_ts, h);
double testResidual = normalisedLeastSquaresResidual(predictions, x_ts_gt);
ASSERT_NEAR(0.05889855, testResidual, 0.00000002);
}
TEST(SupervisedDescentOptimiser, ExpConvergenceCascade) {
// exp(x):
auto h = [](Mat value, size_t, int) { return std::exp(value.at<float>(0)); };
auto h_inv = [](float value) { return std::log(value); };
float startInterval = 1.0f; float stepSize = 3.0f; int numValues = 10; Mat y_tr(numValues, 1, CV_32FC1); // exp: [1:3:28]
{
vector<float> values(numValues);
strided_iota(std::begin(values), std::next(std::begin(values), numValues), startInterval, stepSize);
y_tr = Mat(values, true);
}
Mat x_tr(numValues, 1, CV_32FC1); // Will be the inverse of y_tr
{
vector<float> values(numValues);
std::transform(y_tr.begin<float>(), y_tr.end<float>(), begin(values), h_inv);
x_tr = Mat(values, true);
}
Mat x0 = 0.5f * Mat::ones(numValues, 1, CV_32FC1); // fixed initialization x0 = c = 0.5.
vector<LinearRegressor<>> regressors(10);
SupervisedDescentOptimiser<LinearRegressor<>> sdo(regressors);
sdo.train(x_tr, x0, y_tr, h);
// Make sure the training converges, i.e. the residual is correct on the training data:
Mat predictions = sdo.test(x0, y_tr, h);
double trainingResidual = normalisedLeastSquaresResidual(predictions, x_tr);
EXPECT_NEAR(0.02510868, trainingResidual, 0.00000005);
// Test the trained model:
// Test data with finer resolution:
float startIntervalTest = 1.0f; float stepSizeTest = 0.5f; int numValuesTest = 55; Mat y_ts(numValuesTest, 1, CV_32FC1); // exp: [1:0.5:28]
{
vector<float> values(numValuesTest);
strided_iota(std::begin(values), std::next(std::begin(values), numValuesTest), startIntervalTest, stepSizeTest);
y_ts = Mat(values, true);
}
Mat x_ts_gt(numValuesTest, 1, CV_32FC1); // Will be the inverse of y_ts
{
vector<float> values(numValuesTest);
std::transform(y_ts.begin<float>(), y_ts.end<float>(), begin(values), h_inv);
x_ts_gt = Mat(values, true);
}
Mat x0_ts = 0.5f * Mat::ones(numValuesTest, 1, CV_32FC1); // fixed initialization x0 = c = 0.5.
predictions = sdo.test(x0_ts, y_ts, h);
double testResidual = normalisedLeastSquaresResidual(predictions, x_ts_gt);
ASSERT_NEAR(0.01253494, testResidual, 0.00000004);
}
TEST(SupervisedDescentOptimiser, ErfConvergence) {
// erf(x):
auto h = [](Mat value, size_t, int) { return std::erf(value.at<float>(0)); };
auto h_inv = [](float value) { return boost::math::erf_inv(value); };
float startInterval = -0.99f; float stepSize = 0.11f; int numValues = 19; Mat y_tr(numValues, 1, CV_32FC1); // erf: [-0.99:0.11:0.99]
{
vector<float> values(numValues);
strided_iota(std::begin(values), std::next(std::begin(values), numValues), startInterval, stepSize);
y_tr = Mat(values, true);
}
Mat x_tr(numValues, 1, CV_32FC1); // Will be the inverse of y_tr
{
vector<float> values(numValues);
std::transform(y_tr.begin<float>(), y_tr.end<float>(), begin(values), h_inv);
x_tr = Mat(values, true);
}
Mat x0 = 0.5f * Mat::ones(numValues, 1, CV_32FC1); // fixed initialization x0 = c = 0.5.
SupervisedDescentOptimiser<LinearRegressor<>> sdo({ LinearRegressor<>() });
sdo.train(x_tr, x0, y_tr, h);
// Make sure the training converges, i.e. the residual is correct on the training data:
Mat predictions = sdo.test(x0, y_tr, h);
double trainingResidual = normalisedLeastSquaresResidual(predictions, x_tr);
EXPECT_NEAR(0.30944183, trainingResidual, 0.00000005);
// Test the trained model:
// Test data with finer resolution:
float startIntervalTest = -0.99f; float stepSizeTest = 0.03f; int numValuesTest = 67; Mat y_ts(numValuesTest, 1, CV_32FC1); // erf: [-0.99:0.03:0.99]
{
vector<float> values(numValuesTest);
strided_iota(std::begin(values), std::next(std::begin(values), numValuesTest), startIntervalTest, stepSizeTest);
y_ts = Mat(values, true);
}
Mat x_ts_gt(numValuesTest, 1, CV_32FC1); // Will be the inverse of y_ts
{
vector<float> values(numValuesTest);
std::transform(y_ts.begin<float>(), y_ts.end<float>(), begin(values), h_inv);
x_ts_gt = Mat(values, true);
}
Mat x0_ts = 0.5f * Mat::ones(numValuesTest, 1, CV_32FC1); // fixed initialization x0 = c = 0.5.
predictions = sdo.test(x0_ts, y_ts, h);
double testResidual = normalisedLeastSquaresResidual(predictions, x_ts_gt);
ASSERT_NEAR(0.25736006, testResidual, 0.0000002);
}
Point3d Camera::pixelToWorld(const Point2d& pixel) const
{
double im[2] = {pixel.x, pixel.y};
double wl[3];
double z;
// setup intrinsic and extrinsic parameters
CvMat img_frm = cvMat(1, 1, CV_64FC2, im);
Mat world_frm = Mat(T_ROWS, T_COLS, TYPE, wl);
// convert from distorted pixels to normalized camera frame
// cv:: version does not allow doubles for some odd reason,
// so use the C version
CvMat A = _A;
CvMat k = _k;
cvUndistortPoints(&img_frm, &img_frm, &A, &k);
// convert from camera frame to world frame
/*z = _t(2, 0) ? _t(2, 0) : 1; // Wrong z != t3. z = r31 X + r32 Y + t3
wl[0] = z*im[0] - _t(0, 0);
wl[1] = z*im[1] - _t(1, 0);
wl[2] = 0;
world_frm = _R.t() * world_frm;
*/
wl[0] = (_R(1,1) * _t(0,0)-_R(0,1) * _t(1,0)+_R(2,1) * _t(1,0) * im[0]- _R(1,1)* _t(2,0) * im[0]-_R(2,1)* _t(0,0)* im[1]+_R(0,1) * _t(2,0) * im[1])/(_R(0,1) * _R(1,0)-_R(0,0)* _R(1,1)+_R(1,1)* _R(2,0) *im[0]-_R(1,0) *_R(2,1) *im[0]-_R(0,1) *_R(2,0) * im[1]+_R(0,0) *_R(2,1) *im[1]);
wl[1] = (-_R(0,0) * _t(1,0)+_R(2,0)* _t(1,0)*im[0]+_R(1,0) *(_t(0,0)-_t(2,0)*im[0])-_R(2,0) * _t(0,0)*im[1]+_R(0,0) * _t(2,0) * im[1])/(-_R(0,1) * _R(1,0)+_R(0,0) * _R(1,1)-_R(1,1)*_R(2,0) *im[0]+_R(1,0) * _R(2,1) * im[0]+_R(0,1) * _R(2,0) * im[1]-_R(0,0) * _R(2,1) *im[1]);
wl[2] = 0;
return Point3d(wl[0], wl[1], wl[2]);
}
Mat* MyImage::getOpenCVMat(){
if(mat == NULL) {
try {
Mat orig;
getMagickImage();
magick.magick("BGR");
Blob blb ;
magick.write(&blb);
mat = new Mat();
orig = Mat(magick.size().height(),
magick.size().width(),
CV_8UC3,
(void *) blb.data());
cvtColor(orig, *mat, CV_RGB2HSV);
magick.magick("RGB");
}
catch (Magick::Exception &e)
{
cout << "Caught a magick exception: " << e.what() << endl;
throw e;
}
}
return mat;
}
/**
* Convenience function that fits the shape model and expression blendshapes to
* landmarks. Makes the fitted PCA shape and blendshape coefficients accessible
* via the out parameters \p pca_shape_coefficients and \p blendshape_coefficients.
* It iterates PCA-shape and blendshape fitting until convergence
* (usually it converges within 5 to 10 iterations).
*
* See fit_shape_model(cv::Mat, eos::morphablemodel::MorphableModel, std::vector<eos::morphablemodel::Blendshape>, std::vector<cv::Vec2f>, std::vector<int>, float lambda)
* for a simpler overload that just returns the shape instance.
*
* @param[in] affine_camera_matrix The estimated pose as a 3x4 affine camera matrix that is used to fit the shape.
* @param[in] morphable_model The 3D Morphable Model used for the shape fitting.
* @param[in] blendshapes A vector of blendshapes that are being fit to the landmarks in addition to the PCA model.
* @param[in] image_points 2D landmarks from an image to fit the model to.
* @param[in] vertex_indices The vertex indices in the model that correspond to the 2D points.
* @param[in] lambda Regularisation parameter of the PCA shape fitting.
* @param[out] pca_shape_coefficients Output parameter that will contain the resulting pca shape coefficients.
* @param[out] blendshape_coefficients Output parameter that will contain the resulting blendshape coefficients.
* @return The fitted model shape instance.
*/
cv::Mat fit_shape_model(cv::Mat affine_camera_matrix, eos::morphablemodel::MorphableModel morphable_model, std::vector<eos::morphablemodel::Blendshape> blendshapes, std::vector<cv::Vec2f> image_points, std::vector<int> vertex_indices, float lambda, std::vector<float>& pca_shape_coefficients, std::vector<float>& blendshape_coefficients)
{
using cv::Mat;
Mat blendshapes_as_basis(blendshapes[0].deformation.rows, blendshapes.size(), CV_32FC1); // assert blendshapes.size() > 0 and all of them have same number of rows, and 1 col
for (int i = 0; i < blendshapes.size(); ++i)
{
blendshapes[i].deformation.copyTo(blendshapes_as_basis.col(i));
}
std::vector<float> last_blendshape_coeffs, current_blendshape_coeffs;
std::vector<float> last_pca_coeffs, current_pca_coeffs;
current_blendshape_coeffs.resize(blendshapes.size()); // starting values t_0, all zeros
current_pca_coeffs.resize(morphable_model.get_shape_model().get_num_principal_components()); // starting values, all zeros
Mat combined_shape;
do // run at least once:
{
last_blendshape_coeffs = current_blendshape_coeffs;
last_pca_coeffs = current_pca_coeffs;
// Estimate the PCA shape coefficients with the current blendshape coefficients (0 in the first iteration):
Mat mean_plus_blendshapes = morphable_model.get_shape_model().get_mean() + blendshapes_as_basis * Mat(last_blendshape_coeffs);
current_pca_coeffs = fitting::fit_shape_to_landmarks_linear(morphable_model, affine_camera_matrix, image_points, vertex_indices, mean_plus_blendshapes, lambda);
// Estimate the blendshape coefficients with the current PCA model estimate:
Mat pca_model_shape = morphable_model.get_shape_model().draw_sample(current_pca_coeffs);
current_blendshape_coeffs = eos::fitting::fit_blendshapes_to_landmarks_linear(blendshapes, pca_model_shape, affine_camera_matrix, image_points, vertex_indices, 0.0f);
combined_shape = pca_model_shape + blendshapes_as_basis * Mat(current_blendshape_coeffs);
} while (std::abs(cv::norm(current_pca_coeffs) - cv::norm(last_pca_coeffs)) >= 0.01 || std::abs(cv::norm(current_blendshape_coeffs) - cv::norm(last_blendshape_coeffs)) >= 0.01);
pca_shape_coefficients = current_pca_coeffs;
blendshape_coefficients = current_blendshape_coeffs;
return combined_shape;
};
cv::Mat montageList(const Dataset &dataset, const std::vector<std::pair<int, int>> &list,
int tiles_per_row = 30)
{
using cv::Mat;
const int count = list.size();
const int xMargin = 2;
const int yMargin = 14;
const int width = dataset.imgCols + xMargin;
const int height = dataset.imgRows + yMargin;
assert(dataset.images.size() > 0);
const int type = dataset.images[0].type();
if (list.size() == 0)
return Mat(0, 0, type);
Mat mat = Mat::ones((count / tiles_per_row + 1) * height, tiles_per_row * width, type);
for (size_t i = 0; i < list.size(); i++) {
int x = (i % tiles_per_row) * width;
int y = (i / tiles_per_row) * height;
int id = list[i].first;
dataset.images[id].copyTo(mat(cv::Rect(x, y, dataset.imgCols, dataset.imgRows)));
std::string label =
std::to_string(list[i].second) + "/" + std::to_string(dataset.labels[id]);
cv::putText(mat, label, cv::Point(x, y + height - 2), cv::FONT_HERSHEY_SIMPLEX, 0.4,
cv::Scalar({0.0, 0.0, 0.0, 0.0}));
}
return mat;
}
void CaffeClassifier::Impl::FillBlob(const vector<Mat>& images,
Blobf* blob)
{
// Check that net is configured to use a proper batch size.
CV_Assert(static_cast<size_t>(data_blob->shape(0)) == images.size());
float* blob_data = blob->mutable_cpu_data();
for (size_t i = 0; i < images.size(); ++i)
{
Mat image = images[i];
// Check that all other dimentions of blob and image match.
CV_Assert(blob->shape(1) == image.channels());
CV_Assert(blob->shape(2) == image.rows);
CV_Assert(blob->shape(3) == image.cols);
Mat image_float = image;
if (image.type() != CV_32F) {
image.convertTo(image_float, CV_32F);
}
vector<Mat> image_channels;
for (int j = 0; j < image.channels(); ++j)
{
image_channels.push_back(Mat(image.size(), CV_32F,
blob_data + blob->offset(i, j)));
}
cv::split(image_float, image_channels);
}
}
/*
* Class: pt_chambino_p_pulse_Pulse_Face
* Method: _box
* Signature: (JJ)V
*/
JNIEXPORT void JNICALL Java_pt_chambino_p_pulse_Pulse_00024Face__1box
(JNIEnv *jenv, jclass, jlong self, jlong mat)
{
LOGD("Java_pt_chambino_p_pulse_Pulse_00024Face__1box enter");
try
{
if (self) {
vector<Rect> v;
v.push_back(((Pulse::Face*)self)->evm.box);
*((Mat*)mat) = Mat(v, true);
}
}
catch(cv::Exception& e)
{
jclass je = jenv->FindClass("org/opencv/core/CvException");
if(!je) je = jenv->FindClass("java/lang/Exception");
jenv->ThrowNew(je, e.what());
}
catch (...)
{
jclass je = jenv->FindClass("java/lang/Exception");
jenv->ThrowNew(je, "Unknown exception in JNI code.");
}
LOGD("Java_pt_chambino_p_pulse_Pulse_00024Face__1box exit");
}
static void transpose(Type* data, int height, int width) {
#if HAS_IMGLIB
int elemType;
if (sizeof(Type) == 1) {
elemType = CV_8UC1;
} else if (sizeof(Type) == 4) {
width /= 4;
elemType = CV_32F;
} else {
throw std::runtime_error("Unsupported type in transpose\n");
}
Mat input = Mat(height, width, elemType, data).clone();
Mat output = Mat(width, height, elemType, data);
cv::transpose(input, output);
#else
#warning ("OpenCV support not built-in")
string message = "OpenCV " UNSUPPORTED_MEDIA_MESSAGE;
throw std::runtime_error(message);
#endif
}
//计算直方图;
Mat KswSegment::histCalculate(const Mat& src)
{
Mat hist;
int bins = 256;
int histSize[] = {bins};
float range[] = {0, 256};
const float* ranges[] = {range};
int channels[] = {0};
calcHist(&src, 1, channels, Mat(), hist, 1, histSize, ranges, true, false);//计算原图像直方图
return hist;
}
bool Calibration::readExtrinsics(const string& file, Mat& R, Mat& t)
{
CvMat* _R = NULL;
CvMat* _t = NULL;
if(file.empty()) {
return false;
}
FileStorage fs(file, FileStorage::READ);
if(fs.isOpened()) {
_R = static_cast<CvMat*> (
fs["Rotation Matrix"].readObj());
_t = static_cast<CvMat*>
(fs["Translation Vector"].readObj());
}
R = _R == NULL ? Mat() : Mat(_R);
t = _t == NULL ? Mat() : Mat(_t);
return !R.empty() && !t.empty();
}
double Calibration::getExtrinsics(const Mat& Tr)
{
// set/create various parameters
Mat rvec;
Mat w(views.world.front());
Mat p(views.pixel.front());
Mat& A = intrinsic_params.A;
Mat& k = intrinsic_params.k;
Mat& R = extrinsic_params.R;
Mat& t = extrinsic_params.t;
// get extrinsic parameters
cv::solvePnP(w, p, A, k, rvec, t, solve_pnp.useExtGuess);
cv::Rodrigues(rvec, R);
// convert default coordinate system to new one
if(!Tr.empty()) {
CV_Assert(Tr.rows == 3 && Tr.cols == 4);
Mat T_Tr = Mat(Tr, Range(0, 3), Range(3, 4));
Mat R_Tr = Mat(Tr, Range(0, 3), Range(0, 3));
t += R*T_Tr;
R *= R_Tr;
}
/*std::cout << "you are printing modified R" << std::endl;
for(int i = 0; i < R.rows; i++) {
for(int j = 0; j < R.cols; j++) {
std::cout << R.at<double>(i,j) << " ";
}
std::cout << std::endl;
}*/
return 0.;
}
bool Calibration::readIntrinsics(const string& file, Mat& A, Mat& k)
{
CvMat* _A = NULL;
CvMat* _k = NULL;
if(file.empty()) {
return false;
}
FileStorage fs(file, FileStorage::READ);
if(fs.isOpened()) {
_A = static_cast<CvMat*> (
fs["Camera Matrix"].readObj());
_k = static_cast<CvMat*>
(fs["Distortion Coefficients"].readObj());
}
A = _A == NULL ? Mat() : Mat(_A);
k = _k == NULL ? Mat() : Mat(_k);
return !A.empty() && !k.empty();
}
void train(cv::Mat parameters, cv::Mat initialisations, cv::Mat templates, ProjectionFunction projection, OnTrainingEpochCallback onTrainingEpochCallback)
{
using cv::Mat;
Mat currentX = initialisations;
for (size_t regressorLevel = 0; regressorLevel < regressors.size(); ++regressorLevel) {
// 1) Project current parameters x to feature space:
// Enqueue all tasks in a thread pool:
auto concurentThreadsSupported = std::thread::hardware_concurrency();
if (concurentThreadsSupported == 0) {
concurentThreadsSupported = 4;
}
utils::ThreadPool threadPool(concurentThreadsSupported);
std::vector<std::future<typename std::result_of<ProjectionFunction(Mat, size_t, int)>::type>> results; // will be float or Mat. I might remove float for the sake of code clarity, as it's only useful for very simple examples.
results.reserve(currentX.rows);
for (int sampleIndex = 0; sampleIndex < currentX.rows; ++sampleIndex) {
results.emplace_back(
threadPool.enqueue(projection, currentX.row(sampleIndex), regressorLevel, sampleIndex)
);
}
// Gather the results from all threads and store the features:
Mat features;
for (auto&& result : results) {
features.push_back(result.get());
}
// Set the observed values, depending on if a template y is used:
Mat observedValues;
if (templates.empty()) { // unknown template training case
observedValues = features;
}
else { // known template
observedValues = features - templates;
}
Mat b = currentX - parameters; // currentX - x;
// 2) Learn using that data:
regressors[regressorLevel].learn(observedValues, b);
// 3) Apply the learned regressor and use the predictions to learn the next regressor in next loop iteration:
Mat x_k; // x_k = currentX - R * (h(currentX) - y):
for (int sampleIndex = 0; sampleIndex < currentX.rows; ++sampleIndex) {
// No need to re-extract the features, we already did so in step 1)
x_k.push_back(Mat(currentX.row(sampleIndex) - regressors[regressorLevel].predict(observedValues.row(sampleIndex))));
}
currentX = x_k;
onTrainingEpochCallback(currentX);
}
};
TEST(hole_filling_test, one_pixel_hole_on_random_image_should_produce_correct_target_rect)
{
// Make some random image data.
Mat img = Mat(100, 100, CV_32FC1);
randu(img, 0.f, 1.f);
// Put a hole in middle.
Mat hole = Mat::zeros(100, 100, CV_8U);
// Set one pixel as hole
hole(Rect(50, 50, 1, 1)) = 1;
int patch_size = 7;
HoleFilling hf(img, hole, patch_size);
Rect expected_target_rect(Point(50 - 6, 50 - 6), Point(50 + 7, 50 + 7));
ASSERT_EQ(expected_target_rect, hf._target_rect_pyr[0]);
}
// adjusts contrast, brightness, and saturation according
// to values in cbs[0], cbs[1], cbs[2], respectively
void cbsjitter(Mat& inout, float cbs[]) {
// Skip transformations if given deterministic settings
if (_params->_contrastMin == _params->_contrastMax) {
return;
}
/****************************
* BRIGHTNESS & SATURATION *
*****************************/
Mat satmtx = cbs[1] * (cbs[2] * Mat::eye(3, 3, CV_32FC1) +
(1 - cbs[2]) * Mat::ones(3, 1, CV_32FC1) * GSCL.t());
cv::transform(inout, inout, satmtx);
/*************
* CONTRAST *
**************/
Mat gray_mean;
cv::cvtColor(Mat(1, 1, CV_32FC3, cv::mean(inout)), gray_mean, CV_BGR2GRAY);
inout = cbs[0] * inout + (1 - cbs[0]) * gray_mean.at<Scalar_<float>>(0, 0);
}
cv::Mat test(cv::Mat initialisations, cv::Mat templates, ProjectionFunction projection, OnRegressorIterationCallback onRegressorIterationCallback)
{
using cv::Mat;
Mat currentX = initialisations;
for (size_t regressorLevel = 0; regressorLevel < regressors.size(); ++regressorLevel) {
// Enqueue all tasks in a thread pool:
auto concurentThreadsSupported = std::thread::hardware_concurrency();
if (concurentThreadsSupported == 0) {
concurentThreadsSupported = 4;
}
utils::ThreadPool threadPool(concurentThreadsSupported);
std::vector<std::future<typename std::result_of<ProjectionFunction(Mat, size_t, int)>::type>> results; // will be float or Mat. I might remove float for the sake of code clarity, as it's only useful for very simple examples.
results.reserve(currentX.rows);
for (int sampleIndex = 0; sampleIndex < currentX.rows; ++sampleIndex) {
results.emplace_back(
threadPool.enqueue(projection, currentX.row(sampleIndex), regressorLevel, sampleIndex)
);
}
// Gather the results from all threads and store the features:
Mat features;
for (auto&& result : results) {
features.push_back(result.get());
}
Mat observedValues;
if (templates.empty()) { // unknown template training case
observedValues = features;
}
else { // known template
observedValues = features - templates;
}
Mat x_k;
// Calculate x_k = currentX - R * (h(currentX) - y):
for (int sampleIndex = 0; sampleIndex < currentX.rows; ++sampleIndex) {
x_k.push_back(Mat(currentX.row(sampleIndex) - regressors[regressorLevel].predict(observedValues.row(sampleIndex)))); // we need Mat() because the subtraction yields a (non-persistent) MatExpr
}
currentX = x_k;
onRegressorIterationCallback(currentX);
}
return currentX; // Return the final predictions
};
void decode(char* item, int itemSize, char* buf) {
Mat image = Mat(1, itemSize, CV_8UC3, item);
Mat decodedImage = cv::imdecode(image, CV_LOAD_IMAGE_COLOR);
Rect cropBox;
_augParams->getRandomCrop(decodedImage.size(), &cropBox);
auto cropArea = cropBox.area();
auto innerSize = _augParams->getSize();
Mat croppedImage = decodedImage(cropBox);
// This would be more efficient, but we should allocate separate bufs for each thread
// Mat resizedImage = Mat(innerSize, CV_8UC3, _scratchbuf);
Mat resizedImage;
if (innerSize.width == cropBox.width && innerSize.height == cropBox.height) {
resizedImage = croppedImage;
} else {
int interp_method = cropArea < innerSize.area() ? CV_INTER_AREA : CV_INTER_CUBIC;
cv::resize(croppedImage, resizedImage, innerSize, 0, 0, interp_method);
}
Mat flippedImage;
Mat *finalImage;
if (_augParams->doRandomFlip()) {
cv::flip(resizedImage, flippedImage, 1);
finalImage = &flippedImage;
} else {
finalImage = &resizedImage;
}
Mat newImage;
float alpha = _augParams->getRandomContrast();
if (alpha) {
finalImage->convertTo(newImage, -1, alpha);
finalImage = &newImage;
}
Mat ch_b(innerSize, CV_8U, buf + innerSize.area()*0);
Mat ch_g(innerSize, CV_8U, buf + innerSize.area()*1);
Mat ch_r(innerSize, CV_8U, buf + innerSize.area()*2);
Mat channels[3] = {ch_b, ch_g, ch_r};
cv::split(*finalImage, channels);
}
void resizeInput(vector<char> &jpgdata, int maxDim) {
// Takes the buffer containing encoded jpg, determines if its shortest dimension
// is greater than maxDim. If so, it scales it down so that the shortest dimension
// is equal to maxDim. equivalent to "512x512^>" for maxDim=512 geometry argument in
// imagemagick
Mat image = Mat(1, jpgdata.size(), CV_8UC3, &jpgdata[0]);
Mat decodedImage = cv::imdecode(image, CV_LOAD_IMAGE_COLOR);
int minDim = std::min(decodedImage.rows, decodedImage.cols);
// If no resizing necessary, just return, original image still in jpgdata;
if (minDim <= maxDim)
return;
vector<int> param = {CV_IMWRITE_JPEG_QUALITY, 90};
double scaleFactor = (double) maxDim / (double) minDim;
Mat resizedImage;
cv::resize(decodedImage, resizedImage, Size2i(0, 0), scaleFactor, scaleFactor, CV_INTER_AREA);
cv::imencode(".jpg", resizedImage, *(reinterpret_cast<vector<uchar>*>(&jpgdata)), param);
// cv::imencode(".jpg", resizedImage, jpgdata, param);
return;
}
void FingerTracker::Setup()
{
VideoCapture capture(0);
if (!capture.isOpened())
{
throw std::runtime_error("Could not start camera capture");
}
int windowSize = 25;
int Xpos = 200;
int Ypos = 50;
int update = 0;
int buttonClicked = 0;
namedWindow("RGB", CV_WINDOW_AUTOSIZE);
createTrackbar("X", "RGB", &Xpos, 320, TrackBarCallback, (void*)&update);
createTrackbar("Y", "RGB", &Ypos, 240, TrackBarCallback, (void*)&update);
createTrackbar("Size", "RGB", &windowSize, 100, TrackBarCallback, (void*)&update);
setMouseCallback("RGB", MouseCallback, (void*)&buttonClicked);
Mat fingerWindowBackground, fingerWindowBackgroundGray;
m_calibrationData.reset(new CalibrationData());
bool ticking = false;
std::chrono::system_clock::time_point start = std::chrono::system_clock::now();
while (true)
{
Mat frame, frameHSV;
if (capture.read(frame))
{
flip(frame, frame, 1);
pyrDown(frame, frame, Size(frame.cols / 2, frame.rows / 2));
Rect fingerWindow(Point(Xpos, Ypos), Size(windowSize, windowSize*3));
if (Xpos + windowSize >= frame.cols || Ypos + windowSize*3 >= frame.rows)
{
windowSize = 20;
Xpos = 200;
Ypos = 50;
update = 0;
}
else if (buttonClicked == 1)
{
frame(fingerWindow).copyTo(fingerWindowBackground);
cvtColor(fingerWindowBackground, fingerWindowBackgroundGray, CV_BGR2GRAY);
buttonClicked = 0;
update = 0;
cvDestroyAllWindows();
}
if (fingerWindowBackgroundGray.rows && !m_calibrationData->m_ready)
{
Mat diff, thd;
absdiff(frame(fingerWindow), fingerWindowBackground, diff);
std::vector<Mat> ch;
split(diff, ch);
threshold(ch[0], ch[0], m_calibrationDiffThreshold, 255, 0);
threshold(ch[1], ch[1], m_calibrationDiffThreshold, 255, 0);
threshold(ch[2], ch[2], m_calibrationDiffThreshold, 255, 0);
thd = ch[0];
add(thd, ch[1], thd);
add(thd, ch[2], thd);
medianBlur(thd, thd, 5);
Mat top, middle, bottom;
Rect r1 = Rect(0, 0, thd.cols, thd.rows/3);
Rect r2 = Rect(0, thd.rows / 3 + 1, thd.cols, thd.rows/3);
Rect r3 = Rect(0, thd.rows * 2 / 3 + 1, thd.cols, thd.rows - thd.rows * 2 / 3 - 1);
top = thd(r1);
middle = thd(r2);
bottom = thd(r3);
auto percentageTop = countNonZero(top) * 100.0 / top.size().area();
auto percentageMiddle = countNonZero(middle) * 100.0 / middle.size().area();
auto percentageBottom = countNonZero(bottom) * 100.0 / bottom.size().area();
bool topReady = false;
bool middleReady = false;
bool bottomReady = false;
Scalar c1, c2, c3;
if (percentageTop > m_calibrationTopLowerThd && percentageTop < m_calibrationTopUpperThd)
{
topReady = true;
c1 = Scalar(0, 255, 255);
}
else
{
c1 = Scalar(0, 0, 255);
}
if (percentageMiddle > m_calibrationMiddleLowerThd && percentageMiddle < m_calibrationMiddleUppperThd)
{
middleReady = true;
c2 = Scalar(0, 255, 255);
}
else
{
c2 = Scalar(0, 0, 255);
}
if (percentageBottom > m_calibrationBottomLowerThd && percentageBottom < m_calibrationBottomUpperThd)
{
bottomReady = true;
c3 = Scalar(0, 255, 255);
}
else
{
c3 = Scalar(0, 0, 255);
}
bool readyToGo = false;
if (middleReady && topReady && bottomReady)
{
c1 = Scalar(0, 255, 0);
c2 = Scalar(0, 255, 0);
c3 = Scalar(0, 255, 0);
if (!ticking)
{
start = std::chrono::system_clock::now();
ticking = true;
}
if (std::chrono::system_clock::now() - start > std::chrono::seconds(1))
{
readyToGo = true;
}
}
else
{
ticking = false;
}
#if ENABLE_DEBUG_WINDOWS
std::stringstream ss;
ss << percentageTop << ", " << percentageMiddle << ", " << percentageBottom;
putText(frame, ss.str(), Point(0, getTextSize(ss.str(), 0, 0.5, 1, nullptr).height), 0, 0.5, Scalar::all(255), 1);
cv::imshow("Thresholded", thd);
#endif
if (percentageTop >= m_calibrationTopUpperThd && percentageBottom >= m_calibrationBottomUpperThd && percentageMiddle >= m_calibrationMiddleUppperThd)
{
putText(frame, "Move finger away from camera", Point(0, getTextSize("Move finger away from camera", 0, 0.5, 1, nullptr).height), 0, 0.5, Scalar::all(255), 1);
}
else if (percentageTop <= m_calibrationTopLowerThd && percentageBottom <= m_calibrationBottomLowerThd && percentageMiddle <= m_calibrationMiddleLowerThd)
{
putText(frame, "Move finger closer to camera", Point(0, getTextSize("Move finger closer to camera", 0, 0.5, 1, nullptr).height), 0, 0.5, Scalar::all(255), 1);
}
if (readyToGo)
{
Mat framePatchHSV;
cvtColor(frame(fingerWindow), framePatchHSV, CV_BGR2HSV);
cvtColor(frame, frameHSV, CV_BGR2HSV);
MatND hist;
calcHist(&framePatchHSV, 1, m_calibrationData->m_channels, thd, hist, 2, m_calibrationData->m_histSize, (const float**)m_calibrationData->m_ranges, true, false);
m_calibrationData->m_hist = hist;
normalize(m_calibrationData->m_hist, m_calibrationData->m_hist, 0, 255, NORM_MINMAX, -1, Mat());
#if ENABLE_DEBUG_WINDOWS
double maxVal=0;
minMaxLoc(m_calibrationData->m_hist, 0, &maxVal, 0, 0);
int scale = 10;
Mat histImg = Mat::zeros(m_calibrationData->m_sbins*scale, m_calibrationData->m_hbins*10, CV_8UC3);
for( int h = 0; h < m_calibrationData->m_hbins; h++)
{
for( int s = 0; s < m_calibrationData->m_sbins; s++ )
{
float binVal = m_calibrationData->m_hist.at<float>(h, s);
int intensity = cvRound(binVal*255/maxVal);
rectangle( histImg, Point(h*scale, s*scale),
Point( (h+1)*scale - 1, (s+1)*scale - 1),
Scalar::all(intensity),
CV_FILLED );
}
}
imshow("H-S Histogram", histImg);
#endif
m_calibrationData->m_ready = true;
frame(fingerWindow).copyTo(m_calibrationData->m_fingerPatch);
m_calibrationData->m_fingerRect = fingerWindow;
m_currentCandidate.m_windowRect = fingerWindow;
m_currentCandidate.m_fingerPosition = fingerWindow.tl();
return;
}
rectangle(frame, r1.tl() + fingerWindow.tl(), r1.br() + fingerWindow.tl(), c1);
rectangle(frame, r2.tl() + fingerWindow.tl(), r2.br() + fingerWindow.tl(), c2);
rectangle(frame, r3.tl() + fingerWindow.tl(), r3.br() + fingerWindow.tl(), c3);
imshow("Calibration", frame);
}
else
{
int baseline = 0;
putText(frame, "Adjust calibration window, click when ready", Point(0, getTextSize("Adjust calibration window", 0, 0.4, 2, &baseline).height), 0, 0.4, Scalar::all(255), 1);
rectangle(frame, fingerWindow.tl(), fingerWindow.br(), Scalar(0, 0, 255));
imshow("RGB", frame);
}
auto key = cvWaitKey(10);
if (char(key) == 27)
{
break;
}
}
}
capture.release();
}
void FingerTracker::Process(Mat frame)
{
#if ENABLE_DEBUG_WINDOWS
Mat img_display;
frame.copyTo(img_display);
#endif
// Process only Region of Interest i.e. region around current finger position
Rect roi = Rect(
Point(std::max(m_currentCandidate.m_windowRect.tl().x - m_roiSpanX, 0), std::max(m_currentCandidate.m_windowRect.tl().y - m_roiSpanY, 0)),
Point(std::min(m_currentCandidate.m_windowRect.tl().x + m_roiSpanX + m_calibrationData->m_fingerPatch.cols, frame.cols),
std::min(m_currentCandidate.m_windowRect.tl().y + m_roiSpanY + m_calibrationData->m_fingerPatch.rows, frame.rows)));
Mat frameRoi;
frame(roi).copyTo(frameRoi);
//================TEMPLATE MATCHING
int result_cols = frameRoi.cols - m_calibrationData->m_fingerPatch.cols + 1;
int result_rows = frameRoi.rows - m_calibrationData->m_fingerPatch.rows + 1;
assert(result_cols > 0 && result_rows > 0);
Mat scoreMap;
scoreMap.create(result_cols, result_rows, CV_32FC1);
// Compare current frame roi region to known candidate
// Using OpenCV matchTemplate function with correlation coefficient matching method
matchTemplate(frameRoi, m_calibrationData->m_fingerPatch, scoreMap, 3);
//================HISTOGRAM BACK PROJECTION
MatND backProjection;
Mat frameHSV;
cvtColor(frameRoi, frameHSV, CV_BGR2HSV);
calcBackProject(&frameHSV, 1, m_calibrationData->m_channels, m_calibrationData->m_hist, backProjection, (const float**)(m_calibrationData->m_ranges), 1, true);
Mat backProjectionThresholded;
threshold(backProjection, backProjectionThresholded, m_backProjectionThreshold, 255, 0);
erode(backProjectionThresholded, backProjectionThresholded, getStructuringElement(MORPH_RECT, Size(2 * m_erosionSize + 1, 2 * m_erosionSize + 1), Point(m_erosionSize, m_erosionSize)));
dilate(backProjectionThresholded, backProjectionThresholded, getStructuringElement(MORPH_RECT, Size(2 * m_dilationSize + 1, 2 * m_dilationSize + 1), Point(m_dilationSize, m_dilationSize)));
Mat backProjectionThresholdedShifted;
Rect shifted(Rect(m_calibrationData->m_fingerPatch.cols - 1, m_calibrationData->m_fingerPatch.rows - 1, scoreMap.cols, scoreMap.rows));
backProjectionThresholded(shifted).copyTo(backProjectionThresholdedShifted);
Mat maskedOutScoreMap(scoreMap.size(), CV_8U);
scoreMap.copyTo(maskedOutScoreMap, backProjectionThresholdedShifted);
//====================Localizing the best match with minMaxLoc
double minVal; double maxVal; Point minLoc; Point maxLoc;
Point matchLoc;
minMaxLoc(maskedOutScoreMap, &minVal, &maxVal, &minLoc, &maxLoc, Mat());
matchLoc = maxLoc + roi.tl();
m_currentCandidate.m_confidence = static_cast<float>(maxVal);
if (maxVal > m_candidateDetecionConfidenceThreshold)
{
m_currentCandidate.m_found = true;
m_currentCandidate.m_windowRect = Rect(matchLoc, Point(matchLoc.x + m_calibrationData->m_fingerPatch.cols , matchLoc.y + m_calibrationData->m_fingerPatch.rows));
//================Find finger position
Mat fingerWindowThresholded;
backProjectionThresholded(Rect(maxLoc, Point(maxLoc.x + m_calibrationData->m_fingerPatch.cols , maxLoc.y + m_calibrationData->m_fingerPatch.rows))).copyTo(fingerWindowThresholded);
m_currentCandidate.m_fingerPosition = GetFingerTopPosition(fingerWindowThresholded) + matchLoc;
#if ENABLE_DEBUG_WINDOWS
rectangle(img_display, m_currentCandidate.m_windowRect.tl(), m_currentCandidate.m_windowRect.br(), Scalar(255,0,0), 2, 8, 0 );
rectangle(scoreMap, m_currentCandidate.m_windowRect.tl(), m_currentCandidate.m_windowRect.br(), Scalar::all(0), 2, 8, 0 );
rectangle(img_display, m_currentCandidate.m_fingerPosition, m_currentCandidate.m_fingerPosition + Point(5,5), Scalar(255,0,0));
#endif
}
else
{
m_currentCandidate.m_found = false;
}
#if ENABLE_DEBUG_WINDOWS
std::stringstream ss;
ss << maxVal;
putText(img_display, ss.str(), Point(50, 50), 0, 0.5, Scalar(255,255,255), 1);
imshow("Overlays", img_display);
imshow("Results", scoreMap);
imshow("ResultMasked", maskedOutScoreMap);
#endif
}
void Calibration::reproject(Scalar pixel_color, Scalar reproj_color)
{
std::fstream filess ("extrinsicInfo.txt", fstream::out);
filess << "px py ProX ProY wX wY " <<std::endl;
stringstream ss;
size_t nimgs = views.imgs.size();
size_t npxs = views.pixel.size();
size_t nwls = views.world.size();
if(nimgs == npxs && nimgs == nwls) {
// setup parameters
Mat r;
cv::Rodrigues(extrinsic_params.R, r);
Mat& t = extrinsic_params.t;
Mat& A = intrinsic_params.A;
Mat& k = intrinsic_params.k;
vector<Point2f> pixel;
// iterate through each image
for(size_t i = 0; i < nimgs; ++i) {
// project from world to image frame
pixel.clear();
cv::projectPoints(Mat(views.world[i]), r, t, A, k, pixel);
// get original world and pixel values from image views
vector<Point3f>& w = views.world[i];
vector<Point2f>& p = views.pixel[i];
// get the image number in string format
ss.str("");
ss.seekp(0);
ss.seekg(0);
ss << "Image: " << (i + 1);
std::cout << ss.str() << std::endl;
for(size_t j = 0; j < views.pixel[i].size(); ++j) {
// iterate over each point
cv::circle(views.imgs[i], p[j], 5, pixel_color);
cv::circle(views.imgs[i], pixel[j], 3,
reproj_color, CV_FILLED);
std::cout << " px=" << p[j].x << " rx=" << pixel[j].x
<< " ex=" << p[j].x - pixel[j].x
<< " py=" << p[j].y << " ry=" << pixel[j].y
<< " ey=" << p[j].y - pixel[j].y << std::endl;
// Write original pixel, projected pixel and world to file
filess << p[j].x << " " << p[j].y << " " << pixel[j].x << " " << pixel[j].y << " " << w[j].x << " " << w[j].y << " "<< w[j].z << std::endl;
}
Mat diff;
cv::absdiff(Mat(p), Mat(pixel), diff);
Scalar mean, stddev;
cv::meanStdDev(diff, mean, stddev);
std::cout << "average error |(px,py) - (rx,ry)|=(" << mean[0] <<
"," << mean[1] << ")" << " +/- (" << stddev[0] <<
"," << stddev[1] << ")" << std::endl;
cv::imshow(ss.str(), views.imgs[i]);
cv::waitKey(1);
}
}
filess.close();
}
/**
* Renders the given mesh onto a 2D image using 4x4 model-view and
* projection matrices. Conforms to OpenGL conventions.
*
* @param[in] mesh A 3D mesh.
* @param[in] model_view_matrix A 4x4 OpenGL model-view matrix.
* @param[in] projection_matrix A 4x4 orthographic or perspective OpenGL projection matrix.
* @param[in] viewport_width Screen width.
* @param[in] viewport_height Screen height.
* @param[in] texture An optional texture map (TODO: Not optional yet!).
* @param[in] enable_backface_culling Whether the renderer should perform backface culling. If true, only draw triangles with vertices ordered CCW in screen-space.
* @param[in] enable_near_clipping Screen height.
* @param[in] enable_far_clipping Screen height.
* @return A pair with the colourbuffer as its first element and the depthbuffer as the second element.
*/
std::pair<cv::Mat, cv::Mat> render(Mesh mesh, cv::Mat model_view_matrix, cv::Mat projection_matrix, int viewport_width, int viewport_height, const Texture& texture, bool enable_backface_culling = false, bool enable_near_clipping = true, bool enable_far_clipping = true)
{
// Some internal documentation / old todos or notes:
// maybe change and pass depthBuffer as an optional arg (&?), because usually we never need it outside the renderer. Or maybe even a getDepthBuffer().
// modelViewMatrix goes to eye-space (camera space), projection does ortho or perspective proj.
// bool enable_texturing = false; Maybe re-add later, not sure
// take a cv::Mat texture instead and convert to Texture internally? no, we don't want to recreate mipmap levels on each render() call.
assert(mesh.vertices.size() == mesh.colors.size() || mesh.colors.empty()); // The number of vertices has to be equal for both shape and colour, or, alternatively, it has to be a shape-only model.
assert(mesh.vertices.size() == mesh.texcoords.size() || mesh.texcoords.empty()); // same for the texcoords
// another assert: If cv::Mat texture != empty, then we need texcoords?
using cv::Mat;
using std::vector;
Mat colourbuffer = Mat::zeros(viewport_height, viewport_width, CV_8UC4); // make sure it's CV_8UC4?
Mat depthbuffer = std::numeric_limits<float>::max() * Mat::ones(viewport_height, viewport_width, CV_64FC1);
// Vertex shader:
//processedVertex = shade(Vertex); // processedVertex : pos, col, tex, texweight
// Assemble the vertices, project to clip space, and store as detail::Vertex (the internal representation):
vector<detail::Vertex> clipspace_vertices;
clipspace_vertices.reserve(mesh.vertices.size());
for (int i = 0; i < mesh.vertices.size(); ++i) { // "previously": mesh.vertex
Mat clipspace_coords = projection_matrix * model_view_matrix * Mat(mesh.vertices[i]);
cv::Vec3f vertex_colour;
if (mesh.colors.empty()) {
vertex_colour = cv::Vec3f(0.5f, 0.5f, 0.5f);
}
else {
vertex_colour = mesh.colors[i];
}
clipspace_vertices.push_back(detail::Vertex(clipspace_coords, vertex_colour, mesh.texcoords[i]));
}
// All vertices are in clip-space now.
// Prepare the rasterisation stage.
// For every vertex/tri:
vector<detail::TriangleToRasterize> triangles_to_raster;
for (const auto& tri_indices : mesh.tvi) {
// Todo: Split this whole stuff up. Make a "clip" function, ... rename "processProspective..".. what is "process"... get rid of "continue;"-stuff by moving stuff inside process...
// classify vertices visibility with respect to the planes of the view frustum
// we're in clip-coords (NDC), so just check if outside [-1, 1] x ...
// Actually we're in clip-coords and it's not the same as NDC. We're only in NDC after the division by w.
// We should do the clipping in clip-coords though. See http://www.songho.ca/opengl/gl_projectionmatrix.html for more details.
// However, when comparing against w_c below, we might run into the trouble of the sign again in the affine case.
// 'w' is always positive, as it is -z_camspace, and all z_camspace are negative.
unsigned char visibility_bits[3];
for (unsigned char k = 0; k < 3; k++)
{
visibility_bits[k] = 0;
float x_cc = clipspace_vertices[tri_indices[k]].position[0];
float y_cc = clipspace_vertices[tri_indices[k]].position[1];
float z_cc = clipspace_vertices[tri_indices[k]].position[2];
float w_cc = clipspace_vertices[tri_indices[k]].position[3];
if (x_cc < -w_cc) // true if outside of view frustum. False if on or inside the plane.
visibility_bits[k] |= 1; // set bit if outside of frustum
if (x_cc > w_cc)
visibility_bits[k] |= 2;
if (y_cc < -w_cc)
visibility_bits[k] |= 4;
if (y_cc > w_cc)
visibility_bits[k] |= 8;
if (enable_near_clipping && z_cc < -w_cc) // near plane frustum clipping
visibility_bits[k] |= 16;
if (enable_far_clipping && z_cc > w_cc) // far plane frustum clipping
visibility_bits[k] |= 32;
} // if all bits are 0, then it's inside the frustum
// all vertices are not visible - reject the triangle.
if ((visibility_bits[0] & visibility_bits[1] & visibility_bits[2]) > 0)
{
continue;
}
// all vertices are visible - pass the whole triangle to the rasterizer. = All bits of all 3 triangles are 0.
if ((visibility_bits[0] | visibility_bits[1] | visibility_bits[2]) == 0)
{
boost::optional<detail::TriangleToRasterize> t = detail::process_prospective_tri(clipspace_vertices[tri_indices[0]], clipspace_vertices[tri_indices[1]], clipspace_vertices[tri_indices[2]], viewport_width, viewport_height, enable_backface_culling);
if (t) {
triangles_to_raster.push_back(*t);
}
continue;
}
// at this moment the triangle is known to be intersecting one of the view frustum's planes
std::vector<detail::Vertex> vertices;
vertices.push_back(clipspace_vertices[tri_indices[0]]);
vertices.push_back(clipspace_vertices[tri_indices[1]]);
vertices.push_back(clipspace_vertices[tri_indices[2]]);
// split the triangle if it intersects the near plane:
if (enable_near_clipping)
{
vertices = detail::clip_polygon_to_plane_in_4d(vertices, cv::Vec4f(0.0f, 0.0f, -1.0f, -1.0f)); // "Normal" (or "4D hyperplane") of the near-plane. I tested it and it works like this but I'm a little bit unsure because Songho says the normal of the near-plane is (0,0,-1,1) (maybe I have to switch around the < 0 checks in the function?)
}
// triangulation of the polygon formed of vertices array
if (vertices.size() >= 3)
{
for (unsigned char k = 0; k < vertices.size() - 2; k++)
{
boost::optional<detail::TriangleToRasterize> t = detail::process_prospective_tri(vertices[0], vertices[1 + k], vertices[2 + k], viewport_width, viewport_height, enable_backface_culling);
if (t) {
triangles_to_raster.push_back(*t);
}
}
}
}
// Fragment/pixel shader: Colour the pixel values
// for every tri:
for (const auto& tri : triangles_to_raster) {
detail::raster_triangle(tri, colourbuffer, depthbuffer, texture, enable_far_clipping);
}
return std::make_pair(colourbuffer, depthbuffer);
};
Mat get_L(Mat LR) {
int w = LR.size().width / 2;
int h = LR.size().height;
return Mat(LR, Rect(0, 0, w, h));
}
