vector<cv::Mat> ColorSystemConverter::rgb2cmyk(cv::Mat& img, std::vector<cv::Mat>& cmyk)
{
Mat channel[4];
for (int i = 0; i < 4; i++) {
cmyk.push_back(cv::Mat(img.size(), CV_8UC1));
channel[i] = Mat::eye(img.size(), CV_8UC1);
}
std::vector<cv::Mat> rgb, cmykColored;
cv::split(img, rgb);
for (int i = 0; i < img.rows; i++) {
for (int j = 0; j < img.cols; j++) {
float r = (int)rgb[2].at<uchar>(i, j) / 255.;
float g = (int)rgb[1].at<uchar>(i, j) / 255.;
float b = (int)rgb[0].at<uchar>(i, j) / 255.;
float k = std::min(std::min(1- r, 1- g), 1- b);
// cmyk[0].at<uchar>(i, j) = (1 - r - k) / (1 - k) * 255.;
// cmyk[1].at<uchar>(i, j) = (1 - g - k) / (1 - k) * 255.;
// cmyk[2].at<uchar>(i, j) = (1 - b - k) / (1 - k) * 255.;
// cmyk[3].at<uchar>(i, j) = k * 255.;
uchar ch = channel[0].at<uchar>(i, j);
channel[0].at<uchar>(i, j) = (1 - r - k) / (1 - k) * 255.;
channel[1].at<uchar>(i, j) = (1 - g - k) / (1 - k) * 255.;
channel[2].at<uchar>(i, j) = (1 - b - k) / (1 - k) * 255.;
channel[3].at<uchar>(i, j) = k * 255.;
}
}
bitwise_not(channel[0], channel[0]); //что бы было как в фотошопе
bitwise_not(channel[1], channel[1]);
bitwise_not(channel[2], channel[2]);
bitwise_not(channel[3], channel[3]);
cmyk[0] = channel[0].clone();
cmyk[1] = channel[1].clone();
cmyk[2] = channel[2].clone();
cmyk[3] = channel[3].clone();
std::string cmyk_labels[3] = {"[C]yan", "[M]agenta", "[Y]ellow"};
double std_values[3][3] = {{255, 255, 0}, {255, 0, 255}, {0, 255, 255}};
cmykColored = showChannels(img, channel, cmyk_labels, std_values, COLOR_HSV2BGR, true);
return cmykColored;
}
vector<Mat> produceThresholds(const Mat img_gray, Config* config)
{
const int THRESHOLD_COUNT = 4;
//Mat img_equalized = equalizeBrightness(img_gray);
timespec startTime;
getTime(&startTime);
vector<Mat> thresholds;
for (int i = 0; i < THRESHOLD_COUNT; i++)
thresholds.push_back(Mat(img_gray.size(), CV_8U));
int i = 0;
// Adaptive
//adaptiveThreshold(img_gray, thresholds[i++], 255, ADAPTIVE_THRESH_MEAN_C, THRESH_BINARY_INV , 7, 3);
//adaptiveThreshold(img_gray, thresholds[i++], 255, ADAPTIVE_THRESH_MEAN_C, THRESH_BINARY_INV , 13, 3);
//adaptiveThreshold(img_gray, thresholds[i++], 255, ADAPTIVE_THRESH_MEAN_C, THRESH_BINARY_INV , 17, 3);
// Wolf
int k = 0, win=18;
//NiblackSauvolaWolfJolion (img_gray, thresholds[i++], WOLFJOLION, win, win, 0.05 + (k * 0.35));
//bitwise_not(thresholds[i-1], thresholds[i-1]);
NiblackSauvolaWolfJolion (img_gray, thresholds[i++], WOLFJOLION, win, win, 0.05 + (k * 0.35));
bitwise_not(thresholds[i-1], thresholds[i-1]);
k = 1;
win = 22;
NiblackSauvolaWolfJolion (img_gray, thresholds[i++], WOLFJOLION, win, win, 0.05 + (k * 0.35));
bitwise_not(thresholds[i-1], thresholds[i-1]);
//NiblackSauvolaWolfJolion (img_gray, thresholds[i++], WOLFJOLION, win, win, 0.05 + (k * 0.35));
//bitwise_not(thresholds[i-1], thresholds[i-1]);
// Sauvola
k = 1;
NiblackSauvolaWolfJolion (img_gray, thresholds[i++], SAUVOLA, 12, 12, 0.18 * k);
bitwise_not(thresholds[i-1], thresholds[i-1]);
k=2;
NiblackSauvolaWolfJolion (img_gray, thresholds[i++], SAUVOLA, 12, 12, 0.18 * k);
bitwise_not(thresholds[i-1], thresholds[i-1]);
if (config->debugTiming)
{
timespec endTime;
getTime(&endTime);
cout << " -- Produce Threshold Time: " << diffclock(startTime, endTime) << "ms." << endl;
}
return thresholds;
//threshold(img_equalized, img_threshold, 100, 255, THRESH_BINARY);
}
void LbpMrf::process(const cv::Mat &img_input, cv::Mat &img_output, cv::Mat &img_bgmodel)
{
if(img_input.empty())
return;
loadConfig();
if(firstTime)
{
saveConfig();
}
IplImage TempImage(img_input);
MEImage InputImage(img_input.cols, img_input.rows, img_input.channels());
MEImage OutputImage(img_input.cols, img_input.rows, img_input.channels());
InputImage.SetIplImage((void*)&TempImage);
Detector->DetectMotions(InputImage);
Detector->GetMotionsMask(OutputImage);
img_output = (IplImage*)OutputImage.GetIplImage();
bitwise_not(img_output, img_bgmodel);
if(showOutput)
{
cv::imshow("LBP-MRF FG", img_output);
cv::imshow("LBP-MRF BG", img_bgmodel);
}
firstTime = false;
}
Mat computeWhiteMaskShadow(Mat& img)
{
// I = rgbFrame(:,:,1)>80 & rgbFrame(:,:,3)>80 | rgbFrame(:,:,3)>80 & abs(double(rgbFrame(:,:,1))-double(rgbFrame(:,:,3)))<20;
// I = medfilt2(I,[20,20]);
Mat BGRbands[3];
split(img,BGRbands);
Mat maskB, maskG, maskR, maskD, maskT, mask;
vector< vector<Point> > contours;
threshold(BGRbands[0],maskB,90,255,THRESH_BINARY);
threshold(BGRbands[1],maskG,90,255,THRESH_BINARY);
threshold(BGRbands[2],maskR,90,255,THRESH_BINARY);
absdiff(BGRbands[2],BGRbands[0],maskD);
threshold(maskD,maskD,25,255,THRESH_BINARY);
bitwise_not(maskD,maskD);
bitwise_and(maskR,maskD,maskD);
bitwise_and(maskB,maskR,maskT);
bitwise_or(maskD,maskT,maskT);
findContours(maskT, contours, CV_RETR_CCOMP, CV_CHAIN_APPROX_SIMPLE);
vector<double> areas = computeArea(contours);
for(int j = areas.size()-1; j>=0; j--){
if(areas.at(j)>MAX_AREA || areas.at(j)<MIN_AREA )
contours.erase(contours.begin()+j);
}
Mat out = Mat::zeros(Size(img.cols,img.rows), CV_8U);
for (int idx = 0; idx < contours.size(); idx++)
drawContours(out, contours, idx, Scalar(255,255,255), CV_FILLED, 8);
return out;
}
double get_skew_angle(Mat img)
{
// Binarize
threshold(img, img, 225, 255, THRESH_BINARY);
// Invert colors
bitwise_not(img, img);
Mat element = getStructuringElement(MORPH_RECT, Size(5, 3));
erode(img, img, element);
vector<Point> points;
Mat_<uchar>::iterator it = img.begin<uchar>();
Mat_<uchar>::iterator end = img.end<uchar>();
for (; it != end; ++it)
if (*it)
points.push_back(it.pos());
RotatedRect box = minAreaRect(Mat(points));
double angle = box.angle;
if (angle < -45.)
angle += 90.;
Point2f vertices[4];
box.points(vertices);
for(int i = 0; i < 4; ++i)
line(img, vertices[i], vertices[(i + 1) % 4], Scalar(255, 0, 0), 1, CV_AA);
return angle;
}
std::tuple<cv::Mat, cv::Mat, signed int, signed int> FreeTypeTextPrinter::GetCharMat() const
{
// bitmap_left为左距 bitmap_top为图形顶部到基线的距离基线
auto &bitmap = ft_face_->glyph->bitmap;
// 就地构造 免除手动内存拷贝 注意，生成下一个字符时内存会被清除
cv::Mat char_bitmap(cv::Size(bitmap.width, bitmap.rows), CV_8U, bitmap.buffer);
cv::Mat char_img;
// 处理非打印字符问题
if (0 != char_bitmap.rows || 0 != char_bitmap.cols)
{
char_img = char_bitmap.clone();
}
else
{
char_img = cv::Mat(cv::Size(std::max(bitmap.width, empty_width_), std::max(bitmap.rows, empty_width_)), CV_8U, cv::Scalar(0));
}
cvtColor(char_img, char_img, CV_GRAY2BGR);
cv::Mat mask; // 填涂掩码 用来挖除目标图像的掩码区域以便去除字体色彩不饱和导致字体模糊的副作用
bitwise_not(char_img, mask);
LUT(char_img, color_lut_, char_img);
return std::make_tuple(char_img, mask, ft_face_->glyph->bitmap_top, ft_face_->glyph->bitmap_left);
}
void edgeDetect(Mat& img) {
GaussianBlur(img, img, Size(3, 3), 0, 0);
Mat tmp;
img.copyTo(tmp);
Canny(img, img, 30, 100, 3, true);
bitwise_not(img, img);
erode(img, img, getStructuringElement(MORPH_ELLIPSE, Size(dilation_size + 1, dilation_size + 1), Point(dilation_size, dilation_size)));
if(loaded) {
std::vector<Rect> faces;
cvtColor(tmp, tmp, COLOR_BGR2GRAY );
equalizeHist( tmp, tmp);
face_cascade.detectMultiScale(tmp, faces, 1.1, 2, 0, Size(80, 80));
for( size_t i = 0; i < faces.size(); i++ ) {
Mat faceROI = tmp(faces[i]);
Laplacian(faceROI, faceROI, CV_16S, 3, 2.5, 1, BORDER_DEFAULT);
threshold(faceROI, faceROI, 7, 255, THRESH_BINARY_INV);
convertScaleAbs(faceROI, faceROI);
dilate(faceROI, faceROI, getStructuringElement(MORPH_ELLIPSE, Size(2, 2), Point(1, 1)));
faceROI.copyTo(img(faces[i]));
rectangle(img, faces[i], Scalar( 255, 190, 0 ));
}
}
cvtColor(img, img, CV_GRAY2BGR);
}
/* Other functions */
cv::Mat RemoveSmallRegionsAlgNode::processImage(cv::Mat image)
{
Mat src_gray, res, threshold_output;
vector<vector<Point> > contours;
int thresh = 10;
cvtColor( image, src_gray, CV_BGR2GRAY );
/// Detect edges using Threshold
threshold( src_gray, threshold_output, thresh, 255, THRESH_BINARY );
/// Find contours
findContours( threshold_output, contours, CV_RETR_EXTERNAL, CV_CHAIN_APPROX_NONE, Point(0, 0) );
Mat mask = Mat::zeros( threshold_output.size(), CV_8UC3 );
Scalar color_white = Scalar( 255, 255, 255 );
for( size_t i = 0; i< contours.size(); i++ )
{
if (contours[i].size() < min_area_threshold) {
drawContours( mask, contours, i, color_white, CV_FILLED);
}
}
bitwise_not(mask,mask);
//cv::imshow(WINDOW, mask);
//cv::waitKey(3);
bitwise_and(image,mask,image);
return image;
}
Mat deskew(Mat img, double angle)
{
bitwise_not(img, img);
vector<Point> points;
Mat_<uchar>::iterator it = img.begin<uchar>();
Mat_<uchar>::iterator end = img.end<uchar>();
for (; it != end; ++it)
if (*it)
points.push_back(it.pos());
RotatedRect box = minAreaRect(Mat(points));
Mat rot_mat = getRotationMatrix2D(box.center, angle, 1);
Mat rotated;
warpAffine(img, rotated, rot_mat, img.size(), INTER_CUBIC);
Size box_size = box.size;
if (box.angle < -45.)
swap(box_size.width, box_size.height);
Mat cropped;
getRectSubPix(rotated, box_size, box.center, cropped);
return cropped;
}
RegionMask RegionMask::operator-(const RegionMask &other) const{
RegionMask n(this->clone());
RegionMask temp(other.clone());
bitwise_not(temp,temp);
bitwise_and(n, temp, n);
Mask2QPolygon(n, n.boundary, n.holes);
return n;
}
void edgeDetectCanny(Mat& img) {
//GaussianBlur(img, img, Size(5,5), 0, 0);
bilateralSatured(img);
Canny(img, img, 30, 100, 3, true);
bitwise_not(img, img);
erode(img, img, getStructuringElement(MORPH_ELLIPSE, Size(dilation_size + 1, dilation_size + 1), Point(dilation_size, dilation_size)));
cvtColor(img, img, CV_GRAY2BGR);
}
double PyramidTemplateMatcher::findBest(const MatchingData& data, Rect* roi, Mat& out_result, Point& out_location){
TimingBlock t("PyramidTemplateMatcher::findBest");
double out_score;
Mat source;
if(roi != NULL)
source = data.getSource()(*roi);
else
source = data.getSource();
const Mat& target = data.getTarget();
#ifdef ENABLE_GPU
if(_use_gpu){
gpu::GpuMat gSource, gTarget;
gSource.upload(source);
gTarget.upload(target);
gpu::matchTemplate(gSource,gTarget,gResult,CV_TM_CCOEFF_NORMED);
gpu::minMaxLoc(gResult, NULL, &out_score, NULL, &out_location);
return out_score;
}
#endif
if(data.isSameColor()){ // pure color target
source = data.getOrigSource();
if(roi != NULL)
source = source(*roi);
if(data.isBlack()){ // black target
Mat inv_source, inv_target;
bitwise_not(source, inv_source);
bitwise_not(data.getOrigTarget(), inv_target);
matchTemplate(inv_source, inv_target, out_result, CV_TM_SQDIFF_NORMED);
}
else{
matchTemplate(source, data.getOrigTarget(), out_result, CV_TM_SQDIFF_NORMED);
}
result = Mat::ones(out_result.size(), CV_32F) - result;
}
else{
matchTemplate(source, target, out_result, CV_TM_CCOEFF_NORMED);
}
minMaxLoc(result, NULL, &out_score, NULL, &out_location);
return out_score;
}
//get position of center of eye
CvPoint2D32f getEyeCenter(Mat &face_frame, Rect eye){
//crop eye to get rid of some noise
int crop_y = 10;
eye.y += crop_y;
eye.height -= crop_y * 2;
//eye.width -= 20;
Mat eye_color = face_frame(eye);
Mat eye_scelra;
//convert to HSV and get saturation channel
cvtColor(eye_color, eye_scelra, CV_RGB2HSV);
std::vector<Mat> hsvChannels(3);
split(eye_scelra, hsvChannels);
eye_scelra = hsvChannels[1].mul(hsvChannels[2]) / 30;
//invert
bitwise_not(eye_scelra, eye_scelra);
//blur
blur(eye_scelra, eye_scelra, Size(4,4));
//apply histogram equalization
equalizeHist(eye_scelra, eye_scelra);
//threshold
//threshold(eye_scelra, eye_scelra, 10, 255, THRESH_BINARY_INV); //threshold type 3, thesh. to 0
//calc center of mass
float sum = 0;
float sum_x = 0;
float sum_y = 0;
for (int y = 0; y < eye_scelra.rows; ++y)
{
uchar* row = eye_scelra.ptr<uchar>(y);
for (int x = 0; x < eye_scelra.cols; ++x)
{
sum += row[x];
sum_x += row[x]*x;
sum_y += row[x]*y;
}
}
CvPoint2D32f max = cvPoint2D32f(sum_x/sum, sum_y/sum);
//circle(eye_scelra, max, 3, 0);
//imshow("eye", eye_scelra);
//adjust for crop
max.y += crop_y;
return max;
}
void TargetExtractor::movementDetect2(int threshold, double learningRate)
{
Mat gray, temp, background;
cvtColor(mFrame, gray, CV_BGR2GRAY);
if (mBackground.empty()) {
gray.convertTo(mBackground, CV_64F);
}
mBackground.convertTo(background, CV_8U);
absdiff(background, gray, mMask);
cv::threshold(mMask, mMask, threshold, 255, THRESH_BINARY);
bitwise_not(mMask, temp);
accumulateWeighted(gray, mBackground, learningRate, temp);
}
void canny_find_edges(const cv::Mat &frame, cv::Mat &result, double canny_min, double canny_max, int ksize, int8_t bg_color) {
/** Simply canny algorithm based configurable Find Edges function.
*
* @param canny_min -> first threshold for the hysteresis procedure.
* @param canny_max -> second threshold for the hysteresis procedure.
*
* @param ksize -> aperture size for the Sobel operator.
*
* @param bg_color = -1 ; // background color black.
* @param bg_color = 0 ; // background color gray.
* @param bg_color = 1 ; // background color white.
*
*******************************************************************/
cv::Mat canny_output ;
cv::Canny(frame, canny_output, canny_min, canny_max, ksize) ;
switch (bg_color) {
case -1 :
break ;
case 0 :
canny_output.convertTo(canny_output, CV_8U, 1.0, 127) ;
break ;
case 1 :
bitwise_not(canny_output, canny_output) ;
break ;
default :
// Raise an exception.
fprintf(stderr,"Error argument %s(..., bg_color = -1 | 0 | 1)\nDetermine the background color:\nbg_color = -1 -> black ;\nbg_color = 0 -> gray ;\nbg_color = 1 white\n", __func__) ;
result = frame ;
return ;
}
cvtColor(canny_output, canny_output, cv::COLOR_GRAY2BGR) ;
canny_output.copyTo(result) ;
}
/**
* @brief Gray code pattern generator
* @param imSize Size of the desired generated image
* @param level Gray code bit position level (number of lines)
* @param direction Direction of lines, HORIZONTAL or VERTICAL
* @param inverted Value inversion flag (currently not implemented)
* @return Gray code pattern image of size imSize
*/
Mat ProcGen::getPattern(Size imSize, uint level, uint direction, bool inverted)
{
int x = imSize.width;
int y = imSize.height;
int nextX = pow(2,ceil(log(x)/log(2)));
int nextY = pow(2,ceil(log(y)/log(2)));
int sqSize;
if(nextX > nextY)
sqSize = nextX;
else
sqSize = nextY;
//printf("Generating square version w/ size %d\n",sqSize);
Mat workImg = Mat::zeros(Size(sqSize,sqSize),CV_8UC1); // Create zero image
uint lineWidth, iterator, itEnd;
if(direction == HORIZONTAL)
lineWidth = sqSize/pow(2,level);
else
lineWidth = sqSize/pow(2,level);
if(level >= 2)
itEnd = pow(2,level-2);
else
itEnd = 1;
//printf("Using linewidth %d\n",lineWidth);
for(iterator = 0; iterator < itEnd; iterator++)
{
// Draw white bars
uint start = lineWidth * (1 + iterator*4);
uint end = start + 2*lineWidth;
//printf("Drawing bar from :%d to %d\n",start,end);
drawLine(workImg, start, end, direction);
}
//printf("Number of white bars: %d\n",itEnd);
// Crop
Mat retImg;
retImg = workImg(Rect(0,0,x,y));
if(inverted)
{
//invertImage(retImg, retImg);
bitwise_not(retImg, retImg); // Invert
}
return retImg;
}
CBlobResult computeWhiteMaskOtsu(Mat& imgRGBin, Mat& imgHSVIn, CBlobResult& blobs, int limitRGB, int limitHSV, double RGBratio, double HSVratio, int bmin, int bmax, int i){
waitKey(30);
Mat BGRbands[3];
split(imgRGBin,BGRbands);
Mat imgHSV;
cvtColor(imgHSVIn,imgHSV,CV_BGR2HSV);
Mat HSVbands[3];
split(imgHSV,HSVbands);
Mat maskHSV, maskRGB, maskT;
int otsuTRGB = getThreshVal_Otsu_8u(BGRbands[2]);
do{
threshold(BGRbands[2],maskRGB,otsuTRGB,255,THRESH_BINARY);
otsuTRGB++;
}while(countNonZero(maskRGB)>(RGBratio*limitRGB) & otsuTRGB<=255);
int otsuTHSV = getThreshVal_Otsu_8u(HSVbands[1]);
do{
threshold(HSVbands[1],maskHSV,otsuTHSV,255,THRESH_BINARY_INV);
otsuTHSV--;
}while(countNonZero(maskHSV)>(HSVratio*limitHSV) & otsuTHSV>=0); // 0.1
bitwise_or(maskHSV,maskRGB,maskT);
int blobSizeBefore = blobs.GetNumBlobs();
blobs = blobs + CBlobResult( maskT ,Mat(),8);
blobs.Filter( blobs, B_EXCLUDE, CBlobGetLength(), B_GREATER, bmax );
blobs.Filter( blobs, B_EXCLUDE, CBlobGetLength(), B_LESS, bmin );
int blobSizeAfter = blobs.GetNumBlobs();
Mat newMask(maskT.size(),maskT.type());
newMask.setTo(0);
for(;i<blobs.GetNumBlobs();i++){
double area = blobs.GetBlob(i)->Area();
if(area < 5000 && area > 400)
blobs.GetBlob(i)->FillBlob(newMask,CV_RGB(255,255,255),0,0,true);
}
if(countNonZero(maskRGB)>400 && countNonZero(maskHSV)>400 && blobSizeBefore!=blobSizeAfter){
vector<Mat> BGRbands; split(imgRGBin,BGRbands);
Mat maskedRGB = applyMaskBandByBand(newMask,BGRbands);
bitwise_not(newMask,newMask);
split(imgHSVIn,BGRbands);
Mat maskedHSV = applyMaskBandByBand(newMask,BGRbands);
blobs = computeWhiteMaskOtsu(maskedRGB, maskedHSV, blobs, countNonZero(maskRGB),countNonZero(maskHSV),RGBratio, HSVratio, bmin, bmax, i-1);
}
return blobs;
}
bool OpenCVWidget::showImage(const cv::Mat &image)
{
if ( !(image.channels() == 1 || image.channels() == 3) ) return false;
if (this->isVisible()) {
cv_frame = image;
_imgRatio = (float) cv_frame.cols / (float) cv_frame.rows;
if( select_object && selection.width > 0 && selection.height > 0 )
{
qDebug() << "Selection Pos (x1,y1) : " << selection.x << ", " << selection.y << endl;
// qDebug() << "Selection Size (x2,y2) : " << selection.x+selection.width << ", " << selection.y+selection.height << endl;
roi = cv::Mat(cv_frame, selection);
bitwise_not(roi, roi);
}
updateGL();
}
return true;
}
/**
* Efeito de pintura.
*/
void edgeDetectSobel(Mat& img) {
Mat gray, temp;
GaussianBlur(img, img, Size(3, 3), 0, 0);
cvtColor(img, gray, CV_RGB2GRAY);
//x
Sobel(gray, img, CV_16S, 1, 0, 3, 1.5, 0.3, BORDER_DEFAULT);
convertScaleAbs(img, img);
//y
Sobel(gray, temp, CV_16S, 0, 1, 3, 1.5, 0.4, BORDER_DEFAULT);
convertScaleAbs(temp, temp);
addWeighted(img, .3, temp, .3, .3, img);
erode(img, img,
getStructuringElement(MORPH_ELLIPSE,
Size(dilation_size + 1, dilation_size + 1),
Point(dilation_size, dilation_size)));
bitwise_not(img, img);
cvtColor(img, img, CV_GRAY2BGR);
}
void FieldLineDetector::createThresholdedImg(Mat& src)
{
vector<vector<cv::Point> > cont;
vector<vector<cv::Point> > contPoly;
vector<Vec4i> hierarchy;
Mat imgBwInv;
imgThres = threshold(src);
// find large contours and remove them from thresholded image
bitwise_not(imgThres, imgBwInv);
findContours(imgBwInv, cont, hierarchy, RETR_TREE, CHAIN_APPROX_SIMPLE,
cv::Point(0, 0));
for (int i = 0; i < cont.size(); i++)
{
double area = contourArea(cont[i]);
if (area > 100000)
{
contPoly.push_back(cont[i]);
}
}
fillPoly(imgThres, contPoly, Scalar(255, 255, 255));
imshow("thres", imgThres);
}
void CalibrateThread::run()
{
Size boardSize, imageSize;
float squareSize = 1.f, aspectRatio = 1.f;
Mat cameraMatrix, distCoeffs;
//QString of = ui->lineEdit_WorkFolder->text() + '/' + ui->lineEdit_OutputName->text();
QByteArray ba = strFileName.toLatin1();
const char* outputFilename = ba.data();
int i, nframes = 0;
bool writeExtrinsics = true, writePoints = true;
bool undistortImage = false;
int flags = 0;
VideoCapture capture;
bool flipVertical = false;
bool showUndistorted = false;
int delay = 1000;
clock_t prevTimestamp = 0;
int mode = CAPTURING;
vector<vector<Point2f> > imagePoints;
vector<string> imageList;
Pattern pattern = CHESSBOARD;
boardSize.width = m_width;
boardSize.height = m_height;
squareSize = m_squaresize;
//ui->textEdit_Information->append("\nCalibrating... Please wait for a while\n");
if( imgList.size() == 0 )
{
//QMessageBox::warning(NULL, "Error", "Please choose a right folder");
emit popupErrorInformation("Please choose a right folder");
emit closeImageWindow();
return;
}
else
{
nframes = imgList.size();
}
emit appendText("\nCalibrating... Please wait for a while\n");
//namedWindow( "Image View", 1 );
//bDialog->show();
for(i = 0; i < nframes ;i++)
{
//ui->textEdit_Information->append("Processing the image No. " + QString::number(i + 1));
emit appendText("Processing the image No. " + QString::number(i + 1));
Mat view, viewGray;
bool blink = false;
qDebug(imgList.at(i).toLatin1().data());
if( i < (int)imgList.size() )
view = imread(imgList.at(i).toLatin1().data(), 1);
if(!view.data)
{
//QMessageBox::warning(NULL, "Error", );
emit popupErrorInformation("Could not open image files");
return;
}
imageSize = view.size();
if( flipVertical )
flip( view, view, 0 );
vector<Point2f> pointbuf;
cvtColor(view, viewGray, CV_BGR2GRAY);
bool found;
switch( pattern )
{
case CHESSBOARD:
found = findChessboardCorners( view, boardSize, pointbuf,
CV_CALIB_CB_ADAPTIVE_THRESH | CV_CALIB_CB_FAST_CHECK | CV_CALIB_CB_NORMALIZE_IMAGE);
break;
case CIRCLES_GRID:
found = findCirclesGrid( view, boardSize, pointbuf );
break;
case ASYMMETRIC_CIRCLES_GRID:
found = findCirclesGrid( view, boardSize, pointbuf, CALIB_CB_ASYMMETRIC_GRID );
break;
}
// improve the found corners' coordinate accuracy
if( pattern == CHESSBOARD && found) cornerSubPix( viewGray, pointbuf, Size(11,11),
Size(-1,-1), TermCriteria( CV_TERMCRIT_EPS+CV_TERMCRIT_ITER, 30, 0.1 ));
if( mode == CAPTURING && found &&
(!capture.isOpened() || clock() - prevTimestamp > delay*1e-3*CLOCKS_PER_SEC) )
{
imagePoints.push_back(pointbuf);
prevTimestamp = clock();
blink = capture.isOpened();
}
if(found)
drawChessboardCorners( view, boardSize, Mat(pointbuf), found );
string msg = mode == CAPTURING ? "100/100" :
mode == CALIBRATED ? "Calibrated" : "Press 'g' to start";
int baseLine = 0;
Size textSize = getTextSize(msg, 1, 1, 1, &baseLine);
Point textOrigin(view.cols - 2*textSize.width - 10, view.rows - 2*baseLine - 10);
if( mode == CAPTURING )
{
if(undistortImage)
msg = format( "%d/%d Undist", (int)imagePoints.size(), nframes );
else
msg = format( "%d/%d", (int)imagePoints.size(), nframes );
}
putText( view, msg, textOrigin, 1, 1,
mode != CALIBRATED ? Scalar(0,0,255) : Scalar(0,255,0));
if( blink )
bitwise_not(view, view);
if( mode == CALIBRATED && undistortImage )
{
Mat temp = view.clone();
undistort(temp, view, cameraMatrix, distCoeffs);
}
Mat rgb;
cvtColor(view, rgb, CV_BGR2RGB);
QImage image32 = QImage(rgb.cols, rgb.rows, QImage::Format_RGB32);
QRgb value;
for(int r = 0; r < rgb.rows; r++)
{
for(int c = 0; c < rgb.cols; c++)
{
value = qRgb(rgb.ptr<uchar>(0)[r * rgb.cols * 3 + c * 3 + 0], rgb.ptr<uchar>(0)[r * rgb.cols * 3 + c * 3 + 1], rgb.ptr<uchar>(0)[r * rgb.cols * 3 + c * 3 + 2]);
image32.setPixel(c, r, value);
}
}
emit showBitmap(image32);
int key;
if(i < nframes - 1)
{
key = 0xff & waitKey(500);
}
else
{
key = waitKey(500);
}
if( (key & 255) == 27 )
break;
if( key == 'u' && mode == CALIBRATED )
undistortImage = !undistortImage;
}
if(imagePoints.size() > 0)
{
emit appendText("\n" + QString::number(imagePoints.size()) + " out of " + QString::number(nframes) + " images are effective!\n" );
runAndSave(outputFilename, imagePoints, imageSize,
boardSize, pattern, squareSize, aspectRatio,
flags, cameraMatrix, distCoeffs,
writeExtrinsics, writePoints);
}
else
{
emit appendText("Calibrating is not successful! \nPlease change the parameters and try again!");
emit popupErrorInformation("Sorry, no enough points are detected! Please try another group of images!");
emit closeImageWindow();
return;
}
emit appendText("Calibrating Successfully! \nPlease go to the folder to check the out put files!");
emit closeImageWindow();
if( !capture.isOpened() && showUndistorted )
{
Mat view, rview, map1, map2;
initUndistortRectifyMap(cameraMatrix, distCoeffs, Mat(),
getOptimalNewCameraMatrix(cameraMatrix, distCoeffs, imageSize, 1, imageSize, 0),
imageSize, CV_16SC2, map1, map2);
for( i = 0; i < (int)imageList.size(); i++ )
{
view = imread(imageList[i], 1);
if(!view.data)
continue;
//undistort( view, rview, cameraMatrix, distCoeffs, cameraMatrix );
remap(view, rview, map1, map2, INTER_LINEAR);
imshow("Image View", rview);
int c = 0xff & waitKey();
if( (c & 255) == 27 || c == 'q' || c == 'Q' )
break;
}
}
return;
}
// Gets the hue/sat/val for areas that we believe are license plate characters
// Then uses that to filter the whole image and provide a mask.
void ColorFilter::findCharColors()
{
int MINIMUM_SATURATION = 45;
if (this->debug)
cout << "ColorFilter::findCharColors" << endl;
//charMask.copyTo(this->colorMask);
this->colorMask = Mat::zeros(charMask.size(), CV_8U);
bitwise_not(this->colorMask, this->colorMask);
Mat erodedCharMask(charMask.size(), CV_8U);
Mat element = getStructuringElement( 1,
Size( 2 + 1, 2+1 ),
Point( 1, 1 ) );
erode(charMask, erodedCharMask, element);
vector<vector<Point> > contours;
vector<Vec4i> hierarchy;
findContours(erodedCharMask, contours, hierarchy, CV_RETR_TREE, CV_CHAIN_APPROX_SIMPLE);
vector<float> hMeans, sMeans, vMeans;
vector<float> hStdDevs, sStdDevs, vStdDevs;
for (unsigned int i = 0; i < contours.size(); i++)
{
if (hierarchy[i][3] != -1)
continue;
Mat singleCharMask = Mat::zeros(hsv.size(), CV_8U);
drawContours(singleCharMask, contours,
i, // draw this contour
cv::Scalar(255,255,255), // in
CV_FILLED,
8,
hierarchy
);
// get rid of the outline by drawing a 1 pixel width black line
drawContours(singleCharMask, contours,
i, // draw this contour
cv::Scalar(0,0,0), // in
1,
8,
hierarchy
);
//drawAndWait(&singleCharMask);
Scalar mean;
Scalar stddev;
meanStdDev(hsv, mean, stddev, singleCharMask);
if (this->debug)
{
cout << "ColorFilter " << setw(3) << i << ". Mean: h: " << setw(7) << mean[0] << " s: " << setw(7) <<mean[1] << " v: " << setw(7) << mean[2]
<< " | Std: h: " << setw(7) <<stddev[0] << " s: " << setw(7) <<stddev[1] << " v: " << stddev[2] << endl;
}
if (mean[0] == 0 && mean[1] == 0 && mean[2] == 0)
continue;
hMeans.push_back(mean[0]);
sMeans.push_back(mean[1]);
vMeans.push_back(mean[2]);
hStdDevs.push_back(stddev[0]);
sStdDevs.push_back(stddev[1]);
vStdDevs.push_back(stddev[2]);
}
if (hMeans.size() == 0)
return;
int bestHueIndex = this->getMajorityOpinion(hMeans, .65, 30);
int bestSatIndex = this->getMajorityOpinion(sMeans, .65, 35);
int bestValIndex = this->getMajorityOpinion(vMeans, .65, 30);
if (sMeans[bestSatIndex] < MINIMUM_SATURATION)
return;
bool doHueFilter = false, doSatFilter = false, doValFilter = false;
float hueMin, hueMax;
float satMin, satMax;
float valMin, valMax;
if (this->debug)
cout << "ColorFilter Winning indices:" << endl;
if (bestHueIndex != -1)
{
doHueFilter = true;
hueMin = hMeans[bestHueIndex] - (2 * hStdDevs[bestHueIndex]);
hueMax = hMeans[bestHueIndex] + (2 * hStdDevs[bestHueIndex]);
if (abs(hueMin - hueMax) < 20)
{
hueMin = hMeans[bestHueIndex] - 20;
hueMax = hMeans[bestHueIndex] + 20;
}
if (hueMin < 0)
hueMin = 0;
if (hueMax > 180)
hueMax = 180;
if (this->debug)
cout << "ColorFilter Hue: " << bestHueIndex << " : " << setw(7) << hMeans[bestHueIndex] << " -- " << hueMin << "-" << hueMax << endl;
}
if (bestSatIndex != -1)
{
doSatFilter = true;
satMin = sMeans[bestSatIndex] - (2 * sStdDevs[bestSatIndex]);
satMax = sMeans[bestSatIndex] + (2 * sStdDevs[bestSatIndex]);
if (abs(satMin - satMax) < 20)
{
satMin = sMeans[bestSatIndex] - 20;
satMax = sMeans[bestSatIndex] + 20;
}
if (satMin < 0)
satMin = 0;
if (satMax > 255)
satMax = 255;
if (this->debug)
cout << "ColorFilter Sat: " << bestSatIndex << " : " << setw(7) << sMeans[bestSatIndex] << " -- " << satMin << "-" << satMax << endl;
}
if (bestValIndex != -1)
{
doValFilter = true;
valMin = vMeans[bestValIndex] - (1.5 * vStdDevs[bestValIndex]);
valMax = vMeans[bestValIndex] + (1.5 * vStdDevs[bestValIndex]);
if (abs(valMin - valMax) < 20)
{
valMin = vMeans[bestValIndex] - 20;
valMax = vMeans[bestValIndex] + 20;
}
if (valMin < 0)
valMin = 0;
if (valMax > 255)
valMax = 255;
if (this->debug)
cout << "ColorFilter Val: " << bestValIndex << " : " << setw(7) << vMeans[bestValIndex] << " -- " << valMin << "-" << valMax << endl;
}
Mat imgDebugHueOnly = Mat::zeros(hsv.size(), hsv.type());
Mat imgDebug = Mat::zeros(hsv.size(), hsv.type());
Mat imgDistanceFromCenter = Mat::zeros(hsv.size(), CV_8U);
Mat debugMask = Mat::zeros(hsv.size(), CV_8U);
bitwise_not(debugMask, debugMask);
for (int row = 0; row < charMask.rows; row++)
{
for (int col = 0; col < charMask.cols; col++)
{
int h = (int) hsv.at<Vec3b>(row, col)[0];
int s = (int) hsv.at<Vec3b>(row, col)[1];
int v = (int) hsv.at<Vec3b>(row, col)[2];
bool hPasses = true;
bool sPasses = true;
bool vPasses = true;
int vDistance = abs(v - vMeans[bestValIndex]);
imgDebugHueOnly.at<Vec3b>(row, col)[0] = h;
imgDebugHueOnly.at<Vec3b>(row, col)[1] = 255;
imgDebugHueOnly.at<Vec3b>(row, col)[2] = 255;
imgDebug.at<Vec3b>(row, col)[0] = 255;
imgDebug.at<Vec3b>(row, col)[1] = 255;
imgDebug.at<Vec3b>(row, col)[2] = 255;
if (doHueFilter && (h < hueMin || h > hueMax))
{
hPasses = false;
imgDebug.at<Vec3b>(row, col)[0] = 0;
debugMask.at<uchar>(row, col) = 0;
}
if (doSatFilter && (s < satMin || s > satMax))
{
sPasses = false;
imgDebug.at<Vec3b>(row, col)[1] = 0;
}
if (doValFilter && (v < valMin || v > valMax))
{
vPasses = false;
imgDebug.at<Vec3b>(row, col)[2] = 0;
}
//if (pixelPasses)
// colorMask.at<uchar>(row, col) = 255;
//else
//imgDebug.at<Vec3b>(row, col)[0] = hPasses & 255;
//imgDebug.at<Vec3b>(row, col)[1] = sPasses & 255;
//imgDebug.at<Vec3b>(row, col)[2] = vPasses & 255;
if ((hPasses) || (hPasses && sPasses))//(hPasses && vPasses) || (sPasses && vPasses) ||
this->colorMask.at<uchar>(row, col) = 255;
else
this->colorMask.at<uchar>(row, col) = 0;
if ((hPasses && sPasses) || (hPasses && vPasses) || (sPasses && vPasses))
{
vDistance = pow(vDistance, 0.9);
}
else
{
vDistance = pow(vDistance, 1.1);
}
if (vDistance > 255)
vDistance = 255;
imgDistanceFromCenter.at<uchar>(row, col) = vDistance;
}
}
vector<Mat> debugImagesSet;
if (this->debug)
{
debugImagesSet.push_back(addLabel(charMask, "Charecter mask"));
//debugImagesSet1.push_back(erodedCharMask);
Mat maskCopy(colorMask.size(), colorMask.type());
colorMask.copyTo(maskCopy);
debugImagesSet.push_back(addLabel(maskCopy, "color Mask Before"));
}
Mat bigElement = getStructuringElement( 1,
Size( 3 + 1, 3+1 ),
Point( 1, 1 ) );
Mat smallElement = getStructuringElement( 1,
Size( 1 + 1, 1+1 ),
Point( 1, 1 ) );
morphologyEx(this->colorMask, this->colorMask, MORPH_CLOSE, bigElement);
//dilate(this->colorMask, this->colorMask, bigElement);
Mat combined(charMask.size(), charMask.type());
bitwise_and(charMask, colorMask, combined);
if (this->debug)
{
debugImagesSet.push_back(addLabel(colorMask, "Color Mask After"));
debugImagesSet.push_back(addLabel(combined, "Combined"));
//displayImage(config, "COLOR filter Mask", colorMask);
debugImagesSet.push_back(addLabel(imgDebug, "Color filter Debug"));
cvtColor(imgDebugHueOnly, imgDebugHueOnly, CV_HSV2BGR);
debugImagesSet.push_back(addLabel(imgDebugHueOnly, "Color Filter Hue"));
equalizeHist(imgDistanceFromCenter, imgDistanceFromCenter);
debugImagesSet.push_back(addLabel(imgDistanceFromCenter, "COLOR filter Distance"));
debugImagesSet.push_back(addLabel(debugMask, "COLOR Hues off"));
Mat dashboard = drawImageDashboard(debugImagesSet, imgDebugHueOnly.type(), 3);
displayImage(config, "Color Filter Images", dashboard);
}
}
void PlateLines::processImage(Mat inputImage, vector<TextLine> textLines, float sensitivity)
{
if (this->debug)
cout << "PlateLines findLines" << endl;
timespec startTime;
getTimeMonotonic(&startTime);
// Ignore input images that are pure white or pure black
Scalar avgPixelIntensity = mean(inputImage);
if (avgPixelIntensity[0] >= 252)
return;
else if (avgPixelIntensity[0] <= 3)
return;
// Do a bilateral filter to clean the noise but keep edges sharp
Mat smoothed(inputImage.size(), inputImage.type());
adaptiveBilateralFilter(inputImage, smoothed, Size(3,3), 45, 45);
int morph_elem = 2;
int morph_size = 2;
Mat element = getStructuringElement( morph_elem, Size( 2*morph_size + 1, 2*morph_size+1 ), Point( morph_size, morph_size ) );
Mat edges(inputImage.size(), inputImage.type());
Canny(smoothed, edges, 66, 133);
// Create a mask that is dilated based on the detected characters
Mat mask = Mat::zeros(inputImage.size(), CV_8U);
for (unsigned int i = 0; i < textLines.size(); i++)
{
vector<vector<Point> > polygons;
polygons.push_back(textLines[i].textArea);
fillPoly(mask, polygons, Scalar(255,255,255));
}
dilate(mask, mask, getStructuringElement( 1, Size( 1 + 1, 2*1+1 ), Point( 1, 1 ) ));
bitwise_not(mask, mask);
// AND canny edges with the character mask
bitwise_and(edges, mask, edges);
vector<PlateLine> hlines = this->getLines(edges, sensitivity, false);
vector<PlateLine> vlines = this->getLines(edges, sensitivity, true);
for (unsigned int i = 0; i < hlines.size(); i++)
this->horizontalLines.push_back(hlines[i]);
for (unsigned int i = 0; i < vlines.size(); i++)
this->verticalLines.push_back(vlines[i]);
// if debug is enabled, draw the image
if (this->debug)
{
Mat debugImgHoriz(edges.size(), edges.type());
Mat debugImgVert(edges.size(), edges.type());
edges.copyTo(debugImgHoriz);
edges.copyTo(debugImgVert);
cvtColor(debugImgHoriz,debugImgHoriz,CV_GRAY2BGR);
cvtColor(debugImgVert,debugImgVert,CV_GRAY2BGR);
for( size_t i = 0; i < this->horizontalLines.size(); i++ )
{
line( debugImgHoriz, this->horizontalLines[i].line.p1, this->horizontalLines[i].line.p2, Scalar(0,0,255), 1, CV_AA);
}
for( size_t i = 0; i < this->verticalLines.size(); i++ )
{
line( debugImgVert, this->verticalLines[i].line.p1, this->verticalLines[i].line.p2, Scalar(0,0,255), 1, CV_AA);
}
vector<Mat> images;
images.push_back(debugImgHoriz);
images.push_back(debugImgVert);
Mat dashboard = drawImageDashboard(images, debugImgVert.type(), 1);
displayImage(pipelineData->config, "Hough Lines", dashboard);
}
if (pipelineData->config->debugTiming)
{
timespec endTime;
getTimeMonotonic(&endTime);
cout << "Plate Lines Time: " << diffclock(startTime, endTime) << "ms." << endl;
}
}
void OCRTess::set(UMat u) {
if (this->downsize && u.cols > MIN_OCR) resize(u, u, Size(MIN_OCR, MIN_OCR));
bitwise_not(u, this->img); // 색 반전
}
/* Process the images inside each directory */
bool processDir(std::string path, std::string image_name, std::string metrics_file) {
/** Create the initial data collection skeleton **/
// Open the metrics file
std::ofstream data_stream;
data_stream.open(metrics_file, std::ios::app);
if (!data_stream.is_open()) {
std::cerr << "Could not open the data output file." << std::endl;
return false;
}
// Create the output image directory
std::string out_directory = path + "result/";
struct stat st = {0};
if (stat(out_directory.c_str(), &st) == -1) {
mkdir(out_directory.c_str(), 0700);
}
// Analyzed image name
std::string analyzed_image_name = image_name;
std::size_t found = analyzed_image_name.find("dapi");
analyzed_image_name.replace(found, 4, "merge");
data_stream << analyzed_image_name << ",";
/** Extract the dapi, gfp and rfp streams for each input image **/
// DAPI
std::string in_filename = path + "jpg/" + image_name;
cv::Mat dapi = cv::imread(in_filename.c_str(), -1);
if (dapi.empty()) return false;
// GFP
std::string gfp_image_name = image_name;
found = gfp_image_name.find("dapi");
gfp_image_name.replace(found, 4, "gfp");
in_filename = path + "jpg/" + gfp_image_name;
cv::Mat gfp = cv::imread(in_filename.c_str(), -1);
if (gfp.empty()) return false;
// RFP
std::string rfp_image_name = image_name;
found = rfp_image_name.find("dapi");
rfp_image_name.replace(found, 4, "rfp");
in_filename = path + "jpg/" + rfp_image_name;
cv::Mat rfp = cv::imread(in_filename.c_str(), -1);
if (rfp.empty()) return false;
/** Gather information needed for feature extraction **/
/* DAPI image */
// Enhance
cv::Mat dapi_normalized, dapi_enhanced;
if (!enhanceImage( dapi,
ChannelType::DAPI,
&dapi_normalized,
&dapi_enhanced )) {
return false;
}
// Segment
cv::Mat dapi_segmented;
std::vector<std::vector<cv::Point>> contours_dapi;
std::vector<cv::Vec4i> hierarchy_dapi;
std::vector<HierarchyType> dapi_contour_mask;
std::vector<double> dapi_contour_area;
contourCalc(dapi_enhanced, ChannelType::DAPI, 1.0,
&dapi_segmented, &contours_dapi,
&hierarchy_dapi, &dapi_contour_mask,
&dapi_contour_area);
// Filter the dapi contours
std::vector<std::vector<cv::Point>> contours_dapi_filtered;
filterCells(dapi_enhanced, contours_dapi, dapi_contour_mask, &contours_dapi_filtered);
/* GFP image */
cv::Mat gfp_normalized, gfp_enhanced;
if (!enhanceImage( gfp,
ChannelType::GFP,
&gfp_normalized,
&gfp_enhanced )) {
return false;
}
// GFP Low
cv::Mat gfp_low_normalized, gfp_low_enhanced;
if (!enhanceImage( gfp,
ChannelType::GFP_LOW,
&gfp_low_normalized,
&gfp_low_enhanced )) {
return false;
}
cv::Mat gfp_low_segmented;
std::vector<std::vector<cv::Point>> contours_gfp_low;
std::vector<cv::Vec4i> hierarchy_gfp_low;
std::vector<HierarchyType> gfp_low_contour_mask;
std::vector<double> gfp_low_contour_area;
contourCalc(gfp_low_enhanced, ChannelType::GFP_LOW, 1.0,
&gfp_low_segmented, &contours_gfp_low,
&hierarchy_gfp_low, &gfp_low_contour_mask,
&gfp_low_contour_area);
// GFP Medium
cv::Mat gfp_medium_normalized, gfp_medium_enhanced;
if (!enhanceImage( gfp,
ChannelType::GFP_MEDIUM,
&gfp_medium_normalized,
&gfp_medium_enhanced )) {
return false;
}
cv::Mat gfp_medium_segmented;
std::vector<std::vector<cv::Point>> contours_gfp_medium;
std::vector<cv::Vec4i> hierarchy_gfp_medium;
std::vector<HierarchyType> gfp_medium_contour_mask;
std::vector<double> gfp_medium_contour_area;
contourCalc(gfp_medium_enhanced, ChannelType::GFP_MEDIUM, 1.0,
&gfp_medium_segmented, &contours_gfp_medium,
&hierarchy_gfp_medium, &gfp_medium_contour_mask,
&gfp_medium_contour_area);
// GFP High
cv::Mat gfp_high_normalized, gfp_high_enhanced;
if (!enhanceImage( gfp,
ChannelType::GFP_HIGH,
&gfp_high_normalized,
&gfp_high_enhanced )) {
return false;
}
cv::Mat gfp_high_segmented;
std::vector<std::vector<cv::Point>> contours_gfp_high;
std::vector<cv::Vec4i> hierarchy_gfp_high;
std::vector<HierarchyType> gfp_high_contour_mask;
std::vector<double> gfp_high_contour_area;
contourCalc(gfp_high_enhanced, ChannelType::GFP_HIGH, 1.0,
&gfp_high_segmented, &contours_gfp_high,
&hierarchy_gfp_high, &gfp_high_contour_mask,
&gfp_high_contour_area);
/* RFP image */
cv::Mat rfp_normalized, rfp_enhanced_type1;
if (!enhanceImage( rfp,
ChannelType::RFP_TYPE1,
&rfp_normalized,
&rfp_enhanced_type1 )) {
return false;
}
cv::Mat rfp_enhanced_type2;
if (!enhanceImage( rfp,
ChannelType::RFP_TYPE2,
&rfp_normalized,
&rfp_enhanced_type2 )) {
return false;
}
cv::Mat rfp_segmented;
std::vector<std::vector<cv::Point>> contours_rfp_vec;
std::vector<cv::Vec4i> hierarchy_rfp;
std::vector<HierarchyType> rfp_contour_mask;
std::vector<double> rfp_contour_area;
contourCalc(rfp_enhanced_type2, ChannelType::RFP_TYPE2, 1.0,
&rfp_segmented, &contours_rfp_vec,
&hierarchy_rfp, &rfp_contour_mask,
&rfp_contour_area);
/* RFP Scatter plot */
// Determine whether the image is Control or SZ
bool is_control = false;
std::string sample_id = image_name.substr(0, 4);
// Control - 3440, 3651, 4506, 9319, 9429, BJ2E, BJ3E
if ( (sample_id == "3440") ||
(sample_id == "3651") ||
(sample_id == "4506") ||
(sample_id == "9319") ||
(sample_id == "9429") ||
(sample_id == "BJ2E") ||
(sample_id == "BJ3E") ) {
is_control = true;
}
// Note: Do nothing for SZ - 1792, 1835, 2038, 2497
scatterPlot( path + PLOTS_DIR_NAME,
is_control,
rfp_normalized,
contours_rfp_vec,
rfp_contour_mask );
/** Classify the cell soma **/
std::vector<std::vector<cv::Point>> contours_gfp, contours_rfp;
cv::Mat gfp_intersection = cv::Mat::zeros(gfp_enhanced.size(), CV_8UC1);
cv::Mat rfp_intersection = cv::Mat::zeros(rfp_enhanced_type1.size(), CV_8UC1);
for (size_t i = 0; i < contours_dapi_filtered.size(); i++) {
// Find DAPI-GFP Cell Soma
std::vector<cv::Point> gfp_contour;
cv::Mat temp;
if (findCellSoma( contours_dapi_filtered[i], gfp_enhanced, &temp, &gfp_contour )) {
contours_gfp.push_back(gfp_contour);
bitwise_or(gfp_intersection, temp, gfp_intersection);
cv::Mat temp_not;
bitwise_not(temp, temp_not);
bitwise_and(gfp_enhanced, temp_not, gfp_enhanced);
}
// Find DAPI-RFP Cell Soma
std::vector<cv::Point> rfp_contour;
if (findCellSoma( contours_dapi_filtered[i], rfp_enhanced_type1, &temp, &rfp_contour )) {
contours_rfp.push_back(rfp_contour);
bitwise_or(rfp_intersection, temp, rfp_intersection);
cv::Mat temp_not;
bitwise_not(temp, temp_not);
bitwise_and(rfp_enhanced_type1, temp_not, rfp_enhanced_type1);
}
}
/** Collect the metrics **/
// Separation metrics for dapi-gfp cells
float mean_dia = 0.0, stddev_dia = 0.0;
float mean_aspect_ratio = 0.0, stddev_aspect_ratio = 0.0;
float mean_error_ratio = 0.0, stddev_error_ratio = 0.0;
separationMetrics( contours_gfp,
&mean_dia,
&stddev_dia,
&mean_aspect_ratio,
&stddev_aspect_ratio,
&mean_error_ratio,
&stddev_error_ratio
);
data_stream << contours_gfp.size() << ","
<< mean_dia << ","
<< stddev_dia << ","
<< mean_aspect_ratio << ","
<< stddev_aspect_ratio << ","
<< mean_error_ratio << ","
<< stddev_error_ratio << ",";
// Separation metrics for dapi-rfp cells
mean_dia = 0.0;
stddev_dia = 0.0;
mean_aspect_ratio = 0.0;
stddev_aspect_ratio = 0.0;
mean_error_ratio = 0.0;
stddev_error_ratio = 0.0;
separationMetrics( contours_rfp,
&mean_dia,
&stddev_dia,
&mean_aspect_ratio,
&stddev_aspect_ratio,
&mean_error_ratio,
&stddev_error_ratio
);
data_stream << contours_rfp.size() << ","
<< mean_dia << ","
<< stddev_dia << ","
<< mean_aspect_ratio << ","
<< stddev_aspect_ratio << ","
<< mean_error_ratio << ","
<< stddev_error_ratio << ",";
/* Characterize the gfp channel */
// Gfp low
std::string gfp_low_output;
binArea(gfp_low_contour_mask, gfp_low_contour_area, &gfp_low_output);
data_stream << gfp_low_output << ",";
// Gfp medium
std::string gfp_medium_output;
binArea(gfp_medium_contour_mask, gfp_medium_contour_area, &gfp_medium_output);
data_stream << gfp_medium_output << ",";
// Gfp high
std::string gfp_high_output;
binArea(gfp_high_contour_mask, gfp_high_contour_area, &gfp_high_output);
data_stream << gfp_high_output << ",";
// End of entry
data_stream << std::endl;
data_stream.close();
/** Display the debug image **/
if (DEBUG_FLAG) {
// Initialize
cv::Mat drawing_blue_debug = cv::Mat::zeros(dapi_enhanced.size(), CV_8UC1);
//cv::Mat drawing_blue_debug = 2*dapi_normalized;
//cv::Mat drawing_green_debug = cv::Mat::zeros(gfp_enhanced.size(), CV_8UC1);
cv::Mat drawing_green_debug = gfp_intersection;
cv::Mat drawing_red_debug = cv::Mat::zeros(rfp_enhanced_type1.size(), CV_8UC1);
//cv::Mat drawing_red_debug = rfp_intersection;
// Draw DAPI bondaries
for (size_t i = 0; i < contours_dapi_filtered.size(); i++) {
cv::Moments mu = moments(contours_dapi_filtered[i], true);
cv::Point2f mc = cv::Point2f( static_cast<float>(mu.m10/mu.m00),
static_cast<float>(mu.m01/mu.m00) );
cv::circle(drawing_blue_debug, mc, DAPI_MASK_RADIUS, 255, 1, 8);
cv::circle(drawing_green_debug, mc, DAPI_MASK_RADIUS, 255, 1, 8);
cv::circle(drawing_red_debug, mc, DAPI_MASK_RADIUS, 255, 1, 8);
}
// Merge the modified red, blue and green layers
std::vector<cv::Mat> merge_debug;
merge_debug.push_back(drawing_blue_debug);
merge_debug.push_back(drawing_green_debug);
merge_debug.push_back(drawing_red_debug);
cv::Mat color_debug;
cv::merge(merge_debug, color_debug);
// Draw the debug image
std::string out_debug = out_directory + analyzed_image_name;
out_debug.insert(out_debug.find_last_of("."), "_debug", 6);
cv::imwrite(out_debug.c_str(), color_debug);
}
/** Display the analyzed <dapi,gfp,rfp> image set **/
// Initialize
cv::Mat drawing_blue = 2*dapi_normalized;
cv::Mat drawing_green = gfp_normalized;
cv::Mat drawing_red = rfp_normalized;
// Draw GFP bondaries
for (size_t i = 0; i < contours_gfp.size(); i++) {
//cv::RotatedRect min_ellipse = fitEllipse(cv::Mat(contours_gfp[i]));
//ellipse(drawing_blue, min_ellipse, 255, 1, 8);
//ellipse(drawing_green, min_ellipse, 255, 1, 8);
//ellipse(drawing_red, min_ellipse, 0, 1, 8);
drawContours(drawing_blue, contours_gfp, i, 255, 1, 8);
drawContours(drawing_green, contours_gfp, i, 255, 1, 8);
drawContours(drawing_red, contours_gfp, i, 0, 1, 8);
}
// Draw RFP bondaries
for (size_t i = 0; i < contours_rfp.size(); i++) {
//cv::RotatedRect min_ellipse = fitEllipse(cv::Mat(contours_rfp[i]));
//ellipse(drawing_blue, min_ellipse, 255, 1, 8);
//ellipse(drawing_green, min_ellipse, 0, 1, 8);
//ellipse(drawing_red, min_ellipse, 255, 1, 8);
drawContours(drawing_blue, contours_rfp, i, 255, 1, 8);
drawContours(drawing_green, contours_rfp, i, 0, 1, 8);
drawContours(drawing_red, contours_rfp, i, 255, 1, 8);
}
// Merge the modified red, blue and green layers
std::vector<cv::Mat> merge_analyzed;
merge_analyzed.push_back(drawing_blue);
merge_analyzed.push_back(drawing_green);
merge_analyzed.push_back(drawing_red);
cv::Mat color_analyzed;
cv::merge(merge_analyzed, color_analyzed);
// Draw the analyzed image
std::string out_analyzed = out_directory + analyzed_image_name;
cv::imwrite(out_analyzed.c_str(), color_analyzed);
return true;
}
CvPoint2D32f getPupilCenter(Mat &eye_box){
//find x and y gradients
Mat gradientX = computeGradient(eye_box);
Mat gradientY = computeGradient(eye_box.t()).t();
//normalize and threshold the gradient
Mat mags = matrixMagnitude(gradientX, gradientY);
//create a blurred and inverted image for weighting
Mat weight;
bitwise_not(eye_box, weight);
blur(weight, weight, Size(2,2));
//weight the magnitudes, convert to 8-bit for thresholding
weight.convertTo(weight, CV_32F);
mags = mags.mul(weight);
normalize(mags, mags, 0, 1, NORM_MINMAX, CV_32F);
mags.convertTo(mags, CV_8UC1, 255);
//threshold using Otsu's method
threshold(mags, mags, 0, 255, THRESH_BINARY | THRESH_OTSU);
//convert to CV_32S and filter gradients
mags.convertTo(mags, CV_32S);
gradientY = gradientY.mul(mags);
gradientX = gradientX.mul(mags);
//resize arrays to same size
resize(gradientX, gradientX, Size(EYE_FRAME_SIZE, EYE_FRAME_SIZE), 0, 0, INTER_NEAREST);
resize(gradientY, gradientY, Size(EYE_FRAME_SIZE, EYE_FRAME_SIZE), 0, 0, INTER_NEAREST);
resize(weight, weight, Size(EYE_FRAME_SIZE, EYE_FRAME_SIZE), 0, 0, INTER_NEAREST);
//imshow("gradY", gradientY * 255);
//imshow("weight", weight / 255);
//run the algorithm:
// for each possible gradient location
// Note: these loops are reversed from the way the paper does them
// it evaluates every possible center for each gradient location instead of
// every possible gradient location for every center.
Mat out = Mat::zeros(weight.rows,weight.cols, CV_32F);
float max_val = 0;
//for all pixels in the image
for (int y = 0; y < EYE_FRAME_SIZE; ++y) {
const int *grad_x = gradientX.ptr<int>(y), *grad_y = gradientY.ptr<int>(y);
for (int x = 0; x < EYE_FRAME_SIZE; ++x) {
int gX = grad_x[x], gY = grad_y[x];
if (gX == 0 && gY == 0) {
continue;
}
//for all possible centers
for (int cy = 0; cy < EYE_FRAME_SIZE; ++cy) {
float *Or = out.ptr<float>(cy);
const float *Wr = weight.ptr<float>(cy);
for (int cx = 0; cx < EYE_FRAME_SIZE; ++cx) {
//ignore center of box
if (x == cx && y == cy) {
continue;
}
//create a vector from the possible center to the gradient origin
int dx = x - cx;
int dy = y - cy;
//compute dot product using lookup table
float dotProduct;
if(dx > 0 && dy > 0){
dotProduct = dpX[dx+EYE_FRAME_SIZE*dy]*gX + dpY[dx+EYE_FRAME_SIZE*dy]*gY;
}else if(dx > 0){
dotProduct = dpX[dx-EYE_FRAME_SIZE*dy]*gX - dpY[dx-EYE_FRAME_SIZE*dy]*gY;
}else if(dy > 0){
dotProduct = -dpX[-dx+EYE_FRAME_SIZE*dy]*gX - dpY[-dx+EYE_FRAME_SIZE*dy]*gY;
}else{
dotProduct = -dpX[-dx-EYE_FRAME_SIZE*dy]*gX - dpY[-dx-EYE_FRAME_SIZE*dy]*gY;
}
//ignore negative dot products as they point away from eye
if(dotProduct <= 0.0){
continue;
}
//square and multiply by the weight
Or[cx] += dotProduct * dotProduct * Wr[cx];
//compare with max
if(Or[cx] > max_val){
max_val = Or[cx];
}
}
}
}
}
//resize for debugging
resize(out, out, Size(500,500), 0, 0, INTER_NEAREST);
out = 255 * out / max_val;
//imshow("calc", out / 255);
//histogram setup
Mat hist;
int histSize = 256;
float range[] = { 0, 256 } ;
const float* histRange = { range };
//calculate the histogram
calcHist(&out,1, 0, Mat(), hist, 1, &histSize, &histRange,
true, //uniform
true //accumulate
);
//get cutoff for top 10 pixels
float top_end_sum = 0;
int top_end = 0.92 * 255;
for (int i = 255; i > 0; i--) {
top_end_sum += hist.at<float>(i);
if(top_end_sum > 3000){
top_end = i;
break;
}
}
//draw image for debugging
Mat histImage(400, 512, CV_8UC3, Scalar(0,0,0));
int bin_w = cvRound( (double) 512/histSize );
normalize(hist, hist, 0, histImage.rows, NORM_MINMAX, -1, Mat());
/// Draw for each channel
for( int i = 1; i < histSize; i++)
{
line(histImage, Point(bin_w*(i), 400 - cvRound(hist.at<float>(i))),
Point(bin_w*(i), 400),
Scalar(i, i, i), 2, 8, 0);
}
//imshow("hist", histImage);
//threshold to get just the pupil
//printf("top_end: %d\n", top_end);
threshold(out, out, top_end, 255, THRESH_TOZERO);
//calc center of mass
float sum = 0;
float sum_x = 0;
float sum_y = 0;
for (int y = 0; y < out.rows; ++y)
{
float* row = out.ptr<float>(y);
for (int x = 0; x < out.cols; ++x)
{
float val = row[x]*row[x];
if(val > 0){
sum += val;
sum_x += val*x;
sum_y += val*y;
}
}
}
Size eye_box_size = eye_box.size();
Size out_size = out.size();
//cout << "Size1: "+to_string(eye_box_size.width)+","+to_string(eye_box_size.height)+"\n";
//cout << "Size2: "+to_string(out_size.width)+","+to_string(out_size.height)+"\n";
float x_scale = (float) eye_box_size.width / out_size.width;
float y_scale = (float) eye_box_size.height / out_size.height;
CvPoint2D32f max = cvPoint2D32f(x_scale*sum_x/sum, y_scale*sum_y/sum);
//circle(out, max, 3, 0);
//imshow("thresh", out / 255);
return max;
}
void OCR::performOCR(PipelineData* pipeline_data)
{
const int SPACE_CHAR_CODE = 32;
timespec startTime;
getTimeMonotonic(&startTime);
postProcessor.clear();
// Don't waste time on OCR processing if it is impossible to get sufficient characters
int total_char_spaces = 0;
for (unsigned int i = 0; i < pipeline_data->charRegions.size(); i++)
total_char_spaces += pipeline_data->charRegions[i].size();
if (total_char_spaces < config->postProcessMinCharacters)
{
pipeline_data->disqualify_reason = "Insufficient character boxes detected. No OCR performed.";
pipeline_data->disqualified = true;
return;
}
for (unsigned int i = 0; i < pipeline_data->thresholds.size(); i++)
{
// Make it black text on white background
bitwise_not(pipeline_data->thresholds[i], pipeline_data->thresholds[i]);
tesseract.SetImage((uchar*) pipeline_data->thresholds[i].data,
pipeline_data->thresholds[i].size().width, pipeline_data->thresholds[i].size().height,
pipeline_data->thresholds[i].channels(), pipeline_data->thresholds[i].step1());
int absolute_charpos = 0;
for (unsigned int line_idx = 0; line_idx < pipeline_data->charRegions.size(); line_idx++)
{
for (unsigned int j = 0; j < pipeline_data->charRegions[line_idx].size(); j++)
{
Rect expandedRegion = expandRect( pipeline_data->charRegions[line_idx][j], 2, 2, pipeline_data->thresholds[i].cols, pipeline_data->thresholds[i].rows) ;
tesseract.SetRectangle(expandedRegion.x, expandedRegion.y, expandedRegion.width, expandedRegion.height);
tesseract.Recognize(NULL);
tesseract::ResultIterator* ri = tesseract.GetIterator();
tesseract::PageIteratorLevel level = tesseract::RIL_SYMBOL;
do
{
const char* symbol = ri->GetUTF8Text(level);
float conf = ri->Confidence(level);
bool dontcare;
int fontindex = 0;
int pointsize = 0;
const char* fontName = ri->WordFontAttributes(&dontcare, &dontcare, &dontcare, &dontcare, &dontcare, &dontcare, &pointsize, &fontindex);
// Ignore NULL pointers, spaces, and characters that are way too small to be valid
if(symbol != 0 && symbol[0] != SPACE_CHAR_CODE && pointsize >= config->ocrMinFontSize)
{
postProcessor.addLetter(string(symbol), line_idx, absolute_charpos, conf);
if (this->config->debugOcr)
printf("charpos%d line%d: threshold %d: symbol %s, conf: %f font: %s (index %d) size %dpx\n", absolute_charpos, line_idx, i, symbol, conf, fontName, fontindex, pointsize);
bool indent = false;
tesseract::ChoiceIterator ci(*ri);
do
{
const char* choice = ci.GetUTF8Text();
//1/17/2016 adt adding check to avoid double adding same character if ci is same as symbol. Otherwise first choice will get doubled boost when choiceIterator run.
if (string(symbol) != string(choice))
postProcessor.addLetter(string(choice), line_idx, absolute_charpos, ci.Confidence());
if (this->config->debugOcr)
{
if (indent) printf("\t\t ");
printf("\t- ");
printf("%s conf: %f\n", choice, ci.Confidence());
}
indent = true;
}
while(ci.Next());
}
if (this->config->debugOcr)
printf("---------------------------------------------\n");
delete[] symbol;
}
while((ri->Next(level)));
delete ri;
absolute_charpos++;
}
}
}
if (config->debugTiming)
{
timespec endTime;
getTimeMonotonic(&endTime);
cout << "OCR Time: " << diffclock(startTime, endTime) << "ms." << endl;
}
}
// Tries to find a rectangular area surrounding most of the characters. Not required
// but helpful when determining the plate edges
void PlateMask::findOuterBoxMask( vector<TextContours > contours )
{
double min_parent_area = pipeline_data->config->templateHeightPx * pipeline_data->config->templateWidthPx * 0.10; // Needs to be at least 10% of the plate area to be considered.
int winningIndex = -1;
int winningParentId = -1;
int bestCharCount = 0;
double lowestArea = 99999999999999;
if (pipeline_data->config->debugCharAnalysis)
cout << "CharacterAnalysis::findOuterBoxMask" << endl;
for (unsigned int imgIndex = 0; imgIndex < contours.size(); imgIndex++)
{
//vector<bool> charContours = filter(thresholds[imgIndex], allContours[imgIndex], allHierarchy[imgIndex]);
int charsRecognized = 0;
int parentId = -1;
bool hasParent = false;
for (unsigned int i = 0; i < contours[imgIndex].goodIndices.size(); i++)
{
if (contours[imgIndex].goodIndices[i]) charsRecognized++;
if (contours[imgIndex].goodIndices[i] && contours[imgIndex].hierarchy[i][3] != -1)
{
parentId = contours[imgIndex].hierarchy[i][3];
hasParent = true;
}
}
if (charsRecognized == 0)
continue;
if (hasParent)
{
double boxArea = contourArea(contours[imgIndex].contours[parentId]);
if (boxArea < min_parent_area)
continue;
if ((charsRecognized > bestCharCount) ||
(charsRecognized == bestCharCount && boxArea < lowestArea))
//(boxArea < lowestArea)
{
bestCharCount = charsRecognized;
winningIndex = imgIndex;
winningParentId = parentId;
lowestArea = boxArea;
}
}
}
if (pipeline_data->config->debugCharAnalysis)
cout << "Winning image index (findOuterBoxMask) is: " << winningIndex << endl;
if (winningIndex != -1 && bestCharCount >= 3)
{
Mat mask = Mat::zeros(pipeline_data->thresholds[winningIndex].size(), CV_8U);
// get rid of the outline by drawing a 1 pixel width black line
drawContours(mask, contours[winningIndex].contours,
winningParentId, // draw this contour
cv::Scalar(255,255,255), // in
FILLED,
8,
contours[winningIndex].hierarchy,
0
);
// Morph Open the mask to get rid of any little connectors to non-plate portions
int morph_elem = 2;
int morph_size = 3;
Mat element = getStructuringElement( morph_elem, Size( 2*morph_size + 1, 2*morph_size+1 ), Point( morph_size, morph_size ) );
//morphologyEx( mask, mask, MORPH_CLOSE, element );
morphologyEx( mask, mask, MORPH_OPEN, element );
//morph_size = 1;
//element = getStructuringElement( morph_elem, Size( 2*morph_size + 1, 2*morph_size+1 ), Point( morph_size, morph_size ) );
//dilate(mask, mask, element);
// Drawing the edge black effectively erodes the image. This may clip off some extra junk from the edges.
// We'll want to do the contour again and find the larges one so that we remove the clipped portion.
vector<vector<Point> > contoursSecondRound;
findContours(mask, contoursSecondRound, RETR_EXTERNAL, CHAIN_APPROX_SIMPLE);
int biggestContourIndex = -1;
double largestArea = 0;
for (unsigned int c = 0; c < contoursSecondRound.size(); c++)
{
double area = contourArea(contoursSecondRound[c]);
if (area > largestArea)
{
biggestContourIndex = c;
largestArea = area;
}
}
if (biggestContourIndex != -1)
{
mask = Mat::zeros(pipeline_data->thresholds[winningIndex].size(), CV_8U);
vector<Point> smoothedMaskPoints;
approxPolyDP(contoursSecondRound[biggestContourIndex], smoothedMaskPoints, 2, true);
vector<vector<Point> > tempvec;
tempvec.push_back(smoothedMaskPoints);
//fillPoly(mask, smoothedMaskPoints.data(), smoothedMaskPoints, Scalar(255,255,255));
drawContours(mask, tempvec,
0, // draw this contour
cv::Scalar(255,255,255), // in
FILLED,
8,
contours[winningIndex].hierarchy,
0
);
}
if (pipeline_data->config->debugCharAnalysis)
{
vector<Mat> debugImgs;
Mat debugImgMasked = Mat::zeros(pipeline_data->thresholds[winningIndex].size(), CV_8U);
pipeline_data->thresholds[winningIndex].copyTo(debugImgMasked, mask);
debugImgs.push_back(mask);
debugImgs.push_back(pipeline_data->thresholds[winningIndex]);
debugImgs.push_back(debugImgMasked);
Mat dashboard = drawImageDashboard(debugImgs, CV_8U, 1);
displayImage(pipeline_data->config, "Winning outer box", dashboard);
}
hasPlateMask = true;
this->plateMask = mask;
} else {
hasPlateMask = false;
Mat fullMask = Mat::zeros(pipeline_data->thresholds[0].size(), CV_8U);
bitwise_not(fullMask, fullMask);
this->plateMask = fullMask;
}
}
void CameraCalibration::calibrate()
{
const string inputSettingsFile = "default.xml";
// Read the settings
FileStorage fs( inputSettingsFile, FileStorage::READ );
if ( !fs.isOpened() ) {
FileStorage fs( inputSettingsFile, FileStorage::WRITE );
fs.release();
cerr << "Could not open the configuration file: \"" << inputSettingsFile << "\"" << endl;
return;
}
else {
s.read( fs["Settings"] );
// close Settings file
fs.release();
}
if ( !s.goodInput ) {
cerr << "Invalid input detected. Application stopping." << endl;
return;
}
vector<vector<Point2f> > imagePoints;
Mat distCoeffs;
Size imageSize;
int mode = s.inputType == Settings::IMAGE_LIST ? CAPTURING : DETECTION;
clock_t prevTimestamp = 0;
const Scalar RED( 0, 0, 255 ), GREEN( 0, 255, 0 );
const char ESC_KEY = 27;
for ( int i = 0; ; ++i ) {
Mat view;
bool blinkOutput = false;
view = s.nextImage();
//----- If no more image, or got enough, then stop calibration and show result -------------
if ( mode == CAPTURING && imagePoints.size() >= (unsigned)s.nrFrames ) {
if ( runCalibrationAndSave(s, imageSize, cameraMatrix, distCoeffs, imagePoints ) ) {
mode = CALIBRATED;
}
else {
mode = DETECTION;
}
}
// If no more images then run calibration, save and stop loop.
if ( view.empty() ) {
if ( imagePoints.size() > 0 ) {
runCalibrationAndSave(s, imageSize, cameraMatrix, distCoeffs, imagePoints);
}
break;
}
imageSize = view.size(); // Format input image.
if ( s.flipVertical ) {
flip( view, view, 0 );
}
vector<Point2f> pointBuf;
bool found;
// Find feature points on the input format
switch ( s.calibrationPattern ) {
case Settings::CHESSBOARD:
found = findChessboardCorners( view, s.boardSize, pointBuf,
CALIB_CB_ADAPTIVE_THRESH | CALIB_CB_FAST_CHECK | CALIB_CB_NORMALIZE_IMAGE);
break;
case Settings::CIRCLES_GRID:
found = findCirclesGrid( view, s.boardSize, pointBuf );
break;
case Settings::ASYMMETRIC_CIRCLES_GRID:
found = findCirclesGrid( view, s.boardSize, pointBuf, CALIB_CB_ASYMMETRIC_GRID );
break;
default:
found = false;
break;
}
// If done with success, improve the found corners' coordinate accuracy for chessboard
if ( found ) {
if ( s.calibrationPattern == Settings::CHESSBOARD ) {
Mat viewGray;
cvtColor( view, viewGray, COLOR_BGR2GRAY );
cornerSubPix( viewGray, pointBuf, Size( 11,11 ), Size(-1,-1), TermCriteria( TermCriteria::EPS + TermCriteria::MAX_ITER, 30, 0.1 ) );
}
// For camera only take new samples after delay time
if ( mode == CAPTURING && (!s.inputCapture.isOpened() || clock() - prevTimestamp > s.delay*1e-3*CLOCKS_PER_SEC) ) {
imagePoints.push_back( pointBuf );
prevTimestamp = clock();
blinkOutput = s.inputCapture.isOpened();
}
// Draw the corners.
drawChessboardCorners( view, s.boardSize, Mat( pointBuf ), found );
}
//----------------------------- Output Text ------------------------------------------------
string msg = ( mode == CAPTURING ) ? "100/100" :
mode == CALIBRATED ? "Calibrated" : "Press 'g' to start";
int baseLine = 0;
Size textSize = getTextSize( msg, 1, 1, 1, &baseLine );
Point textOrigin( view.cols - 2*textSize.width - 10, view.rows - 2*baseLine - 10 );
if ( mode == CAPTURING ) {
if ( s.showUndistorsed ) {
msg = format( "%d/%d Undist", (int)imagePoints.size(), s.nrFrames );
}
else {
msg = format( "%d/%d", (int)imagePoints.size(), s.nrFrames );
}
}
putText( view, msg, textOrigin, 1, 1, mode == CALIBRATED ? GREEN : RED );
if ( blinkOutput ) {
bitwise_not( view, view );
}
//------------------------- Video capture output undistorted ------------------------------
if ( mode == CALIBRATED && s.showUndistorsed ) {
Mat temp = view.clone();
undistort( temp, view, cameraMatrix, distCoeffs );
}
//------------------------------ Show image and check for input commands -------------------
imshow( "Image View", view );
char key = (char)waitKey( s.inputCapture.isOpened() ? 50 : s.delay );
if ( key == ESC_KEY ) {
break;
}
if ( key == 'u' && mode == CALIBRATED ) {
s.showUndistorsed = !s.showUndistorsed;
}
if ( s.inputCapture.isOpened() && key == 'g' ) {
mode = CAPTURING;
imagePoints.clear();
}
}
// -----------------------Show the undistorted image for the image list ------------------------
if ( s.inputType == Settings::IMAGE_LIST && s.showUndistorsed ) {
Mat view, rview, map1, map2;
initUndistortRectifyMap( cameraMatrix, distCoeffs, Mat(),
getOptimalNewCameraMatrix( cameraMatrix, distCoeffs, imageSize, 1, imageSize, 0 ),
imageSize, CV_16SC2, map1, map2 );
for ( int i = 0; i < (int)s.imageList.size(); i++ ) {
view = imread( s.imageList[i], 1 );
if ( view.empty() ) {
continue;
}
remap( view, rview, map1, map2, INTER_LINEAR );
imshow( "Image View", rview );
char c = (char)waitKey();
if ( c == ESC_KEY || c == 'q' || c == 'Q' ) {
break;
}
}
}
}
int StereoCameraCalibration::handleStereoCalibration(struct StereoCameraCalibration::CalibrationConfigStruct struct_calibrationConfig)
{
//The capture for the image
VideoCapture leftCameraCapture;
VideoCapture rightCameraCapture;
//Used to signify the capture portion is done
int quitCapture = 0;
int lastKey = -1;
int key = -1;
double totalAvgErr;
double rmsErr;
CalibrationStates currentCalibrationState = CALIBRATION_STATE_INIT;
//A vector to hold the points found during calibration
vector< vector<Point2f> >calibPoints[CAMERA_LOCATION_RIGHT+1];
//How many frames we have
int frameCount = 0;
clock_t prevTimeStamp = clock();
Mat currentLeftFrame;
Mat currentRightFrame;
Mat leftCaptureFrame;
Mat rightCaptureFrame;
Mat flippedLeftFrame;
Mat flippedRightFrame;
Mat cameraMatrix[2];
Mat distCoeffs[2];
Mat R, T, R1, R2, P1, P2, Q;
Rect validRoi[2];
Mat rmap[2][2];
while(currentCalibrationState != CALIBRATION_STATE_DONE)
{
key = waitKey(33) & 0xff;
switch(currentCalibrationState)
{
case CALIBRATION_STATE_INIT:
{
//Open the captures
leftCameraCapture.open(struct_calibrationConfig.leftCameraId);
rightCameraCapture.open(struct_calibrationConfig.rightCameraId);
if(!(leftCameraCapture.set(CV_CAP_PROP_FPS, 30.0)))
{
cout << "Left frame rate set failed" << endl;
}
if(!(rightCameraCapture.set(CV_CAP_PROP_FPS, 30.0)))
{
cout << "Right frame rate set failed" << endl;
}
if(!(leftCameraCapture.set(CV_CAP_PROP_FRAME_WIDTH, 640)))
{
cout << "Left frame width set failed" << endl;
}
if(!(leftCameraCapture.set(CV_CAP_PROP_FRAME_HEIGHT, 480)))
{
cout << "Left frame height set failed" << endl;
}
if(!(rightCameraCapture.set(CV_CAP_PROP_FRAME_WIDTH, 640)))
{
cout << "Right frame width set failed" << endl;
}
if(!(rightCameraCapture.set(CV_CAP_PROP_FRAME_HEIGHT, 480)))
{
cout << "Right frame height set failed" << endl;
}
//Named window for calibration
namedWindow("Current Left Calibration Image Raw", 1);
namedWindow("Current Right Calibration Image Raw", 1);
//Named window for calibration
namedWindow("Current Left Calibration Image", 1);
namedWindow("Current Right Calibration Image", 1);
cout << "Starting calibration feature point capture." << endl;
currentCalibrationState = CALIBRATION_STATE_SHOW_IMAGE_BEFORE_CAPTURE;
break;
}
case CALIBRATION_STATE_SHOW_IMAGE_BEFORE_CAPTURE:
{
if(leftCameraCapture.isOpened() && rightCameraCapture.isOpened())
{
leftCameraCapture >> leftCaptureFrame;
rightCameraCapture >> rightCaptureFrame;
leftCaptureFrame.copyTo(currentLeftFrame);
rightCaptureFrame.copyTo(currentRightFrame);
}
//cout << "Left Frame Size" <<currentLeftFrame.rows <<"x" << currentLeftFrame.cols << endl;
//cout << "Right Frame Size" <<currentRightFrame.rows <<"x" << currentRightFrame.cols << endl;
if(!currentLeftFrame.data)
{
cout << "No Frame Data from Left Camera" << endl;
return 2;
}
if(!currentRightFrame.data)
{
cout << "No Frame Data from Right Camera" << endl;
return 2;
}
//currentFrame.copyTo(flippedFrame);
flip(currentLeftFrame, flippedLeftFrame,1);
flip(currentRightFrame, flippedRightFrame,1);
imshow("Current Left Calibration Image Raw", flippedLeftFrame);
imshow("Current Right Calibration Image Raw", flippedRightFrame);
if(key == 's' || key == ' ')
{
prevTimeStamp = clock();
currentCalibrationState = CALIBRATION_STATE_IMAGE_CAPTURE;
}
if(key == 27)
{
currentCalibrationState =CALIBRATION_STATE_ABORT;
}
break;
}
case CALIBRATION_STATE_IMAGE_CAPTURE:
{
if(leftCameraCapture.isOpened() && rightCameraCapture.isOpened())
{
leftCameraCapture >> leftCaptureFrame;
leftCaptureFrame.copyTo(currentLeftFrame);
rightCameraCapture >> rightCaptureFrame;
rightCaptureFrame.copyTo(currentRightFrame);
}
if(!currentLeftFrame.data)
{
cout << "No Frame Data from Left Camera" << endl;
return 2;
}
if(!currentRightFrame.data)
{
cout << "No Frame Data from Right Camera" << endl;
return 2;
}
Mat flippedLeftFrame;
Mat flippedRightFrame;
//currentFrame.copyTo(flippedFrame);
flip(currentLeftFrame, flippedLeftFrame,1);
flip(currentRightFrame, flippedRightFrame,1);
imshow("Current Left Calibration Image Raw", flippedRightFrame);
imshow("Current Right Calibration Image Raw", flippedLeftFrame);
Mat currentFrameGray[CAMERA_LOCATION_RIGHT+1];
cvtColor(currentLeftFrame,currentFrameGray[CAMERA_LOCATION_LEFT],CV_BGR2GRAY);
cvtColor(currentRightFrame,currentFrameGray[CAMERA_LOCATION_RIGHT],CV_BGR2GRAY);
vector<Point2f> currentFramePoints[CAMERA_LOCATION_RIGHT+1];
bool foundPoints[CAMERA_LOCATION_RIGHT+1];
//Find the corners of the Chessboard
foundPoints[CAMERA_LOCATION_LEFT] = findChessboardCorners( currentFrameGray[CAMERA_LOCATION_LEFT], struct_calibrationConfig.boardSize, currentFramePoints[CAMERA_LOCATION_LEFT], CV_CALIB_CB_ADAPTIVE_THRESH & CV_CALIB_CB_FAST_CHECK & CV_CALIB_CB_NORMALIZE_IMAGE);
foundPoints[CAMERA_LOCATION_RIGHT] = findChessboardCorners( currentFrameGray[CAMERA_LOCATION_RIGHT], struct_calibrationConfig.boardSize, currentFramePoints[CAMERA_LOCATION_RIGHT], CV_CALIB_CB_ADAPTIVE_THRESH & CV_CALIB_CB_FAST_CHECK & CV_CALIB_CB_NORMALIZE_IMAGE);
if(foundPoints[CAMERA_LOCATION_LEFT] || foundPoints[CAMERA_LOCATION_RIGHT])
{
if(foundPoints[CAMERA_LOCATION_LEFT])
{
cornerSubPix( currentFrameGray[CAMERA_LOCATION_LEFT], currentFramePoints[CAMERA_LOCATION_LEFT], Size(5,5), Size(-1,-1), TermCriteria( CV_TERMCRIT_EPS+CV_TERMCRIT_ITER, 30, 0.1 ));
}
if(foundPoints[CAMERA_LOCATION_RIGHT])
{
cornerSubPix( currentFrameGray[CAMERA_LOCATION_RIGHT], currentFramePoints[CAMERA_LOCATION_RIGHT], Size(5,5), Size(-1,-1), TermCriteria( CV_TERMCRIT_EPS+CV_TERMCRIT_ITER, 30, 0.1 ));
}
if(foundPoints[CAMERA_LOCATION_LEFT] && foundPoints[CAMERA_LOCATION_RIGHT])
{
if(clock() - prevTimeStamp > struct_calibrationConfig.delay*1e-3*CLOCKS_PER_SEC)
{
prevTimeStamp = clock();
//blink = capture.isOpened();
bitwise_not(currentLeftFrame, currentLeftFrame);
bitwise_not(currentRightFrame, currentRightFrame);
calibPoints[CAMERA_LOCATION_LEFT].push_back(currentFramePoints[CAMERA_LOCATION_LEFT]);
calibPoints[CAMERA_LOCATION_RIGHT].push_back(currentFramePoints[CAMERA_LOCATION_RIGHT]);
frameCount++;
}
}
if(foundPoints[CAMERA_LOCATION_LEFT])
{
drawChessboardCorners( currentLeftFrame, struct_calibrationConfig.boardSize, Mat(currentFramePoints[CAMERA_LOCATION_LEFT]), foundPoints[CAMERA_LOCATION_LEFT] );
}
if(foundPoints[CAMERA_LOCATION_RIGHT])
{
drawChessboardCorners( currentRightFrame, struct_calibrationConfig.boardSize, Mat(currentFramePoints[CAMERA_LOCATION_RIGHT]), foundPoints[CAMERA_LOCATION_RIGHT] );
}
cout << "Good Frames: " << frameCount << endl;
imshow("Current Left Calibration Image", currentLeftFrame);
imshow("Current Right Calibration Image", currentRightFrame);
struct_calibrationConfig.imageSize = currentLeftFrame.size();
}
else
{
imshow("Current Left Calibration Image", currentFrameGray[CAMERA_LOCATION_LEFT]);
imshow("Current Right Calibration Image", currentFrameGray[CAMERA_LOCATION_RIGHT]);
}
if((key == 'd' || key == 'c' || key == 'r' || key == ' ') && frameCount >= 15)
{
currentCalibrationState =CALIBRATION_STATE_RUN;
}
if(key==27)
{
currentCalibrationState =CALIBRATION_STATE_ABORT;
}
break;
}
case CALIBRATION_STATE_RUN:
{
bool ok = runStereoCalibration(calibPoints, struct_calibrationConfig.imageSize, struct_calibrationConfig.boardSize, struct_calibrationConfig.squareSize,
struct_calibrationConfig.aspectRatio, struct_calibrationConfig.flags, cameraMatrix, distCoeffs,
R, T, R1, P1, R2, P2, Q, validRoi, rmsErr, totalAvgErr);
printf("%s. avg reprojection error = %.2f\n",
ok ? "Calibration succeeded" : "Calibration failed",
totalAvgErr);
if(ok)
{
cout << "Moving to save option." << endl;
//Precompute maps for cv::remap()
initUndistortRectifyMap(cameraMatrix[CAMERA_LOCATION_LEFT], distCoeffs[CAMERA_LOCATION_LEFT], R1, P1, struct_calibrationConfig.imageSize, CV_16SC2, rmap[CAMERA_LOCATION_LEFT][CAMERA_REMAP_AXIS_X], rmap[CAMERA_LOCATION_LEFT][CAMERA_REMAP_AXIS_Y]);
initUndistortRectifyMap(cameraMatrix[CAMERA_LOCATION_RIGHT], distCoeffs[CAMERA_LOCATION_RIGHT], R2, P2, struct_calibrationConfig.imageSize, CV_16SC2, rmap[CAMERA_LOCATION_RIGHT][CAMERA_REMAP_AXIS_X], rmap[CAMERA_LOCATION_RIGHT][CAMERA_REMAP_AXIS_Y]);
currentCalibrationState =CALIBRATION_STATE_SAVE;
}
else
{
cout << "Moving to waiting for image capture." << endl;
currentCalibrationState =CALIBRATION_STATE_SHOW_IMAGE_BEFORE_CAPTURE;
}
break;
}
case CALIBRATION_STATE_SAVE:
{
key = displayRemappedStereoImages(leftCameraCapture, rightCameraCapture,validRoi,struct_calibrationConfig.imageSize,rmap);
bool ok = false;
vector<vector<Point2f> > noPoints[2];
if( key == 's' )
{
this->imageSize = struct_calibrationConfig.imageSize;
this->boardSize = struct_calibrationConfig.boardSize;
this->squareSize = struct_calibrationConfig.squareSize;
this ->aspectRatio = struct_calibrationConfig.aspectRatio;
this->flags = struct_calibrationConfig.flags;
this->cameraMatrix[0] = cameraMatrix[0].clone();
this->cameraMatrix[1] = cameraMatrix[1].clone();
this->distCoeffs[0] = distCoeffs[0].clone();
this->distCoeffs[1] = distCoeffs[1].clone();
this->R = R.clone();
this->T = T.clone();
this->R1 = R1.clone();
this->P1 = P1.clone();
this->R2 = R2.clone();
this->P2 = P2.clone();
this->Q = Q.clone();
this->validRoi[0] = validRoi[0];
this->validRoi[1] = validRoi[1];
this->rmsErr = rmsErr;
this->imagePoints[0] = calibPoints[0];
this->imagePoints[1] = calibPoints[1];
this->totalAvgErr = totalAvgErr;
saveStereoCameraParams( struct_calibrationConfig.outputFilename, struct_calibrationConfig.imageSize,
struct_calibrationConfig.boardSize, struct_calibrationConfig.squareSize, struct_calibrationConfig.aspectRatio,
struct_calibrationConfig.flags, cameraMatrix, distCoeffs,
R, T, R1, P1, R2, P2, Q,validRoi,
rmsErr,
struct_calibrationConfig.writePoints ? calibPoints : noPoints,
totalAvgErr );
cout << "Stereo Calibration Data Saved" << endl;
currentCalibrationState =CALIBRATION_STATE_COMPLETE;
}
if(key == 27)
{
cout << "Move to abort" << endl;
currentCalibrationState =CALIBRATION_STATE_ABORT;
}
break;
}
case CALIBRATION_STATE_ABORT:
cout << "Calibration Aborted" << endl;
currentCalibrationState =CALIBRATION_STATE_COMPLETE;
break;
case CALIBRATION_STATE_COMPLETE:
cout << "Calibration Completed" << endl;
currentCalibrationState =CALIBRATION_STATE_DONE;
break;
default:
break;
}//switch
