// Gets the hue/sat/val for areas that we believe are license plate characters
// Then uses that to filter the whole image and provide a mask.
void ColorFilter::findCharColors()
{
int MINIMUM_SATURATION = 45;
if (this->debug)
cout << "ColorFilter::findCharColors" << endl;
//charMask.copyTo(this->colorMask);
this->colorMask = Mat::zeros(charMask.size(), CV_8U);
bitwise_not(this->colorMask, this->colorMask);
Mat erodedCharMask(charMask.size(), CV_8U);
Mat element = getStructuringElement( 1,
Size( 2 + 1, 2+1 ),
Point( 1, 1 ) );
erode(charMask, erodedCharMask, element);
vector<vector<Point> > contours;
vector<Vec4i> hierarchy;
findContours(erodedCharMask, contours, hierarchy, CV_RETR_TREE, CV_CHAIN_APPROX_SIMPLE);
vector<float> hMeans, sMeans, vMeans;
vector<float> hStdDevs, sStdDevs, vStdDevs;
for (unsigned int i = 0; i < contours.size(); i++)
{
if (hierarchy[i][3] != -1)
continue;
Mat singleCharMask = Mat::zeros(hsv.size(), CV_8U);
drawContours(singleCharMask, contours,
i, // draw this contour
cv::Scalar(255,255,255), // in
CV_FILLED,
8,
hierarchy
);
// get rid of the outline by drawing a 1 pixel width black line
drawContours(singleCharMask, contours,
i, // draw this contour
cv::Scalar(0,0,0), // in
1,
8,
hierarchy
);
//drawAndWait(&singleCharMask);
Scalar mean;
Scalar stddev;
meanStdDev(hsv, mean, stddev, singleCharMask);
if (this->debug)
{
cout << "ColorFilter " << setw(3) << i << ". Mean: h: " << setw(7) << mean[0] << " s: " << setw(7) <<mean[1] << " v: " << setw(7) << mean[2]
<< " | Std: h: " << setw(7) <<stddev[0] << " s: " << setw(7) <<stddev[1] << " v: " << stddev[2] << endl;
}
if (mean[0] == 0 && mean[1] == 0 && mean[2] == 0)
continue;
hMeans.push_back(mean[0]);
sMeans.push_back(mean[1]);
vMeans.push_back(mean[2]);
hStdDevs.push_back(stddev[0]);
sStdDevs.push_back(stddev[1]);
vStdDevs.push_back(stddev[2]);
}
if (hMeans.size() == 0)
return;
int bestHueIndex = this->getMajorityOpinion(hMeans, .65, 30);
int bestSatIndex = this->getMajorityOpinion(sMeans, .65, 35);
int bestValIndex = this->getMajorityOpinion(vMeans, .65, 30);
if (sMeans[bestSatIndex] < MINIMUM_SATURATION)
return;
bool doHueFilter = false, doSatFilter = false, doValFilter = false;
float hueMin, hueMax;
float satMin, satMax;
float valMin, valMax;
if (this->debug)
cout << "ColorFilter Winning indices:" << endl;
if (bestHueIndex != -1)
{
doHueFilter = true;
hueMin = hMeans[bestHueIndex] - (2 * hStdDevs[bestHueIndex]);
hueMax = hMeans[bestHueIndex] + (2 * hStdDevs[bestHueIndex]);
if (abs(hueMin - hueMax) < 20)
{
hueMin = hMeans[bestHueIndex] - 20;
hueMax = hMeans[bestHueIndex] + 20;
}
if (hueMin < 0)
hueMin = 0;
if (hueMax > 180)
hueMax = 180;
if (this->debug)
cout << "ColorFilter Hue: " << bestHueIndex << " : " << setw(7) << hMeans[bestHueIndex] << " -- " << hueMin << "-" << hueMax << endl;
}
if (bestSatIndex != -1)
{
doSatFilter = true;
satMin = sMeans[bestSatIndex] - (2 * sStdDevs[bestSatIndex]);
satMax = sMeans[bestSatIndex] + (2 * sStdDevs[bestSatIndex]);
if (abs(satMin - satMax) < 20)
{
satMin = sMeans[bestSatIndex] - 20;
satMax = sMeans[bestSatIndex] + 20;
}
if (satMin < 0)
satMin = 0;
if (satMax > 255)
satMax = 255;
if (this->debug)
cout << "ColorFilter Sat: " << bestSatIndex << " : " << setw(7) << sMeans[bestSatIndex] << " -- " << satMin << "-" << satMax << endl;
}
if (bestValIndex != -1)
{
doValFilter = true;
valMin = vMeans[bestValIndex] - (1.5 * vStdDevs[bestValIndex]);
valMax = vMeans[bestValIndex] + (1.5 * vStdDevs[bestValIndex]);
if (abs(valMin - valMax) < 20)
{
valMin = vMeans[bestValIndex] - 20;
valMax = vMeans[bestValIndex] + 20;
}
if (valMin < 0)
valMin = 0;
if (valMax > 255)
valMax = 255;
if (this->debug)
cout << "ColorFilter Val: " << bestValIndex << " : " << setw(7) << vMeans[bestValIndex] << " -- " << valMin << "-" << valMax << endl;
}
Mat imgDebugHueOnly = Mat::zeros(hsv.size(), hsv.type());
Mat imgDebug = Mat::zeros(hsv.size(), hsv.type());
Mat imgDistanceFromCenter = Mat::zeros(hsv.size(), CV_8U);
Mat debugMask = Mat::zeros(hsv.size(), CV_8U);
bitwise_not(debugMask, debugMask);
for (int row = 0; row < charMask.rows; row++)
{
for (int col = 0; col < charMask.cols; col++)
{
int h = (int) hsv.at<Vec3b>(row, col)[0];
int s = (int) hsv.at<Vec3b>(row, col)[1];
int v = (int) hsv.at<Vec3b>(row, col)[2];
bool hPasses = true;
bool sPasses = true;
bool vPasses = true;
int vDistance = abs(v - vMeans[bestValIndex]);
imgDebugHueOnly.at<Vec3b>(row, col)[0] = h;
imgDebugHueOnly.at<Vec3b>(row, col)[1] = 255;
imgDebugHueOnly.at<Vec3b>(row, col)[2] = 255;
imgDebug.at<Vec3b>(row, col)[0] = 255;
imgDebug.at<Vec3b>(row, col)[1] = 255;
imgDebug.at<Vec3b>(row, col)[2] = 255;
if (doHueFilter && (h < hueMin || h > hueMax))
{
hPasses = false;
imgDebug.at<Vec3b>(row, col)[0] = 0;
debugMask.at<uchar>(row, col) = 0;
}
if (doSatFilter && (s < satMin || s > satMax))
{
sPasses = false;
imgDebug.at<Vec3b>(row, col)[1] = 0;
}
if (doValFilter && (v < valMin || v > valMax))
{
vPasses = false;
imgDebug.at<Vec3b>(row, col)[2] = 0;
}
//if (pixelPasses)
// colorMask.at<uchar>(row, col) = 255;
//else
//imgDebug.at<Vec3b>(row, col)[0] = hPasses & 255;
//imgDebug.at<Vec3b>(row, col)[1] = sPasses & 255;
//imgDebug.at<Vec3b>(row, col)[2] = vPasses & 255;
if ((hPasses) || (hPasses && sPasses))//(hPasses && vPasses) || (sPasses && vPasses) ||
this->colorMask.at<uchar>(row, col) = 255;
else
this->colorMask.at<uchar>(row, col) = 0;
if ((hPasses && sPasses) || (hPasses && vPasses) || (sPasses && vPasses))
{
vDistance = pow(vDistance, 0.9);
}
else
{
vDistance = pow(vDistance, 1.1);
}
if (vDistance > 255)
vDistance = 255;
imgDistanceFromCenter.at<uchar>(row, col) = vDistance;
}
}
vector<Mat> debugImagesSet;
if (this->debug)
{
debugImagesSet.push_back(addLabel(charMask, "Charecter mask"));
//debugImagesSet1.push_back(erodedCharMask);
Mat maskCopy(colorMask.size(), colorMask.type());
colorMask.copyTo(maskCopy);
debugImagesSet.push_back(addLabel(maskCopy, "color Mask Before"));
}
Mat bigElement = getStructuringElement( 1,
Size( 3 + 1, 3+1 ),
Point( 1, 1 ) );
Mat smallElement = getStructuringElement( 1,
Size( 1 + 1, 1+1 ),
Point( 1, 1 ) );
morphologyEx(this->colorMask, this->colorMask, MORPH_CLOSE, bigElement);
//dilate(this->colorMask, this->colorMask, bigElement);
Mat combined(charMask.size(), charMask.type());
bitwise_and(charMask, colorMask, combined);
if (this->debug)
{
debugImagesSet.push_back(addLabel(colorMask, "Color Mask After"));
debugImagesSet.push_back(addLabel(combined, "Combined"));
//displayImage(config, "COLOR filter Mask", colorMask);
debugImagesSet.push_back(addLabel(imgDebug, "Color filter Debug"));
cvtColor(imgDebugHueOnly, imgDebugHueOnly, CV_HSV2BGR);
debugImagesSet.push_back(addLabel(imgDebugHueOnly, "Color Filter Hue"));
equalizeHist(imgDistanceFromCenter, imgDistanceFromCenter);
debugImagesSet.push_back(addLabel(imgDistanceFromCenter, "COLOR filter Distance"));
debugImagesSet.push_back(addLabel(debugMask, "COLOR Hues off"));
Mat dashboard = drawImageDashboard(debugImagesSet, imgDebugHueOnly.type(), 3);
displayImage(config, "Color Filter Images", dashboard);
}
}
// Tries to find a rectangular area surrounding most of the characters. Not required
// but helpful when determining the plate edges
void PlateMask::findOuterBoxMask( vector<TextContours > contours )
{
double min_parent_area = pipeline_data->config->templateHeightPx * pipeline_data->config->templateWidthPx * 0.10; // Needs to be at least 10% of the plate area to be considered.
int winningIndex = -1;
int winningParentId = -1;
int bestCharCount = 0;
double lowestArea = 99999999999999;
if (pipeline_data->config->debugCharAnalysis)
cout << "CharacterAnalysis::findOuterBoxMask" << endl;
for (unsigned int imgIndex = 0; imgIndex < contours.size(); imgIndex++)
{
//vector<bool> charContours = filter(thresholds[imgIndex], allContours[imgIndex], allHierarchy[imgIndex]);
int charsRecognized = 0;
int parentId = -1;
bool hasParent = false;
for (unsigned int i = 0; i < contours[imgIndex].goodIndices.size(); i++)
{
if (contours[imgIndex].goodIndices[i]) charsRecognized++;
if (contours[imgIndex].goodIndices[i] && contours[imgIndex].hierarchy[i][3] != -1)
{
parentId = contours[imgIndex].hierarchy[i][3];
hasParent = true;
}
}
if (charsRecognized == 0)
continue;
if (hasParent)
{
double boxArea = contourArea(contours[imgIndex].contours[parentId]);
if (boxArea < min_parent_area)
continue;
if ((charsRecognized > bestCharCount) ||
(charsRecognized == bestCharCount && boxArea < lowestArea))
//(boxArea < lowestArea)
{
bestCharCount = charsRecognized;
winningIndex = imgIndex;
winningParentId = parentId;
lowestArea = boxArea;
}
}
}
if (pipeline_data->config->debugCharAnalysis)
cout << "Winning image index (findOuterBoxMask) is: " << winningIndex << endl;
if (winningIndex != -1 && bestCharCount >= 3)
{
Mat mask = Mat::zeros(pipeline_data->thresholds[winningIndex].size(), CV_8U);
// get rid of the outline by drawing a 1 pixel width black line
drawContours(mask, contours[winningIndex].contours,
winningParentId, // draw this contour
cv::Scalar(255,255,255), // in
FILLED,
8,
contours[winningIndex].hierarchy,
0
);
// Morph Open the mask to get rid of any little connectors to non-plate portions
int morph_elem = 2;
int morph_size = 3;
Mat element = getStructuringElement( morph_elem, Size( 2*morph_size + 1, 2*morph_size+1 ), Point( morph_size, morph_size ) );
//morphologyEx( mask, mask, MORPH_CLOSE, element );
morphologyEx( mask, mask, MORPH_OPEN, element );
//morph_size = 1;
//element = getStructuringElement( morph_elem, Size( 2*morph_size + 1, 2*morph_size+1 ), Point( morph_size, morph_size ) );
//dilate(mask, mask, element);
// Drawing the edge black effectively erodes the image. This may clip off some extra junk from the edges.
// We'll want to do the contour again and find the larges one so that we remove the clipped portion.
vector<vector<Point> > contoursSecondRound;
findContours(mask, contoursSecondRound, RETR_EXTERNAL, CHAIN_APPROX_SIMPLE);
int biggestContourIndex = -1;
double largestArea = 0;
for (unsigned int c = 0; c < contoursSecondRound.size(); c++)
{
double area = contourArea(contoursSecondRound[c]);
if (area > largestArea)
{
biggestContourIndex = c;
largestArea = area;
}
}
if (biggestContourIndex != -1)
{
mask = Mat::zeros(pipeline_data->thresholds[winningIndex].size(), CV_8U);
vector<Point> smoothedMaskPoints;
approxPolyDP(contoursSecondRound[biggestContourIndex], smoothedMaskPoints, 2, true);
vector<vector<Point> > tempvec;
tempvec.push_back(smoothedMaskPoints);
//fillPoly(mask, smoothedMaskPoints.data(), smoothedMaskPoints, Scalar(255,255,255));
drawContours(mask, tempvec,
0, // draw this contour
cv::Scalar(255,255,255), // in
FILLED,
8,
contours[winningIndex].hierarchy,
0
);
}
if (pipeline_data->config->debugCharAnalysis)
{
vector<Mat> debugImgs;
Mat debugImgMasked = Mat::zeros(pipeline_data->thresholds[winningIndex].size(), CV_8U);
pipeline_data->thresholds[winningIndex].copyTo(debugImgMasked, mask);
debugImgs.push_back(mask);
debugImgs.push_back(pipeline_data->thresholds[winningIndex]);
debugImgs.push_back(debugImgMasked);
Mat dashboard = drawImageDashboard(debugImgs, CV_8U, 1);
displayImage(pipeline_data->config, "Winning outer box", dashboard);
}
hasPlateMask = true;
this->plateMask = mask;
} else {
hasPlateMask = false;
Mat fullMask = Mat::zeros(pipeline_data->thresholds[0].size(), CV_8U);
bitwise_not(fullMask, fullMask);
this->plateMask = fullMask;
}
}
void CharacterAnalysis::analyze()
{
timespec startTime;
getTimeMonotonic(&startTime);
pipeline_data->clearThresholds();
pipeline_data->thresholds = produceThresholds(pipeline_data->crop_gray, config);
timespec contoursStartTime;
getTimeMonotonic(&contoursStartTime);
pipeline_data->textLines.clear();
for (unsigned int i = 0; i < pipeline_data->thresholds.size(); i++)
{
TextContours tc(pipeline_data->thresholds[i]);
allTextContours.push_back(tc);
}
if (config->debugTiming)
{
timespec contoursEndTime;
getTimeMonotonic(&contoursEndTime);
cout << " -- Character Analysis Find Contours Time: " << diffclock(contoursStartTime, contoursEndTime) << "ms." << endl;
}
//Mat img_equalized = equalizeBrightness(img_gray);
timespec filterStartTime;
getTimeMonotonic(&filterStartTime);
for (unsigned int i = 0; i < pipeline_data->thresholds.size(); i++)
{
this->filter(pipeline_data->thresholds[i], allTextContours[i]);
if (config->debugCharAnalysis)
cout << "Threshold " << i << " had " << allTextContours[i].getGoodIndicesCount() << " good indices." << endl;
}
if (config->debugTiming)
{
timespec filterEndTime;
getTimeMonotonic(&filterEndTime);
cout << " -- Character Analysis Filter Time: " << diffclock(filterStartTime, filterEndTime) << "ms." << endl;
}
PlateMask plateMask(pipeline_data);
plateMask.findOuterBoxMask(allTextContours);
pipeline_data->hasPlateBorder = plateMask.hasPlateMask;
pipeline_data->plateBorderMask = plateMask.getMask();
if (plateMask.hasPlateMask)
{
// Filter out bad contours now that we have an outer box mask...
for (unsigned int i = 0; i < pipeline_data->thresholds.size(); i++)
{
filterByOuterMask(allTextContours[i]);
}
}
int bestFitScore = -1;
int bestFitIndex = -1;
for (unsigned int i = 0; i < pipeline_data->thresholds.size(); i++)
{
int segmentCount = allTextContours[i].getGoodIndicesCount();
if (segmentCount > bestFitScore)
{
bestFitScore = segmentCount;
bestFitIndex = i;
bestThreshold = pipeline_data->thresholds[i];
bestContours = allTextContours[i];
}
}
if (this->config->debugCharAnalysis)
cout << "Best fit score: " << bestFitScore << " Index: " << bestFitIndex << endl;
if (bestFitScore <= 1)
{
pipeline_data->disqualified = true;
pipeline_data->disqualify_reason = "Low best fit score in characteranalysis";
return;
}
//getColorMask(img, allContours, allHierarchy, charSegments);
if (this->config->debugCharAnalysis)
{
Mat img_contours = bestContours.drawDebugImage(bestThreshold);
displayImage(config, "Matching Contours", img_contours);
}
LineFinder lf(pipeline_data);
vector<vector<Point> > linePolygons = lf.findLines(pipeline_data->crop_gray, bestContours);
vector<TextLine> tempTextLines;
for (unsigned int i = 0; i < linePolygons.size(); i++)
{
vector<Point> linePolygon = linePolygons[i];
LineSegment topLine = LineSegment(linePolygon[0].x, linePolygon[0].y, linePolygon[1].x, linePolygon[1].y);
LineSegment bottomLine = LineSegment(linePolygon[3].x, linePolygon[3].y, linePolygon[2].x, linePolygon[2].y);
vector<Point> textArea = getCharArea(topLine, bottomLine);
TextLine textLine(textArea, linePolygon, pipeline_data->crop_gray.size());
tempTextLines.push_back(textLine);
}
filterBetweenLines(bestThreshold, bestContours, tempTextLines);
// Sort the lines from top to bottom.
std::sort(tempTextLines.begin(), tempTextLines.end(), sort_text_line);
// Now that we've filtered a few more contours, re-do the text area.
for (unsigned int i = 0; i < tempTextLines.size(); i++)
{
vector<Point> updatedTextArea = getCharArea(tempTextLines[i].topLine, tempTextLines[i].bottomLine);
vector<Point> linePolygon = tempTextLines[i].linePolygon;
if (updatedTextArea.size() > 0 && linePolygon.size() > 0)
{
pipeline_data->textLines.push_back(TextLine(updatedTextArea, linePolygon, pipeline_data->crop_gray.size()));
}
}
pipeline_data->plate_inverted = isPlateInverted();
if (pipeline_data->textLines.size() > 0)
{
int confidenceDrainers = 0;
int charSegmentCount = this->bestContours.getGoodIndicesCount();
if (charSegmentCount == 1)
confidenceDrainers += 91;
else if (charSegmentCount < 5)
confidenceDrainers += (5 - charSegmentCount) * 10;
// Use the angle for the first line -- assume they'll always be parallel for multi-line plates
int absangle = abs(pipeline_data->textLines[0].topLine.angle);
if (absangle > config->maxPlateAngleDegrees)
confidenceDrainers += 91;
else if (absangle > 1)
confidenceDrainers += (config->maxPlateAngleDegrees - absangle) ;
// If a multiline plate has only one line, disqualify
if (pipeline_data->isMultiline && pipeline_data->textLines.size() < 2)
{
if (config->debugCharAnalysis)
std::cout << "Did not detect multiple lines on multi-line plate" << std::endl;
confidenceDrainers += 95;
}
if (confidenceDrainers >= 90)
{
pipeline_data->disqualified = true;
pipeline_data->disqualify_reason = "Low confidence in characteranalysis";
}
else
{
float confidence = 100 - confidenceDrainers;
pipeline_data->confidence_weights.setScore("CHARACTER_ANALYSIS_SCORE", confidence, 1.0);
}
}
else
{
pipeline_data->disqualified = true;
pipeline_data->disqualify_reason = "No text lines found in characteranalysis";
}
if (config->debugTiming)
{
timespec endTime;
getTimeMonotonic(&endTime);
cout << "Character Analysis Time: " << diffclock(startTime, endTime) << "ms." << endl;
}
// Draw debug dashboard
if (this->pipeline_data->config->debugCharAnalysis && pipeline_data->textLines.size() > 0)
{
vector<Mat> tempDash;
for (unsigned int z = 0; z < pipeline_data->thresholds.size(); z++)
{
Mat tmp(pipeline_data->thresholds[z].size(), pipeline_data->thresholds[z].type());
pipeline_data->thresholds[z].copyTo(tmp);
cvtColor(tmp, tmp, CV_GRAY2BGR);
tempDash.push_back(tmp);
}
Mat bestVal(this->bestThreshold.size(), this->bestThreshold.type());
this->bestThreshold.copyTo(bestVal);
cvtColor(bestVal, bestVal, CV_GRAY2BGR);
for (unsigned int z = 0; z < this->bestContours.size(); z++)
{
Scalar dcolor(255,0,0);
if (this->bestContours.goodIndices[z])
dcolor = Scalar(0,255,0);
drawContours(bestVal, this->bestContours.contours, z, dcolor, 1);
}
tempDash.push_back(bestVal);
displayImage(config, "Character Region Step 1 Thresholds", drawImageDashboard(tempDash, bestVal.type(), 3));
}
}
void PlateLines::processImage(Mat inputImage, vector<TextLine> textLines, float sensitivity)
{
if (this->debug)
cout << "PlateLines findLines" << endl;
timespec startTime;
getTimeMonotonic(&startTime);
// Ignore input images that are pure white or pure black
Scalar avgPixelIntensity = mean(inputImage);
if (avgPixelIntensity[0] >= 252)
return;
else if (avgPixelIntensity[0] <= 3)
return;
// Do a bilateral filter to clean the noise but keep edges sharp
Mat smoothed(inputImage.size(), inputImage.type());
adaptiveBilateralFilter(inputImage, smoothed, Size(3,3), 45, 45);
int morph_elem = 2;
int morph_size = 2;
Mat element = getStructuringElement( morph_elem, Size( 2*morph_size + 1, 2*morph_size+1 ), Point( morph_size, morph_size ) );
Mat edges(inputImage.size(), inputImage.type());
Canny(smoothed, edges, 66, 133);
// Create a mask that is dilated based on the detected characters
Mat mask = Mat::zeros(inputImage.size(), CV_8U);
for (unsigned int i = 0; i < textLines.size(); i++)
{
vector<vector<Point> > polygons;
polygons.push_back(textLines[i].textArea);
fillPoly(mask, polygons, Scalar(255,255,255));
}
dilate(mask, mask, getStructuringElement( 1, Size( 1 + 1, 2*1+1 ), Point( 1, 1 ) ));
bitwise_not(mask, mask);
// AND canny edges with the character mask
bitwise_and(edges, mask, edges);
vector<PlateLine> hlines = this->getLines(edges, sensitivity, false);
vector<PlateLine> vlines = this->getLines(edges, sensitivity, true);
for (unsigned int i = 0; i < hlines.size(); i++)
this->horizontalLines.push_back(hlines[i]);
for (unsigned int i = 0; i < vlines.size(); i++)
this->verticalLines.push_back(vlines[i]);
// if debug is enabled, draw the image
if (this->debug)
{
Mat debugImgHoriz(edges.size(), edges.type());
Mat debugImgVert(edges.size(), edges.type());
edges.copyTo(debugImgHoriz);
edges.copyTo(debugImgVert);
cvtColor(debugImgHoriz,debugImgHoriz,CV_GRAY2BGR);
cvtColor(debugImgVert,debugImgVert,CV_GRAY2BGR);
for( size_t i = 0; i < this->horizontalLines.size(); i++ )
{
line( debugImgHoriz, this->horizontalLines[i].line.p1, this->horizontalLines[i].line.p2, Scalar(0,0,255), 1, CV_AA);
}
for( size_t i = 0; i < this->verticalLines.size(); i++ )
{
line( debugImgVert, this->verticalLines[i].line.p1, this->verticalLines[i].line.p2, Scalar(0,0,255), 1, CV_AA);
}
vector<Mat> images;
images.push_back(debugImgHoriz);
images.push_back(debugImgVert);
Mat dashboard = drawImageDashboard(images, debugImgVert.type(), 1);
displayImage(pipelineData->config, "Hough Lines", dashboard);
}
if (pipelineData->config->debugTiming)
{
timespec endTime;
getTimeMonotonic(&endTime);
cout << "Plate Lines Time: " << diffclock(startTime, endTime) << "ms." << endl;
}
}
CharacterSegmenter::CharacterSegmenter(Mat img, bool invertedColors, Config* config)
{
this->config = config;
this->confidence = 0;
if (this->config->debugCharSegmenter)
cout << "Starting CharacterSegmenter" << endl;
//CharacterRegion charRegion(img, debug);
timespec startTime;
getTime(&startTime);
Mat img_gray(img.size(), CV_8U);
cvtColor( img, img_gray, CV_BGR2GRAY );
//normalize(img_gray, img_gray, 0, 255, CV_MINMAX );
medianBlur(img_gray, img_gray, 3);
if (invertedColors)
bitwise_not(img_gray, img_gray);
charAnalysis = new CharacterAnalysis(img_gray, config);
charAnalysis->analyze();
if (this->config->debugCharSegmenter)
{
displayImage(config, "CharacterSegmenter Thresholds", drawImageDashboard(charAnalysis->thresholds, CV_8U, 3));
}
if (this->config->debugCharSegmenter)
{
Mat img_contours(charAnalysis->bestThreshold.size(), CV_8U);
charAnalysis->bestThreshold.copyTo(img_contours);
cvtColor(img_contours, img_contours, CV_GRAY2RGB);
vector<vector<Point> > allowedContours;
for (int i = 0; i < charAnalysis->bestContours.size(); i++)
{
if (charAnalysis->bestCharSegments[i])
allowedContours.push_back(charAnalysis->bestContours[i]);
}
drawContours(img_contours, charAnalysis->bestContours,
-1, // draw all contours
cv::Scalar(255,0,0), // in blue
1); // with a thickness of 1
drawContours(img_contours, allowedContours,
-1, // draw all contours
cv::Scalar(0,255,0), // in green
1); // with a thickness of 1
if (charAnalysis->linePolygon.size() > 0)
{
line(img_contours, charAnalysis->linePolygon[0], charAnalysis->linePolygon[1], Scalar(255, 0, 255), 1);
line(img_contours, charAnalysis->linePolygon[3], charAnalysis->linePolygon[2], Scalar(255, 0, 255), 1);
}
Mat bordered = addLabel(img_contours, "Best Contours");
imgDbgGeneral.push_back(bordered);
}
// Figure out the average character width
float totalCharWidth = 0;
float totalCharHeight = 0;
if (charAnalysis->linePolygon.size() > 0)
{
this->top = LineSegment(charAnalysis->linePolygon[0].x, charAnalysis->linePolygon[0].y, charAnalysis->linePolygon[1].x, charAnalysis->linePolygon[1].y);
this->bottom = LineSegment(charAnalysis->linePolygon[3].x, charAnalysis->linePolygon[3].y, charAnalysis->linePolygon[2].x, charAnalysis->linePolygon[2].y);
for (int i = 0; i < charAnalysis->bestContours.size(); i++)
{
if (charAnalysis->bestCharSegments[i] == false)
continue;
Rect mr = boundingRect(charAnalysis->bestContours[i]);
totalCharWidth += mr.width;
totalCharHeight += mr.height;
}
int numSamples = charAnalysis->bestCharSegmentsCount;
float avgCharWidth = totalCharWidth / numSamples;
float avgCharHeight = totalCharHeight / numSamples;
removeSmallContours(charAnalysis->thresholds, charAnalysis->allContours, avgCharWidth, avgCharHeight);
// Do the histogram analysis to figure out char regions
timespec startTime;
getTime(&startTime);
vector<Mat> allHistograms;
vector<Rect> allBoxes;
for (int i = 0; i < charAnalysis->allContours.size(); i++)
{
Mat histogramMask = Mat::zeros(charAnalysis->thresholds[i].size(), CV_8U);
fillConvexPoly(histogramMask, charAnalysis->linePolygon.data(), charAnalysis->linePolygon.size(), Scalar(255,255,255));
VerticalHistogram vertHistogram(charAnalysis->thresholds[i], histogramMask);
if (this->config->debugCharSegmenter)
{
Mat histoCopy(vertHistogram.debugImg.size(), vertHistogram.debugImg.type());
//vertHistogram.copyTo(histoCopy);
cvtColor(vertHistogram.debugImg, histoCopy, CV_GRAY2RGB);
allHistograms.push_back(histoCopy);
}
//
float score = 0;
vector<Rect> charBoxes = getHistogramBoxes(vertHistogram.debugImg, avgCharWidth, avgCharHeight, &score);
if (this->config->debugCharSegmenter)
{
for (int cboxIdx = 0; cboxIdx < charBoxes.size(); cboxIdx++)
{
rectangle(allHistograms[i], charBoxes[cboxIdx], Scalar(0, 255, 0));
}
Mat histDashboard = drawImageDashboard(allHistograms, allHistograms[0].type(), 3);
displayImage(config, "Char seg histograms", histDashboard);
}
for (int z = 0; z < charBoxes.size(); z++)
allBoxes.push_back(charBoxes[z]);
//drawAndWait(&histogramMask);
}
float biggestCharWidth = avgCharWidth;
// Compute largest char width
for (int i = 0; i < allBoxes.size(); i++)
{
if (allBoxes[i].width > biggestCharWidth)
biggestCharWidth = allBoxes[i].width;
}
if (config->debugTiming)
{
timespec endTime;
getTime(&endTime);
cout << " -- Character Segmentation Create and Score Histograms Time: " << diffclock(startTime, endTime) << "ms." << endl;
}
//ColorFilter colorFilter(img, charAnalysis->getCharacterMask());
vector<Rect> candidateBoxes = getBestCharBoxes(charAnalysis->thresholds[0], allBoxes, biggestCharWidth);
if (this->config->debugCharSegmenter)
{
// Setup the dashboard images to show the cleaning filters
for (int i = 0; i < charAnalysis->thresholds.size(); i++)
{
Mat cleanImg = Mat::zeros(charAnalysis->thresholds[i].size(), charAnalysis->thresholds[i].type());
Mat boxMask = getCharBoxMask(charAnalysis->thresholds[i], candidateBoxes);
charAnalysis->thresholds[i].copyTo(cleanImg);
bitwise_and(cleanImg, boxMask, cleanImg);
cvtColor(cleanImg, cleanImg, CV_GRAY2BGR);
for (int c = 0; c < candidateBoxes.size(); c++)
rectangle(cleanImg, candidateBoxes[c], Scalar(0, 255, 0), 1);
imgDbgCleanStages.push_back(cleanImg);
}
}
getTime(&startTime);
filterEdgeBoxes(charAnalysis->thresholds, candidateBoxes, biggestCharWidth, avgCharHeight);
candidateBoxes = filterMostlyEmptyBoxes(charAnalysis->thresholds, candidateBoxes);
candidateBoxes = combineCloseBoxes(candidateBoxes, biggestCharWidth);
cleanCharRegions(charAnalysis->thresholds, candidateBoxes);
cleanMostlyFullBoxes(charAnalysis->thresholds, candidateBoxes);
//cleanBasedOnColor(thresholds, colorFilter.colorMask, candidateBoxes);
candidateBoxes = filterMostlyEmptyBoxes(charAnalysis->thresholds, candidateBoxes);
this->characters = candidateBoxes;
if (config->debugTiming)
{
timespec endTime;
getTime(&endTime);
cout << " -- Character Segmentation Box cleaning/filtering Time: " << diffclock(startTime, endTime) << "ms." << endl;
}
if (this->config->debugCharSegmenter)
{
Mat imgDash = drawImageDashboard(charAnalysis->thresholds, CV_8U, 3);
displayImage(config, "Segmentation after cleaning", imgDash);
Mat generalDash = drawImageDashboard(this->imgDbgGeneral, this->imgDbgGeneral[0].type(), 2);
displayImage(config, "Segmentation General", generalDash);
Mat cleanImgDash = drawImageDashboard(this->imgDbgCleanStages, this->imgDbgCleanStages[0].type(), 3);
displayImage(config, "Segmentation Clean Filters", cleanImgDash);
}
}
if (config->debugTiming)
{
timespec endTime;
getTime(&endTime);
cout << "Character Segmenter Time: " << diffclock(startTime, endTime) << "ms." << endl;
}
}
