// Gets the hue/sat/val for areas that we believe are license plate characters // Then uses that to filter the whole image and provide a mask. void ColorFilter::findCharColors() { int MINIMUM_SATURATION = 45; if (this->debug) cout << "ColorFilter::findCharColors" << endl; //charMask.copyTo(this->colorMask); this->colorMask = Mat::zeros(charMask.size(), CV_8U); bitwise_not(this->colorMask, this->colorMask); Mat erodedCharMask(charMask.size(), CV_8U); Mat element = getStructuringElement( 1, Size( 2 + 1, 2+1 ), Point( 1, 1 ) ); erode(charMask, erodedCharMask, element); vector<vector<Point> > contours; vector<Vec4i> hierarchy; findContours(erodedCharMask, contours, hierarchy, CV_RETR_TREE, CV_CHAIN_APPROX_SIMPLE); vector<float> hMeans, sMeans, vMeans; vector<float> hStdDevs, sStdDevs, vStdDevs; for (unsigned int i = 0; i < contours.size(); i++) { if (hierarchy[i][3] != -1) continue; Mat singleCharMask = Mat::zeros(hsv.size(), CV_8U); drawContours(singleCharMask, contours, i, // draw this contour cv::Scalar(255,255,255), // in CV_FILLED, 8, hierarchy ); // get rid of the outline by drawing a 1 pixel width black line drawContours(singleCharMask, contours, i, // draw this contour cv::Scalar(0,0,0), // in 1, 8, hierarchy ); //drawAndWait(&singleCharMask); Scalar mean; Scalar stddev; meanStdDev(hsv, mean, stddev, singleCharMask); if (this->debug) { cout << "ColorFilter " << setw(3) << i << ". Mean: h: " << setw(7) << mean[0] << " s: " << setw(7) <<mean[1] << " v: " << setw(7) << mean[2] << " | Std: h: " << setw(7) <<stddev[0] << " s: " << setw(7) <<stddev[1] << " v: " << stddev[2] << endl; } if (mean[0] == 0 && mean[1] == 0 && mean[2] == 0) continue; hMeans.push_back(mean[0]); sMeans.push_back(mean[1]); vMeans.push_back(mean[2]); hStdDevs.push_back(stddev[0]); sStdDevs.push_back(stddev[1]); vStdDevs.push_back(stddev[2]); } if (hMeans.size() == 0) return; int bestHueIndex = this->getMajorityOpinion(hMeans, .65, 30); int bestSatIndex = this->getMajorityOpinion(sMeans, .65, 35); int bestValIndex = this->getMajorityOpinion(vMeans, .65, 30); if (sMeans[bestSatIndex] < MINIMUM_SATURATION) return; bool doHueFilter = false, doSatFilter = false, doValFilter = false; float hueMin, hueMax; float satMin, satMax; float valMin, valMax; if (this->debug) cout << "ColorFilter Winning indices:" << endl; if (bestHueIndex != -1) { doHueFilter = true; hueMin = hMeans[bestHueIndex] - (2 * hStdDevs[bestHueIndex]); hueMax = hMeans[bestHueIndex] + (2 * hStdDevs[bestHueIndex]); if (abs(hueMin - hueMax) < 20) { hueMin = hMeans[bestHueIndex] - 20; hueMax = hMeans[bestHueIndex] + 20; } if (hueMin < 0) hueMin = 0; if (hueMax > 180) hueMax = 180; if (this->debug) cout << "ColorFilter Hue: " << bestHueIndex << " : " << setw(7) << hMeans[bestHueIndex] << " -- " << hueMin << "-" << hueMax << endl; } if (bestSatIndex != -1) { doSatFilter = true; satMin = sMeans[bestSatIndex] - (2 * sStdDevs[bestSatIndex]); satMax = sMeans[bestSatIndex] + (2 * sStdDevs[bestSatIndex]); if (abs(satMin - satMax) < 20) { satMin = sMeans[bestSatIndex] - 20; satMax = sMeans[bestSatIndex] + 20; } if (satMin < 0) satMin = 0; if (satMax > 255) satMax = 255; if (this->debug) cout << "ColorFilter Sat: " << bestSatIndex << " : " << setw(7) << sMeans[bestSatIndex] << " -- " << satMin << "-" << satMax << endl; } if (bestValIndex != -1) { doValFilter = true; valMin = vMeans[bestValIndex] - (1.5 * vStdDevs[bestValIndex]); valMax = vMeans[bestValIndex] + (1.5 * vStdDevs[bestValIndex]); if (abs(valMin - valMax) < 20) { valMin = vMeans[bestValIndex] - 20; valMax = vMeans[bestValIndex] + 20; } if (valMin < 0) valMin = 0; if (valMax > 255) valMax = 255; if (this->debug) cout << "ColorFilter Val: " << bestValIndex << " : " << setw(7) << vMeans[bestValIndex] << " -- " << valMin << "-" << valMax << endl; } Mat imgDebugHueOnly = Mat::zeros(hsv.size(), hsv.type()); Mat imgDebug = Mat::zeros(hsv.size(), hsv.type()); Mat imgDistanceFromCenter = Mat::zeros(hsv.size(), CV_8U); Mat debugMask = Mat::zeros(hsv.size(), CV_8U); bitwise_not(debugMask, debugMask); for (int row = 0; row < charMask.rows; row++) { for (int col = 0; col < charMask.cols; col++) { int h = (int) hsv.at<Vec3b>(row, col)[0]; int s = (int) hsv.at<Vec3b>(row, col)[1]; int v = (int) hsv.at<Vec3b>(row, col)[2]; bool hPasses = true; bool sPasses = true; bool vPasses = true; int vDistance = abs(v - vMeans[bestValIndex]); imgDebugHueOnly.at<Vec3b>(row, col)[0] = h; imgDebugHueOnly.at<Vec3b>(row, col)[1] = 255; imgDebugHueOnly.at<Vec3b>(row, col)[2] = 255; imgDebug.at<Vec3b>(row, col)[0] = 255; imgDebug.at<Vec3b>(row, col)[1] = 255; imgDebug.at<Vec3b>(row, col)[2] = 255; if (doHueFilter && (h < hueMin || h > hueMax)) { hPasses = false; imgDebug.at<Vec3b>(row, col)[0] = 0; debugMask.at<uchar>(row, col) = 0; } if (doSatFilter && (s < satMin || s > satMax)) { sPasses = false; imgDebug.at<Vec3b>(row, col)[1] = 0; } if (doValFilter && (v < valMin || v > valMax)) { vPasses = false; imgDebug.at<Vec3b>(row, col)[2] = 0; } //if (pixelPasses) // colorMask.at<uchar>(row, col) = 255; //else //imgDebug.at<Vec3b>(row, col)[0] = hPasses & 255; //imgDebug.at<Vec3b>(row, col)[1] = sPasses & 255; //imgDebug.at<Vec3b>(row, col)[2] = vPasses & 255; if ((hPasses) || (hPasses && sPasses))//(hPasses && vPasses) || (sPasses && vPasses) || this->colorMask.at<uchar>(row, col) = 255; else this->colorMask.at<uchar>(row, col) = 0; if ((hPasses && sPasses) || (hPasses && vPasses) || (sPasses && vPasses)) { vDistance = pow(vDistance, 0.9); } else { vDistance = pow(vDistance, 1.1); } if (vDistance > 255) vDistance = 255; imgDistanceFromCenter.at<uchar>(row, col) = vDistance; } } vector<Mat> debugImagesSet; if (this->debug) { debugImagesSet.push_back(addLabel(charMask, "Charecter mask")); //debugImagesSet1.push_back(erodedCharMask); Mat maskCopy(colorMask.size(), colorMask.type()); colorMask.copyTo(maskCopy); debugImagesSet.push_back(addLabel(maskCopy, "color Mask Before")); } Mat bigElement = getStructuringElement( 1, Size( 3 + 1, 3+1 ), Point( 1, 1 ) ); Mat smallElement = getStructuringElement( 1, Size( 1 + 1, 1+1 ), Point( 1, 1 ) ); morphologyEx(this->colorMask, this->colorMask, MORPH_CLOSE, bigElement); //dilate(this->colorMask, this->colorMask, bigElement); Mat combined(charMask.size(), charMask.type()); bitwise_and(charMask, colorMask, combined); if (this->debug) { debugImagesSet.push_back(addLabel(colorMask, "Color Mask After")); debugImagesSet.push_back(addLabel(combined, "Combined")); //displayImage(config, "COLOR filter Mask", colorMask); debugImagesSet.push_back(addLabel(imgDebug, "Color filter Debug")); cvtColor(imgDebugHueOnly, imgDebugHueOnly, CV_HSV2BGR); debugImagesSet.push_back(addLabel(imgDebugHueOnly, "Color Filter Hue")); equalizeHist(imgDistanceFromCenter, imgDistanceFromCenter); debugImagesSet.push_back(addLabel(imgDistanceFromCenter, "COLOR filter Distance")); debugImagesSet.push_back(addLabel(debugMask, "COLOR Hues off")); Mat dashboard = drawImageDashboard(debugImagesSet, imgDebugHueOnly.type(), 3); displayImage(config, "Color Filter Images", dashboard); } }
// Tries to find a rectangular area surrounding most of the characters. Not required // but helpful when determining the plate edges void PlateMask::findOuterBoxMask( vector<TextContours > contours ) { double min_parent_area = pipeline_data->config->templateHeightPx * pipeline_data->config->templateWidthPx * 0.10; // Needs to be at least 10% of the plate area to be considered. int winningIndex = -1; int winningParentId = -1; int bestCharCount = 0; double lowestArea = 99999999999999; if (pipeline_data->config->debugCharAnalysis) cout << "CharacterAnalysis::findOuterBoxMask" << endl; for (unsigned int imgIndex = 0; imgIndex < contours.size(); imgIndex++) { //vector<bool> charContours = filter(thresholds[imgIndex], allContours[imgIndex], allHierarchy[imgIndex]); int charsRecognized = 0; int parentId = -1; bool hasParent = false; for (unsigned int i = 0; i < contours[imgIndex].goodIndices.size(); i++) { if (contours[imgIndex].goodIndices[i]) charsRecognized++; if (contours[imgIndex].goodIndices[i] && contours[imgIndex].hierarchy[i][3] != -1) { parentId = contours[imgIndex].hierarchy[i][3]; hasParent = true; } } if (charsRecognized == 0) continue; if (hasParent) { double boxArea = contourArea(contours[imgIndex].contours[parentId]); if (boxArea < min_parent_area) continue; if ((charsRecognized > bestCharCount) || (charsRecognized == bestCharCount && boxArea < lowestArea)) //(boxArea < lowestArea) { bestCharCount = charsRecognized; winningIndex = imgIndex; winningParentId = parentId; lowestArea = boxArea; } } } if (pipeline_data->config->debugCharAnalysis) cout << "Winning image index (findOuterBoxMask) is: " << winningIndex << endl; if (winningIndex != -1 && bestCharCount >= 3) { Mat mask = Mat::zeros(pipeline_data->thresholds[winningIndex].size(), CV_8U); // get rid of the outline by drawing a 1 pixel width black line drawContours(mask, contours[winningIndex].contours, winningParentId, // draw this contour cv::Scalar(255,255,255), // in FILLED, 8, contours[winningIndex].hierarchy, 0 ); // Morph Open the mask to get rid of any little connectors to non-plate portions int morph_elem = 2; int morph_size = 3; Mat element = getStructuringElement( morph_elem, Size( 2*morph_size + 1, 2*morph_size+1 ), Point( morph_size, morph_size ) ); //morphologyEx( mask, mask, MORPH_CLOSE, element ); morphologyEx( mask, mask, MORPH_OPEN, element ); //morph_size = 1; //element = getStructuringElement( morph_elem, Size( 2*morph_size + 1, 2*morph_size+1 ), Point( morph_size, morph_size ) ); //dilate(mask, mask, element); // Drawing the edge black effectively erodes the image. This may clip off some extra junk from the edges. // We'll want to do the contour again and find the larges one so that we remove the clipped portion. vector<vector<Point> > contoursSecondRound; findContours(mask, contoursSecondRound, RETR_EXTERNAL, CHAIN_APPROX_SIMPLE); int biggestContourIndex = -1; double largestArea = 0; for (unsigned int c = 0; c < contoursSecondRound.size(); c++) { double area = contourArea(contoursSecondRound[c]); if (area > largestArea) { biggestContourIndex = c; largestArea = area; } } if (biggestContourIndex != -1) { mask = Mat::zeros(pipeline_data->thresholds[winningIndex].size(), CV_8U); vector<Point> smoothedMaskPoints; approxPolyDP(contoursSecondRound[biggestContourIndex], smoothedMaskPoints, 2, true); vector<vector<Point> > tempvec; tempvec.push_back(smoothedMaskPoints); //fillPoly(mask, smoothedMaskPoints.data(), smoothedMaskPoints, Scalar(255,255,255)); drawContours(mask, tempvec, 0, // draw this contour cv::Scalar(255,255,255), // in FILLED, 8, contours[winningIndex].hierarchy, 0 ); } if (pipeline_data->config->debugCharAnalysis) { vector<Mat> debugImgs; Mat debugImgMasked = Mat::zeros(pipeline_data->thresholds[winningIndex].size(), CV_8U); pipeline_data->thresholds[winningIndex].copyTo(debugImgMasked, mask); debugImgs.push_back(mask); debugImgs.push_back(pipeline_data->thresholds[winningIndex]); debugImgs.push_back(debugImgMasked); Mat dashboard = drawImageDashboard(debugImgs, CV_8U, 1); displayImage(pipeline_data->config, "Winning outer box", dashboard); } hasPlateMask = true; this->plateMask = mask; } else { hasPlateMask = false; Mat fullMask = Mat::zeros(pipeline_data->thresholds[0].size(), CV_8U); bitwise_not(fullMask, fullMask); this->plateMask = fullMask; } }
void CharacterAnalysis::analyze() { timespec startTime; getTimeMonotonic(&startTime); pipeline_data->clearThresholds(); pipeline_data->thresholds = produceThresholds(pipeline_data->crop_gray, config); timespec contoursStartTime; getTimeMonotonic(&contoursStartTime); pipeline_data->textLines.clear(); for (unsigned int i = 0; i < pipeline_data->thresholds.size(); i++) { TextContours tc(pipeline_data->thresholds[i]); allTextContours.push_back(tc); } if (config->debugTiming) { timespec contoursEndTime; getTimeMonotonic(&contoursEndTime); cout << " -- Character Analysis Find Contours Time: " << diffclock(contoursStartTime, contoursEndTime) << "ms." << endl; } //Mat img_equalized = equalizeBrightness(img_gray); timespec filterStartTime; getTimeMonotonic(&filterStartTime); for (unsigned int i = 0; i < pipeline_data->thresholds.size(); i++) { this->filter(pipeline_data->thresholds[i], allTextContours[i]); if (config->debugCharAnalysis) cout << "Threshold " << i << " had " << allTextContours[i].getGoodIndicesCount() << " good indices." << endl; } if (config->debugTiming) { timespec filterEndTime; getTimeMonotonic(&filterEndTime); cout << " -- Character Analysis Filter Time: " << diffclock(filterStartTime, filterEndTime) << "ms." << endl; } PlateMask plateMask(pipeline_data); plateMask.findOuterBoxMask(allTextContours); pipeline_data->hasPlateBorder = plateMask.hasPlateMask; pipeline_data->plateBorderMask = plateMask.getMask(); if (plateMask.hasPlateMask) { // Filter out bad contours now that we have an outer box mask... for (unsigned int i = 0; i < pipeline_data->thresholds.size(); i++) { filterByOuterMask(allTextContours[i]); } } int bestFitScore = -1; int bestFitIndex = -1; for (unsigned int i = 0; i < pipeline_data->thresholds.size(); i++) { int segmentCount = allTextContours[i].getGoodIndicesCount(); if (segmentCount > bestFitScore) { bestFitScore = segmentCount; bestFitIndex = i; bestThreshold = pipeline_data->thresholds[i]; bestContours = allTextContours[i]; } } if (this->config->debugCharAnalysis) cout << "Best fit score: " << bestFitScore << " Index: " << bestFitIndex << endl; if (bestFitScore <= 1) { pipeline_data->disqualified = true; pipeline_data->disqualify_reason = "Low best fit score in characteranalysis"; return; } //getColorMask(img, allContours, allHierarchy, charSegments); if (this->config->debugCharAnalysis) { Mat img_contours = bestContours.drawDebugImage(bestThreshold); displayImage(config, "Matching Contours", img_contours); } LineFinder lf(pipeline_data); vector<vector<Point> > linePolygons = lf.findLines(pipeline_data->crop_gray, bestContours); vector<TextLine> tempTextLines; for (unsigned int i = 0; i < linePolygons.size(); i++) { vector<Point> linePolygon = linePolygons[i]; LineSegment topLine = LineSegment(linePolygon[0].x, linePolygon[0].y, linePolygon[1].x, linePolygon[1].y); LineSegment bottomLine = LineSegment(linePolygon[3].x, linePolygon[3].y, linePolygon[2].x, linePolygon[2].y); vector<Point> textArea = getCharArea(topLine, bottomLine); TextLine textLine(textArea, linePolygon, pipeline_data->crop_gray.size()); tempTextLines.push_back(textLine); } filterBetweenLines(bestThreshold, bestContours, tempTextLines); // Sort the lines from top to bottom. std::sort(tempTextLines.begin(), tempTextLines.end(), sort_text_line); // Now that we've filtered a few more contours, re-do the text area. for (unsigned int i = 0; i < tempTextLines.size(); i++) { vector<Point> updatedTextArea = getCharArea(tempTextLines[i].topLine, tempTextLines[i].bottomLine); vector<Point> linePolygon = tempTextLines[i].linePolygon; if (updatedTextArea.size() > 0 && linePolygon.size() > 0) { pipeline_data->textLines.push_back(TextLine(updatedTextArea, linePolygon, pipeline_data->crop_gray.size())); } } pipeline_data->plate_inverted = isPlateInverted(); if (pipeline_data->textLines.size() > 0) { int confidenceDrainers = 0; int charSegmentCount = this->bestContours.getGoodIndicesCount(); if (charSegmentCount == 1) confidenceDrainers += 91; else if (charSegmentCount < 5) confidenceDrainers += (5 - charSegmentCount) * 10; // Use the angle for the first line -- assume they'll always be parallel for multi-line plates int absangle = abs(pipeline_data->textLines[0].topLine.angle); if (absangle > config->maxPlateAngleDegrees) confidenceDrainers += 91; else if (absangle > 1) confidenceDrainers += (config->maxPlateAngleDegrees - absangle) ; // If a multiline plate has only one line, disqualify if (pipeline_data->isMultiline && pipeline_data->textLines.size() < 2) { if (config->debugCharAnalysis) std::cout << "Did not detect multiple lines on multi-line plate" << std::endl; confidenceDrainers += 95; } if (confidenceDrainers >= 90) { pipeline_data->disqualified = true; pipeline_data->disqualify_reason = "Low confidence in characteranalysis"; } else { float confidence = 100 - confidenceDrainers; pipeline_data->confidence_weights.setScore("CHARACTER_ANALYSIS_SCORE", confidence, 1.0); } } else { pipeline_data->disqualified = true; pipeline_data->disqualify_reason = "No text lines found in characteranalysis"; } if (config->debugTiming) { timespec endTime; getTimeMonotonic(&endTime); cout << "Character Analysis Time: " << diffclock(startTime, endTime) << "ms." << endl; } // Draw debug dashboard if (this->pipeline_data->config->debugCharAnalysis && pipeline_data->textLines.size() > 0) { vector<Mat> tempDash; for (unsigned int z = 0; z < pipeline_data->thresholds.size(); z++) { Mat tmp(pipeline_data->thresholds[z].size(), pipeline_data->thresholds[z].type()); pipeline_data->thresholds[z].copyTo(tmp); cvtColor(tmp, tmp, CV_GRAY2BGR); tempDash.push_back(tmp); } Mat bestVal(this->bestThreshold.size(), this->bestThreshold.type()); this->bestThreshold.copyTo(bestVal); cvtColor(bestVal, bestVal, CV_GRAY2BGR); for (unsigned int z = 0; z < this->bestContours.size(); z++) { Scalar dcolor(255,0,0); if (this->bestContours.goodIndices[z]) dcolor = Scalar(0,255,0); drawContours(bestVal, this->bestContours.contours, z, dcolor, 1); } tempDash.push_back(bestVal); displayImage(config, "Character Region Step 1 Thresholds", drawImageDashboard(tempDash, bestVal.type(), 3)); } }
void PlateLines::processImage(Mat inputImage, vector<TextLine> textLines, float sensitivity) { if (this->debug) cout << "PlateLines findLines" << endl; timespec startTime; getTimeMonotonic(&startTime); // Ignore input images that are pure white or pure black Scalar avgPixelIntensity = mean(inputImage); if (avgPixelIntensity[0] >= 252) return; else if (avgPixelIntensity[0] <= 3) return; // Do a bilateral filter to clean the noise but keep edges sharp Mat smoothed(inputImage.size(), inputImage.type()); adaptiveBilateralFilter(inputImage, smoothed, Size(3,3), 45, 45); int morph_elem = 2; int morph_size = 2; Mat element = getStructuringElement( morph_elem, Size( 2*morph_size + 1, 2*morph_size+1 ), Point( morph_size, morph_size ) ); Mat edges(inputImage.size(), inputImage.type()); Canny(smoothed, edges, 66, 133); // Create a mask that is dilated based on the detected characters Mat mask = Mat::zeros(inputImage.size(), CV_8U); for (unsigned int i = 0; i < textLines.size(); i++) { vector<vector<Point> > polygons; polygons.push_back(textLines[i].textArea); fillPoly(mask, polygons, Scalar(255,255,255)); } dilate(mask, mask, getStructuringElement( 1, Size( 1 + 1, 2*1+1 ), Point( 1, 1 ) )); bitwise_not(mask, mask); // AND canny edges with the character mask bitwise_and(edges, mask, edges); vector<PlateLine> hlines = this->getLines(edges, sensitivity, false); vector<PlateLine> vlines = this->getLines(edges, sensitivity, true); for (unsigned int i = 0; i < hlines.size(); i++) this->horizontalLines.push_back(hlines[i]); for (unsigned int i = 0; i < vlines.size(); i++) this->verticalLines.push_back(vlines[i]); // if debug is enabled, draw the image if (this->debug) { Mat debugImgHoriz(edges.size(), edges.type()); Mat debugImgVert(edges.size(), edges.type()); edges.copyTo(debugImgHoriz); edges.copyTo(debugImgVert); cvtColor(debugImgHoriz,debugImgHoriz,CV_GRAY2BGR); cvtColor(debugImgVert,debugImgVert,CV_GRAY2BGR); for( size_t i = 0; i < this->horizontalLines.size(); i++ ) { line( debugImgHoriz, this->horizontalLines[i].line.p1, this->horizontalLines[i].line.p2, Scalar(0,0,255), 1, CV_AA); } for( size_t i = 0; i < this->verticalLines.size(); i++ ) { line( debugImgVert, this->verticalLines[i].line.p1, this->verticalLines[i].line.p2, Scalar(0,0,255), 1, CV_AA); } vector<Mat> images; images.push_back(debugImgHoriz); images.push_back(debugImgVert); Mat dashboard = drawImageDashboard(images, debugImgVert.type(), 1); displayImage(pipelineData->config, "Hough Lines", dashboard); } if (pipelineData->config->debugTiming) { timespec endTime; getTimeMonotonic(&endTime); cout << "Plate Lines Time: " << diffclock(startTime, endTime) << "ms." << endl; } }
CharacterSegmenter::CharacterSegmenter(Mat img, bool invertedColors, Config* config) { this->config = config; this->confidence = 0; if (this->config->debugCharSegmenter) cout << "Starting CharacterSegmenter" << endl; //CharacterRegion charRegion(img, debug); timespec startTime; getTime(&startTime); Mat img_gray(img.size(), CV_8U); cvtColor( img, img_gray, CV_BGR2GRAY ); //normalize(img_gray, img_gray, 0, 255, CV_MINMAX ); medianBlur(img_gray, img_gray, 3); if (invertedColors) bitwise_not(img_gray, img_gray); charAnalysis = new CharacterAnalysis(img_gray, config); charAnalysis->analyze(); if (this->config->debugCharSegmenter) { displayImage(config, "CharacterSegmenter Thresholds", drawImageDashboard(charAnalysis->thresholds, CV_8U, 3)); } if (this->config->debugCharSegmenter) { Mat img_contours(charAnalysis->bestThreshold.size(), CV_8U); charAnalysis->bestThreshold.copyTo(img_contours); cvtColor(img_contours, img_contours, CV_GRAY2RGB); vector<vector<Point> > allowedContours; for (int i = 0; i < charAnalysis->bestContours.size(); i++) { if (charAnalysis->bestCharSegments[i]) allowedContours.push_back(charAnalysis->bestContours[i]); } drawContours(img_contours, charAnalysis->bestContours, -1, // draw all contours cv::Scalar(255,0,0), // in blue 1); // with a thickness of 1 drawContours(img_contours, allowedContours, -1, // draw all contours cv::Scalar(0,255,0), // in green 1); // with a thickness of 1 if (charAnalysis->linePolygon.size() > 0) { line(img_contours, charAnalysis->linePolygon[0], charAnalysis->linePolygon[1], Scalar(255, 0, 255), 1); line(img_contours, charAnalysis->linePolygon[3], charAnalysis->linePolygon[2], Scalar(255, 0, 255), 1); } Mat bordered = addLabel(img_contours, "Best Contours"); imgDbgGeneral.push_back(bordered); } // Figure out the average character width float totalCharWidth = 0; float totalCharHeight = 0; if (charAnalysis->linePolygon.size() > 0) { this->top = LineSegment(charAnalysis->linePolygon[0].x, charAnalysis->linePolygon[0].y, charAnalysis->linePolygon[1].x, charAnalysis->linePolygon[1].y); this->bottom = LineSegment(charAnalysis->linePolygon[3].x, charAnalysis->linePolygon[3].y, charAnalysis->linePolygon[2].x, charAnalysis->linePolygon[2].y); for (int i = 0; i < charAnalysis->bestContours.size(); i++) { if (charAnalysis->bestCharSegments[i] == false) continue; Rect mr = boundingRect(charAnalysis->bestContours[i]); totalCharWidth += mr.width; totalCharHeight += mr.height; } int numSamples = charAnalysis->bestCharSegmentsCount; float avgCharWidth = totalCharWidth / numSamples; float avgCharHeight = totalCharHeight / numSamples; removeSmallContours(charAnalysis->thresholds, charAnalysis->allContours, avgCharWidth, avgCharHeight); // Do the histogram analysis to figure out char regions timespec startTime; getTime(&startTime); vector<Mat> allHistograms; vector<Rect> allBoxes; for (int i = 0; i < charAnalysis->allContours.size(); i++) { Mat histogramMask = Mat::zeros(charAnalysis->thresholds[i].size(), CV_8U); fillConvexPoly(histogramMask, charAnalysis->linePolygon.data(), charAnalysis->linePolygon.size(), Scalar(255,255,255)); VerticalHistogram vertHistogram(charAnalysis->thresholds[i], histogramMask); if (this->config->debugCharSegmenter) { Mat histoCopy(vertHistogram.debugImg.size(), vertHistogram.debugImg.type()); //vertHistogram.copyTo(histoCopy); cvtColor(vertHistogram.debugImg, histoCopy, CV_GRAY2RGB); allHistograms.push_back(histoCopy); } // float score = 0; vector<Rect> charBoxes = getHistogramBoxes(vertHistogram.debugImg, avgCharWidth, avgCharHeight, &score); if (this->config->debugCharSegmenter) { for (int cboxIdx = 0; cboxIdx < charBoxes.size(); cboxIdx++) { rectangle(allHistograms[i], charBoxes[cboxIdx], Scalar(0, 255, 0)); } Mat histDashboard = drawImageDashboard(allHistograms, allHistograms[0].type(), 3); displayImage(config, "Char seg histograms", histDashboard); } for (int z = 0; z < charBoxes.size(); z++) allBoxes.push_back(charBoxes[z]); //drawAndWait(&histogramMask); } float biggestCharWidth = avgCharWidth; // Compute largest char width for (int i = 0; i < allBoxes.size(); i++) { if (allBoxes[i].width > biggestCharWidth) biggestCharWidth = allBoxes[i].width; } if (config->debugTiming) { timespec endTime; getTime(&endTime); cout << " -- Character Segmentation Create and Score Histograms Time: " << diffclock(startTime, endTime) << "ms." << endl; } //ColorFilter colorFilter(img, charAnalysis->getCharacterMask()); vector<Rect> candidateBoxes = getBestCharBoxes(charAnalysis->thresholds[0], allBoxes, biggestCharWidth); if (this->config->debugCharSegmenter) { // Setup the dashboard images to show the cleaning filters for (int i = 0; i < charAnalysis->thresholds.size(); i++) { Mat cleanImg = Mat::zeros(charAnalysis->thresholds[i].size(), charAnalysis->thresholds[i].type()); Mat boxMask = getCharBoxMask(charAnalysis->thresholds[i], candidateBoxes); charAnalysis->thresholds[i].copyTo(cleanImg); bitwise_and(cleanImg, boxMask, cleanImg); cvtColor(cleanImg, cleanImg, CV_GRAY2BGR); for (int c = 0; c < candidateBoxes.size(); c++) rectangle(cleanImg, candidateBoxes[c], Scalar(0, 255, 0), 1); imgDbgCleanStages.push_back(cleanImg); } } getTime(&startTime); filterEdgeBoxes(charAnalysis->thresholds, candidateBoxes, biggestCharWidth, avgCharHeight); candidateBoxes = filterMostlyEmptyBoxes(charAnalysis->thresholds, candidateBoxes); candidateBoxes = combineCloseBoxes(candidateBoxes, biggestCharWidth); cleanCharRegions(charAnalysis->thresholds, candidateBoxes); cleanMostlyFullBoxes(charAnalysis->thresholds, candidateBoxes); //cleanBasedOnColor(thresholds, colorFilter.colorMask, candidateBoxes); candidateBoxes = filterMostlyEmptyBoxes(charAnalysis->thresholds, candidateBoxes); this->characters = candidateBoxes; if (config->debugTiming) { timespec endTime; getTime(&endTime); cout << " -- Character Segmentation Box cleaning/filtering Time: " << diffclock(startTime, endTime) << "ms." << endl; } if (this->config->debugCharSegmenter) { Mat imgDash = drawImageDashboard(charAnalysis->thresholds, CV_8U, 3); displayImage(config, "Segmentation after cleaning", imgDash); Mat generalDash = drawImageDashboard(this->imgDbgGeneral, this->imgDbgGeneral[0].type(), 2); displayImage(config, "Segmentation General", generalDash); Mat cleanImgDash = drawImageDashboard(this->imgDbgCleanStages, this->imgDbgCleanStages[0].type(), 3); displayImage(config, "Segmentation Clean Filters", cleanImgDash); } } if (config->debugTiming) { timespec endTime; getTime(&endTime); cout << "Character Segmenter Time: " << diffclock(startTime, endTime) << "ms." << endl; } }