void Faze::computePupil(int mode) { assert(mode == MODE_PUPIL_SP || mode == MODE_PUPIL_CDF); std::vector<cv::Point> leftEyePoints = getDescriptors(INDEX_LEFT_EYE); cv::Rect rectLeftEye = cv::boundingRect(leftEyePoints); cv::Mat roiLeftEye = imageGray(rectLeftEye); preprocessROI(roiLeftEye); std::vector<cv::Point> rightEyePoints = getDescriptors(INDEX_RIGHT_EYE); cv::Rect rectRightEye = cv::boundingRect(rightEyePoints); cv::Mat roiRightEye = imageGray(rectRightEye); preprocessROI(roiRightEye); if(mode == MODE_PUPIL_SP) { descriptors.push_back(get_pupil_coordinates(roiLeftEye,rectLeftEye)); descriptors.push_back(get_pupil_coordinates(roiRightEye,rectRightEye)); } else { descriptors.push_back(computePupilCDF(roiLeftEye)); descriptors.push_back(computePupilCDF(roiRightEye)); } }
void caasCLR4Tx1::FindIsolatorAngle() { //We cut 4/5 width of isolator out int height = 3 * (isolatorBottomEdge - isolatorTopEdge) / 2; int middle = (isolatorBottomEdge + isolatorTopEdge) / 2; Rect rect = Rect(isolatorRightEdge - 4 * isolatorWidth / 5, middle - height / 2, 4 * isolatorWidth / 5, height); Mat imageIsolator = imageGray(rect); int scale = 4; resize(imageIsolator, imageIsolator, Size(imageIsolator.cols / scale, imageIsolator.rows / scale)); #if _DEBUG imwrite("Isolator.jpg", imageIsolator); #endif //sharpen the image //Unsharping masking: Use a Gaussian smoothing filter and subtract the smoothed version from the original image (in a weighted way so the values of a constant area remain constant). Mat imageBlurred, imageGraySharpened; double GAUSSIAN_RADIUS = 4.0; GaussianBlur(imageIsolator, imageBlurred, Size(0, 0), GAUSSIAN_RADIUS); addWeighted(imageIsolator, 1.5, imageBlurred, -0.5, 0, imageGraySharpened); #if _DEBUG //imageGraySharpened = imageGrayQuarter; imwrite("IsolatorSharpened.jpg", imageGraySharpened); #endif imageGraySharpened = imageIsolator; //Histogram Equalization equalizeHist(imageGraySharpened, imageGraySharpened); #if _DEBUG imwrite("IsolatorEqualized.jpg", imageGraySharpened); #endif //Otsu binarization Mat imageOtsu; threshold(imageGraySharpened, imageOtsu, 0, 255, CV_THRESH_BINARY | CV_THRESH_OTSU); #if _DEBUG imwrite("IsolatorOtsu.jpg", imageOtsu); #endif //Canny Edge Detection int median = Median(imageGraySharpened); Mat imageCanny; Canny(imageGraySharpened, imageCanny, 0.66 * median, 1.33 * median); #if _DEBUG imwrite("IsolatorCanny.jpg", imageCanny); #endif }
void caasCLR4Tx1::RefineIsolator() { //int width = isolatorRightEdge - isolatorLeftEdge; //int height = isolatorBottomEdge - isolatorTopEdge; //Expand horizontally 1 time to the right; expand vertically 1/2 int left = isolatorLeftEdge; int right = left + 2 * isolatorWidth; if (right > targetLeftEdge - 20) right = targetLeftEdge - 20; int top = isolatorTopEdge - isolatorHeight / 4; int bottom = top + isolatorHeight + isolatorHeight / 2; Rect roi = Rect(left, top, right - left, bottom - top); imageIsolatorRoi = imageGray(roi); #if _DEBUG imwrite("5.1.IsolatorRoi.jpg", imageIsolatorRoi); #endif int scale = 4; resize(imageIsolatorRoi, imageIsolatorRoi, Size(imageIsolatorRoi.cols / scale, imageIsolatorRoi.rows / scale)); Mat imageBlurred, imageGraySharpened; double GAUSSIAN_RADIUS = 4.0; GaussianBlur(imageIsolatorRoi, imageBlurred, Size(0, 0), GAUSSIAN_RADIUS); addWeighted(imageIsolatorRoi, 2.5, imageBlurred, -1.5, 0, imageGraySharpened); #if _DEBUG imwrite("5.2.IsolatorRoiSharpened.jpg", imageGraySharpened); #endif int median = Median(imageGraySharpened); Mat imageCanny; Canny(imageGraySharpened, imageCanny, 0.66 * median, 1.33 * median); #if _DEBUG imwrite("5.3.IsolatorRoiCanny.jpg", imageCanny); #endif Mat Points; findNonZero(imageCanny, Points); Rect Min_Rect = boundingRect(Points); isolatorRightEdge = roi.x + (Min_Rect.x + Min_Rect.width) * scale; RotatedRect rect = minAreaRect(Points); isolatorAngle = rect.angle; }
/** Step 4: Find Isolator and its orientation Currently we use around 0.75 pixel per micron. So, 10 microns correspond to 7.5 pixels. We reduce width and height to 1/4, which makes 10 microns correspond to 1.875 pixels, 50 microns --> 9.375 pixels, We assume the isolator has high texture, and therefore high gradients, and the regions between isolator and target has less gradients. */ void caasCLR4Tx1::FindIsolator() { int high = targetBottomEdge - targetTopEdge, top = targetTopEdge + high / 4; RoiTargetLeftMiddleHalf = Rect(0, top, targetLeftEdge, high / 2); //We assume the isolator will be in the middle part of the target. imageTargetLeftMiddleHalf = imageGray(RoiTargetLeftMiddleHalf); #if _DEBUG imwrite("4.1.TargetLeftMiddleHalf.jpg", imageTargetLeftMiddleHalf); #endif int scale = 4; //We reduce the original image resolution resize(imageTargetLeftMiddleHalf, imageOneFourthTargetLeftMiddleHalf, Size(imageTargetLeftMiddleHalf.cols / scale, imageTargetLeftMiddleHalf.rows / scale)); #if _DEBUG imwrite("4.2.OneFourthTargetLeftMiddleHalf.jpg", imageOneFourthTargetLeftMiddleHalf); #endif //sharpen the image //Unsharping masking: Use a Gaussian smoothing filter and subtract the smoothed version from the original image (in a weighted way so the values of a constant area remain constant). Mat imageBlurred, imageGraySharpened; double GAUSSIAN_RADIUS = 4.0; GaussianBlur(imageOneFourthTargetLeftMiddleHalf, imageBlurred, Size(0, 0), GAUSSIAN_RADIUS); addWeighted(imageOneFourthTargetLeftMiddleHalf, 1.5, imageBlurred, -0.5, 0, imageGraySharpened); #if _DEBUG //imageGraySharpened = imageGrayQuarter; imwrite("4.3.OneFourthTargetLeftMiddleHalfSharpened.jpg", imageGraySharpened); #endif //Histogram Equalization equalizeHist(imageGraySharpened, imageGraySharpened); #if _DEBUG imwrite("4.4.OneFourthTargetLeftMiddleHalfEqualized.jpg", imageGraySharpened); #endif //Canny Edge Detection int median = Median(imageGraySharpened); Canny(imageGraySharpened, imageOneFourthTargetLeftMiddleHalfCanny, 0.66 * median, 1.33 * median); #if _DEBUG imwrite("4.5.OneFourthTargetLeftMiddleHalfCanny.jpg", imageOneFourthTargetLeftMiddleHalfCanny); #endif { //Find the isolator right edge //vertical projection profile Mat verProjection(1, imageOneFourthTargetLeftMiddleHalfCanny.cols, CV_32FC1); reduce(imageOneFourthTargetLeftMiddleHalfCanny, verProjection, 0, CV_REDUCE_SUM, CV_32FC1); //Vertical projection generate Horizontal profile float minValue = -1.0, maxValue = -1.0; int minIndex, maxIndex; float values[4000]; ProjectionProfileAnalysis(verProjection, minValue, minIndex, maxValue, maxIndex, values); //Set the start and end 3 to be zero for convenience vector<int> starts, ends; int start, end; for (int i = 0; i < 3; i++) { values[verProjection.cols - 1 - i] = 0; values[i] = 0; } //Find all non-zero runs for (int i = verProjection.cols - 2; i >= 0; i--) { if (values[i + 1] < 0.5 && values[i] < 0.5) continue; if (values[i + 1] > 0.5 && values[i] > 0.5) continue; if (values[i + 1] < 0.5 && values[i] > 0.5) { end = i; continue; } if (values[i + 1] > 0.5 && values[i] < 0.5) { start = i + 1; starts.push_back(start); ends.push_back(end); } } //Check every runs, cut 3/4 of the expected isolator width, and we expect to see many edges (high profile values) for (int i = 0; i < starts.size(); i++) { end = ends[i]; start = end - 3 * (isolatorWidth / scale) / 4; float average = 0; for (int k = start; k <= end; k++) average += values[k]; average /= (float)(end - start + 1); if (average < 3) continue; //Has to be larger than a threshold isolatorRightEdge = end * scale; break; } } { //Find isolator top and bottom edges //We just look at the half width of the isolator, trying to find its top and bottom edges isolatorLeftEdge = isolatorRightEdge - isolatorWidth / 2; Rect roi = Rect(isolatorLeftEdge / scale, 0, (isolatorRightEdge - isolatorLeftEdge) / scale, imageOneFourthTargetLeftMiddleHalfCanny.rows); imageIsolatorRoiCanny = imageOneFourthTargetLeftMiddleHalfCanny(roi); #if _DEBUG imwrite("4.6.IsolatorRoiCanny.jpg", imageIsolatorRoiCanny); #endif //horizontal projection profile Mat horProjection(imageIsolatorRoiCanny.rows, 1, CV_32FC1); reduce(imageIsolatorRoiCanny, horProjection, 1, CV_REDUCE_SUM, CV_32FC1); //horizontal projection generate vertical profile float minValue = -1.0, maxValue = -1.0; int minIndex, maxIndex; float values[4000]; ProjectionProfileAnalysis(horProjection, minValue, minIndex, maxValue, maxIndex, values); //Find all 0's and remove 0's which are smaller than a threshold float prev, curr; vector<int> starts, ends; for (int i = 0; i <= imageIsolatorRoiCanny.rows; i++) { prev = i == 0 ? prev = 1 : values[i - 1]; curr = i == imageIsolatorRoiCanny.rows ? 1 : values[i]; if (prev > 0.5 && curr < 0.5) starts.push_back(i); if (prev < 0.5 && curr > 0.5) ends.push_back(i); } if (starts.size() == 0) { isolatorTopEdge = roi.y * scale; isolatorBottomEdge = (roi.y + roi.height) * scale; } else { vector<int> zero_starts, zero_ends; for (int i = 0; i < starts.size(); i++) { int length = ends[i] - starts[i]; if (length > 5) { zero_starts.push_back(starts[i]); zero_ends.push_back(ends[i]); } } //Pick one 1's, which is closest to the expected isolator width vector<int> one_starts, one_ends; if (zero_starts[0] != 0) { one_starts.push_back(0); one_ends.push_back(zero_starts[0]); } for (int i = 0; i < zero_starts.size() - 1; i++) { one_starts.push_back(zero_ends[i]); one_ends.push_back(zero_starts[i + 1]); } if (zero_ends[zero_ends.size() - 1] != imageIsolatorRoiCanny.rows) { one_starts.push_back(zero_ends[zero_ends.size() - 1]); one_ends.push_back(imageIsolatorRoiCanny.cols); } int min_index, min_diff = 4000; for (int i = 0; i < one_starts.size(); i++) { int length = one_ends[i] - one_starts[i]; int diff = std::abs(length - this->isolatorWidth / scale); if (diff < min_diff) { min_diff = diff; min_index = i; } } isolatorTopEdge = this->RoiTargetLeftMiddleHalf.y + (roi.y + one_starts[min_index]) * scale; isolatorBottomEdge = this->RoiTargetLeftMiddleHalf.y + (roi.y + one_ends[min_index]) * scale; } #if _DEBUG roi = Rect(isolatorLeftEdge, isolatorTopEdge, isolatorRightEdge - isolatorLeftEdge, isolatorBottomEdge - isolatorTopEdge); imageIsolator = imageGray(roi); imwrite("4.7.Isolator.jpg", imageIsolator); #endif } }
/** Step 3: Find the top and bottom edge of the target. We assume that the regions above and below the target are darker than the target. */ void caasCLR4Tx1::FindTargetTopBottomEdges() { int scale = 4; Rect roi = Rect(targetLeftEdge / scale, 0, (targetRightEdge - targetLeftEdge) / scale, this->imageOneFourth.rows); imageOneFourthTarget = this->imageOneFourth(roi); #if _DEBUG imwrite("3.1.OneFourthTarget.jpg", imageOneFourthTarget); #endif Mat imageOtsu; threshold(imageOneFourthTarget, imageOtsu, 0, 255, CV_THRESH_BINARY | CV_THRESH_OTSU); #if _DEBUG imwrite("3.2.OneFourthTargetOtsu.jpg", imageOtsu); #endif //horizontal projection profile Mat horProjection(imageOtsu.rows, 1, CV_32FC1); reduce(imageOtsu, horProjection, 1, CV_REDUCE_SUM, CV_32FC1); //horizontal projection generate vertical profile float minValue = -1.0, maxValue = -1.0; int minIndex, maxIndex; float values[3000]; float gradients[3000]; ProjectionProfileAnalysis(horProjection, minValue, minIndex, maxValue, maxIndex, values); Gradient(horProjection.rows, values, gradients); //Find the top edge of the target //From top to bottom, this is a positive gradient, i.e., from low gray value to high gray value float maxGrad = gradients[0]; int maxGradIndex = 0; for (int i = 1; i < horProjection.rows / 3; i++) //We search in only top 1/3 { if (gradients[i] > maxGrad) { maxGrad = gradients[i]; maxGradIndex = i; } } targetTopEdge = maxGradIndex * scale; //Find the bottom edge of the target //From top to bottom, this is a negative gradient, i.e., from high gray value to low gray value float minGrad = gradients[horProjection.rows - 1]; int minGradIndex = horProjection.rows - 1; for (int i = horProjection.rows - 2; i >= 2 * horProjection.rows / 3; i--) //We search in only bottom 1/3 { if (gradients[i] < minGrad) { minGrad = gradients[i]; minGradIndex = i; } } targetBottomEdge = minGradIndex * scale; #if _DEBUG roi = Rect(targetLeftEdge, targetTopEdge, targetRightEdge - targetLeftEdge, targetBottomEdge - targetTopEdge); imageTarget = imageGray(roi); imwrite("3.3.Target.jpg", imageTarget); #endif return; ////Find the top edge of the target //for (int i = 0; i < horProjection.rows; i++) //{ // if (values[i] > maxValue / 2) // { // targetTopEdge = i*scale; break; // } //} ////Find the bottom edge of the target //for (int i = horProjection.rows - 1; i >= 0; i--) //{ // if (values[i] > maxValue / 2) // { // targetBottomEdge = i*scale; break; // } //} //return; }
cv::Mat CameraUtils::getImageFromCameraProxy(jderobot::ImageDataPtr dataPtr) { cv::Mat outImage; colorspaces::Image::FormatPtr fmt; fmt = colorspaces::Image::Format::searchFormat(dataPtr->description->format); if (!fmt) throw "Format not supported"; if (dataPtr->description->format == colorspaces::ImageRGB8::FORMAT_RGB8_Z.get()->name || dataPtr->description->format == colorspaces::ImageRGB8::FORMAT_DEPTH8_16_Z.get()->name ) { size_t dest_len = dataPtr->description->width*dataPtr->description->height*3; size_t source_len = dataPtr->pixelData.size(); unsigned char* origin_buf = (uchar*) malloc(dest_len); int r = uncompress((Bytef *) origin_buf, (uLongf *) &dest_len, (const Bytef *) &(dataPtr->pixelData[0]), (uLong)source_len); if(r != Z_OK) { fprintf(stderr, "[CMPR] Error:\n"); switch(r) { case Z_MEM_ERROR: fprintf(stderr, "[CMPR] Error: Not enough memory to compress.\n"); break; case Z_BUF_ERROR: fprintf(stderr, "[CMPR] Error: Target buffer too small.\n"); break; case Z_STREAM_ERROR: // Invalid compression level fprintf(stderr, "[CMPR] Error: Invalid compression level.\n"); break; } } else { colorspaces::Image imageRGB(dataPtr->description->width,dataPtr->description->height,colorspaces::ImageRGB8::FORMAT_RGB8,&(origin_buf[0])); colorspaces::ImageRGB8 img_rgb888(imageRGB);//conversion will happen if needed cv::Mat(cvSize(img_rgb888.width,img_rgb888.height), CV_8UC3, img_rgb888.data).copyTo(outImage); img_rgb888.release(); } if (origin_buf) free(origin_buf); } else if (dataPtr->description->format == colorspaces::ImageRGB8::FORMAT_RGB8.get()->name || dataPtr->description->format == colorspaces::ImageRGB8::FORMAT_DEPTH8_16.get()->name ) { colorspaces::Image imageRGB(dataPtr->description->width,dataPtr->description->height,colorspaces::ImageRGB8::FORMAT_RGB8,&(dataPtr->pixelData[0])); colorspaces::ImageRGB8 img_rgb888(imageRGB);//conversion will happen if needed cv::Mat(cvSize(img_rgb888.width,img_rgb888.height), CV_8UC3, img_rgb888.data).copyTo(outImage); img_rgb888.release(); } else if (dataPtr->description->format == colorspaces::ImageGRAY8::FORMAT_GRAY8_Z.get()->name) { //gay compressed size_t dest_len = dataPtr->description->width*dataPtr->description->height; size_t source_len = dataPtr->pixelData.size(); unsigned char* origin_buf = (uchar*) malloc(dest_len); int r = uncompress((Bytef *) origin_buf, (uLongf *) &dest_len, (const Bytef *) &(dataPtr->pixelData[0]), (uLong)source_len); if(r != Z_OK) { fprintf(stderr, "[CMPR] Error:\n"); switch(r) { case Z_MEM_ERROR: fprintf(stderr, "[CMPR] Error: Not enough memory to compress.\n"); break; case Z_BUF_ERROR: fprintf(stderr, "[CMPR] Error: Target buffer too small.\n"); break; case Z_STREAM_ERROR: // Invalid compression level fprintf(stderr, "[CMPR] Error: Invalid compression level.\n"); break; } } else { colorspaces::Image imageGray(dataPtr->description->width,dataPtr->description->height,colorspaces::ImageGRAY8::FORMAT_GRAY8,&(origin_buf[0])); colorspaces::ImageGRAY8 img_gray8(imageGray);//conversion will happen if needed cv::Mat(cvSize(img_gray8.width,img_gray8.height), CV_8UC1, img_gray8.data).copyTo(outImage); img_gray8.release(); } if (origin_buf) free(origin_buf); } else if (dataPtr->description->format == colorspaces::ImageGRAY8::FORMAT_GRAY8.get()->name){ colorspaces::Image imageGray(dataPtr->description->width,dataPtr->description->height,colorspaces::ImageGRAY8::FORMAT_GRAY8,&(dataPtr->pixelData[0])); colorspaces::ImageGRAY8 img_gray8(imageGray);//conversion will happen if needed cv::Mat(cvSize(img_gray8.width,img_gray8.height), CV_8UC1, img_gray8.data).copyTo(outImage); img_gray8.release(); } else{ LOG(ERROR) << "Unkown image format"; } return outImage; }