void Faze::computePupil(int mode) {
assert(mode == MODE_PUPIL_SP || mode == MODE_PUPIL_CDF);
std::vector<cv::Point> leftEyePoints = getDescriptors(INDEX_LEFT_EYE);
cv::Rect rectLeftEye = cv::boundingRect(leftEyePoints);
cv::Mat roiLeftEye = imageGray(rectLeftEye);
preprocessROI(roiLeftEye);
std::vector<cv::Point> rightEyePoints = getDescriptors(INDEX_RIGHT_EYE);
cv::Rect rectRightEye = cv::boundingRect(rightEyePoints);
cv::Mat roiRightEye = imageGray(rectRightEye);
preprocessROI(roiRightEye);
if(mode == MODE_PUPIL_SP) {
descriptors.push_back(get_pupil_coordinates(roiLeftEye,rectLeftEye));
descriptors.push_back(get_pupil_coordinates(roiRightEye,rectRightEye));
}
else {
descriptors.push_back(computePupilCDF(roiLeftEye));
descriptors.push_back(computePupilCDF(roiRightEye));
}
}
void caasCLR4Tx1::FindIsolatorAngle()
{
//We cut 4/5 width of isolator out
int height = 3 * (isolatorBottomEdge - isolatorTopEdge) / 2; int middle = (isolatorBottomEdge + isolatorTopEdge) / 2;
Rect rect = Rect(isolatorRightEdge - 4 * isolatorWidth / 5, middle - height / 2, 4 * isolatorWidth / 5, height);
Mat imageIsolator = imageGray(rect);
int scale = 4;
resize(imageIsolator, imageIsolator, Size(imageIsolator.cols / scale, imageIsolator.rows / scale));
#if _DEBUG
imwrite("Isolator.jpg", imageIsolator);
#endif
//sharpen the image
//Unsharping masking: Use a Gaussian smoothing filter and subtract the smoothed version from the original image (in a weighted way so the values of a constant area remain constant).
Mat imageBlurred, imageGraySharpened; double GAUSSIAN_RADIUS = 4.0;
GaussianBlur(imageIsolator, imageBlurred, Size(0, 0), GAUSSIAN_RADIUS);
addWeighted(imageIsolator, 1.5, imageBlurred, -0.5, 0, imageGraySharpened);
#if _DEBUG
//imageGraySharpened = imageGrayQuarter;
imwrite("IsolatorSharpened.jpg", imageGraySharpened);
#endif
imageGraySharpened = imageIsolator;
//Histogram Equalization
equalizeHist(imageGraySharpened, imageGraySharpened);
#if _DEBUG
imwrite("IsolatorEqualized.jpg", imageGraySharpened);
#endif
//Otsu binarization
Mat imageOtsu; threshold(imageGraySharpened, imageOtsu, 0, 255, CV_THRESH_BINARY | CV_THRESH_OTSU);
#if _DEBUG
imwrite("IsolatorOtsu.jpg", imageOtsu);
#endif
//Canny Edge Detection
int median = Median(imageGraySharpened);
Mat imageCanny; Canny(imageGraySharpened, imageCanny, 0.66 * median, 1.33 * median);
#if _DEBUG
imwrite("IsolatorCanny.jpg", imageCanny);
#endif
}
void caasCLR4Tx1::RefineIsolator()
{
//int width = isolatorRightEdge - isolatorLeftEdge;
//int height = isolatorBottomEdge - isolatorTopEdge;
//Expand horizontally 1 time to the right; expand vertically 1/2
int left = isolatorLeftEdge;
int right = left + 2 * isolatorWidth; if (right > targetLeftEdge - 20) right = targetLeftEdge - 20;
int top = isolatorTopEdge - isolatorHeight / 4;
int bottom = top + isolatorHeight + isolatorHeight / 2;
Rect roi = Rect(left, top, right - left, bottom - top);
imageIsolatorRoi = imageGray(roi);
#if _DEBUG
imwrite("5.1.IsolatorRoi.jpg", imageIsolatorRoi);
#endif
int scale = 4;
resize(imageIsolatorRoi, imageIsolatorRoi, Size(imageIsolatorRoi.cols / scale, imageIsolatorRoi.rows / scale));
Mat imageBlurred, imageGraySharpened; double GAUSSIAN_RADIUS = 4.0;
GaussianBlur(imageIsolatorRoi, imageBlurred, Size(0, 0), GAUSSIAN_RADIUS);
addWeighted(imageIsolatorRoi, 2.5, imageBlurred, -1.5, 0, imageGraySharpened);
#if _DEBUG
imwrite("5.2.IsolatorRoiSharpened.jpg", imageGraySharpened);
#endif
int median = Median(imageGraySharpened);
Mat imageCanny; Canny(imageGraySharpened, imageCanny, 0.66 * median, 1.33 * median);
#if _DEBUG
imwrite("5.3.IsolatorRoiCanny.jpg", imageCanny);
#endif
Mat Points; findNonZero(imageCanny, Points);
Rect Min_Rect = boundingRect(Points);
isolatorRightEdge = roi.x + (Min_Rect.x + Min_Rect.width) * scale;
RotatedRect rect = minAreaRect(Points);
isolatorAngle = rect.angle;
}
/**
Step 4: Find Isolator and its orientation
Currently we use around 0.75 pixel per micron. So, 10 microns correspond to 7.5 pixels.
We reduce width and height to 1/4, which makes 10 microns correspond to 1.875 pixels, 50 microns --> 9.375 pixels,
We assume the isolator has high texture, and therefore high gradients, and the regions between isolator and target has less gradients.
*/
void caasCLR4Tx1::FindIsolator()
{
int high = targetBottomEdge - targetTopEdge, top = targetTopEdge + high / 4;
RoiTargetLeftMiddleHalf = Rect(0, top, targetLeftEdge, high / 2); //We assume the isolator will be in the middle part of the target.
imageTargetLeftMiddleHalf = imageGray(RoiTargetLeftMiddleHalf);
#if _DEBUG
imwrite("4.1.TargetLeftMiddleHalf.jpg", imageTargetLeftMiddleHalf);
#endif
int scale = 4; //We reduce the original image resolution
resize(imageTargetLeftMiddleHalf, imageOneFourthTargetLeftMiddleHalf, Size(imageTargetLeftMiddleHalf.cols / scale, imageTargetLeftMiddleHalf.rows / scale));
#if _DEBUG
imwrite("4.2.OneFourthTargetLeftMiddleHalf.jpg", imageOneFourthTargetLeftMiddleHalf);
#endif
//sharpen the image
//Unsharping masking: Use a Gaussian smoothing filter and subtract the smoothed version from the original image (in a weighted way so the values of a constant area remain constant).
Mat imageBlurred, imageGraySharpened; double GAUSSIAN_RADIUS = 4.0;
GaussianBlur(imageOneFourthTargetLeftMiddleHalf, imageBlurred, Size(0, 0), GAUSSIAN_RADIUS);
addWeighted(imageOneFourthTargetLeftMiddleHalf, 1.5, imageBlurred, -0.5, 0, imageGraySharpened);
#if _DEBUG
//imageGraySharpened = imageGrayQuarter;
imwrite("4.3.OneFourthTargetLeftMiddleHalfSharpened.jpg", imageGraySharpened);
#endif
//Histogram Equalization
equalizeHist(imageGraySharpened, imageGraySharpened);
#if _DEBUG
imwrite("4.4.OneFourthTargetLeftMiddleHalfEqualized.jpg", imageGraySharpened);
#endif
//Canny Edge Detection
int median = Median(imageGraySharpened);
Canny(imageGraySharpened, imageOneFourthTargetLeftMiddleHalfCanny, 0.66 * median, 1.33 * median);
#if _DEBUG
imwrite("4.5.OneFourthTargetLeftMiddleHalfCanny.jpg", imageOneFourthTargetLeftMiddleHalfCanny);
#endif
{ //Find the isolator right edge
//vertical projection profile
Mat verProjection(1, imageOneFourthTargetLeftMiddleHalfCanny.cols, CV_32FC1);
reduce(imageOneFourthTargetLeftMiddleHalfCanny, verProjection, 0, CV_REDUCE_SUM, CV_32FC1); //Vertical projection generate Horizontal profile
float minValue = -1.0, maxValue = -1.0; int minIndex, maxIndex; float values[4000];
ProjectionProfileAnalysis(verProjection, minValue, minIndex, maxValue, maxIndex, values);
//Set the start and end 3 to be zero for convenience
vector<int> starts, ends; int start, end;
for (int i = 0; i < 3; i++) { values[verProjection.cols - 1 - i] = 0; values[i] = 0; }
//Find all non-zero runs
for (int i = verProjection.cols - 2; i >= 0; i--)
{
if (values[i + 1] < 0.5 && values[i] < 0.5) continue;
if (values[i + 1] > 0.5 && values[i] > 0.5) continue;
if (values[i + 1] < 0.5 && values[i] > 0.5) { end = i; continue; }
if (values[i + 1] > 0.5 && values[i] < 0.5)
{
start = i + 1; starts.push_back(start); ends.push_back(end);
}
}
//Check every runs, cut 3/4 of the expected isolator width, and we expect to see many edges (high profile values)
for (int i = 0; i < starts.size(); i++)
{
end = ends[i]; start = end - 3 * (isolatorWidth / scale) / 4;
float average = 0;
for (int k = start; k <= end; k++) average += values[k];
average /= (float)(end - start + 1);
if (average < 3) continue; //Has to be larger than a threshold
isolatorRightEdge = end * scale;
break;
}
}
{ //Find isolator top and bottom edges
//We just look at the half width of the isolator, trying to find its top and bottom edges
isolatorLeftEdge = isolatorRightEdge - isolatorWidth / 2;
Rect roi = Rect(isolatorLeftEdge / scale, 0, (isolatorRightEdge - isolatorLeftEdge) / scale, imageOneFourthTargetLeftMiddleHalfCanny.rows);
imageIsolatorRoiCanny = imageOneFourthTargetLeftMiddleHalfCanny(roi);
#if _DEBUG
imwrite("4.6.IsolatorRoiCanny.jpg", imageIsolatorRoiCanny);
#endif
//horizontal projection profile
Mat horProjection(imageIsolatorRoiCanny.rows, 1, CV_32FC1);
reduce(imageIsolatorRoiCanny, horProjection, 1, CV_REDUCE_SUM, CV_32FC1); //horizontal projection generate vertical profile
float minValue = -1.0, maxValue = -1.0; int minIndex, maxIndex; float values[4000];
ProjectionProfileAnalysis(horProjection, minValue, minIndex, maxValue, maxIndex, values);
//Find all 0's and remove 0's which are smaller than a threshold
float prev, curr; vector<int> starts, ends;
for (int i = 0; i <= imageIsolatorRoiCanny.rows; i++)
{
prev = i == 0 ? prev = 1 : values[i - 1];
curr = i == imageIsolatorRoiCanny.rows ? 1 : values[i];
if (prev > 0.5 && curr < 0.5) starts.push_back(i);
if (prev < 0.5 && curr > 0.5) ends.push_back(i);
}
if (starts.size() == 0)
{
isolatorTopEdge = roi.y * scale;
isolatorBottomEdge = (roi.y + roi.height) * scale;
}
else
{
vector<int> zero_starts, zero_ends;
for (int i = 0; i < starts.size(); i++)
{
int length = ends[i] - starts[i];
if (length > 5) { zero_starts.push_back(starts[i]); zero_ends.push_back(ends[i]); }
}
//Pick one 1's, which is closest to the expected isolator width
vector<int> one_starts, one_ends;
if (zero_starts[0] != 0) { one_starts.push_back(0); one_ends.push_back(zero_starts[0]); }
for (int i = 0; i < zero_starts.size() - 1; i++)
{
one_starts.push_back(zero_ends[i]);
one_ends.push_back(zero_starts[i + 1]);
}
if (zero_ends[zero_ends.size() - 1] != imageIsolatorRoiCanny.rows) { one_starts.push_back(zero_ends[zero_ends.size() - 1]); one_ends.push_back(imageIsolatorRoiCanny.cols); }
int min_index, min_diff = 4000;
for (int i = 0; i < one_starts.size(); i++)
{
int length = one_ends[i] - one_starts[i];
int diff = std::abs(length - this->isolatorWidth / scale);
if (diff < min_diff) { min_diff = diff; min_index = i; }
}
isolatorTopEdge = this->RoiTargetLeftMiddleHalf.y + (roi.y + one_starts[min_index]) * scale;
isolatorBottomEdge = this->RoiTargetLeftMiddleHalf.y + (roi.y + one_ends[min_index]) * scale;
}
#if _DEBUG
roi = Rect(isolatorLeftEdge, isolatorTopEdge, isolatorRightEdge - isolatorLeftEdge, isolatorBottomEdge - isolatorTopEdge);
imageIsolator = imageGray(roi);
imwrite("4.7.Isolator.jpg", imageIsolator);
#endif
}
}
/**
Step 3: Find the top and bottom edge of the target.
We assume that the regions above and below the target are darker than the target.
*/
void caasCLR4Tx1::FindTargetTopBottomEdges()
{
int scale = 4;
Rect roi = Rect(targetLeftEdge / scale, 0, (targetRightEdge - targetLeftEdge) / scale, this->imageOneFourth.rows);
imageOneFourthTarget = this->imageOneFourth(roi);
#if _DEBUG
imwrite("3.1.OneFourthTarget.jpg", imageOneFourthTarget);
#endif
Mat imageOtsu; threshold(imageOneFourthTarget, imageOtsu, 0, 255, CV_THRESH_BINARY | CV_THRESH_OTSU);
#if _DEBUG
imwrite("3.2.OneFourthTargetOtsu.jpg", imageOtsu);
#endif
//horizontal projection profile
Mat horProjection(imageOtsu.rows, 1, CV_32FC1);
reduce(imageOtsu, horProjection, 1, CV_REDUCE_SUM, CV_32FC1); //horizontal projection generate vertical profile
float minValue = -1.0, maxValue = -1.0; int minIndex, maxIndex; float values[3000]; float gradients[3000];
ProjectionProfileAnalysis(horProjection, minValue, minIndex, maxValue, maxIndex, values);
Gradient(horProjection.rows, values, gradients);
//Find the top edge of the target
//From top to bottom, this is a positive gradient, i.e., from low gray value to high gray value
float maxGrad = gradients[0]; int maxGradIndex = 0;
for (int i = 1; i < horProjection.rows / 3; i++) //We search in only top 1/3
{
if (gradients[i] > maxGrad) { maxGrad = gradients[i]; maxGradIndex = i; }
}
targetTopEdge = maxGradIndex * scale;
//Find the bottom edge of the target
//From top to bottom, this is a negative gradient, i.e., from high gray value to low gray value
float minGrad = gradients[horProjection.rows - 1]; int minGradIndex = horProjection.rows - 1;
for (int i = horProjection.rows - 2; i >= 2 * horProjection.rows / 3; i--) //We search in only bottom 1/3
{
if (gradients[i] < minGrad) { minGrad = gradients[i]; minGradIndex = i; }
}
targetBottomEdge = minGradIndex * scale;
#if _DEBUG
roi = Rect(targetLeftEdge, targetTopEdge, targetRightEdge - targetLeftEdge, targetBottomEdge - targetTopEdge);
imageTarget = imageGray(roi);
imwrite("3.3.Target.jpg", imageTarget);
#endif
return;
////Find the top edge of the target
//for (int i = 0; i < horProjection.rows; i++)
//{
// if (values[i] > maxValue / 2)
// {
// targetTopEdge = i*scale; break;
// }
//}
////Find the bottom edge of the target
//for (int i = horProjection.rows - 1; i >= 0; i--)
//{
// if (values[i] > maxValue / 2)
// {
// targetBottomEdge = i*scale; break;
// }
//}
//return;
}
cv::Mat CameraUtils::getImageFromCameraProxy(jderobot::ImageDataPtr dataPtr) {
cv::Mat outImage;
colorspaces::Image::FormatPtr fmt;
fmt = colorspaces::Image::Format::searchFormat(dataPtr->description->format);
if (!fmt)
throw "Format not supported";
if (dataPtr->description->format == colorspaces::ImageRGB8::FORMAT_RGB8_Z.get()->name ||
dataPtr->description->format == colorspaces::ImageRGB8::FORMAT_DEPTH8_16_Z.get()->name )
{
size_t dest_len = dataPtr->description->width*dataPtr->description->height*3;
size_t source_len = dataPtr->pixelData.size();
unsigned char* origin_buf = (uchar*) malloc(dest_len);
int r = uncompress((Bytef *) origin_buf, (uLongf *) &dest_len, (const Bytef *) &(dataPtr->pixelData[0]), (uLong)source_len);
if(r != Z_OK) {
fprintf(stderr, "[CMPR] Error:\n");
switch(r) {
case Z_MEM_ERROR:
fprintf(stderr, "[CMPR] Error: Not enough memory to compress.\n");
break;
case Z_BUF_ERROR:
fprintf(stderr, "[CMPR] Error: Target buffer too small.\n");
break;
case Z_STREAM_ERROR: // Invalid compression level
fprintf(stderr, "[CMPR] Error: Invalid compression level.\n");
break;
}
}
else
{
colorspaces::Image imageRGB(dataPtr->description->width,dataPtr->description->height,colorspaces::ImageRGB8::FORMAT_RGB8,&(origin_buf[0]));
colorspaces::ImageRGB8 img_rgb888(imageRGB);//conversion will happen if needed
cv::Mat(cvSize(img_rgb888.width,img_rgb888.height), CV_8UC3, img_rgb888.data).copyTo(outImage);
img_rgb888.release();
}
if (origin_buf)
free(origin_buf);
}
else if (dataPtr->description->format == colorspaces::ImageRGB8::FORMAT_RGB8.get()->name ||
dataPtr->description->format == colorspaces::ImageRGB8::FORMAT_DEPTH8_16.get()->name )
{
colorspaces::Image imageRGB(dataPtr->description->width,dataPtr->description->height,colorspaces::ImageRGB8::FORMAT_RGB8,&(dataPtr->pixelData[0]));
colorspaces::ImageRGB8 img_rgb888(imageRGB);//conversion will happen if needed
cv::Mat(cvSize(img_rgb888.width,img_rgb888.height), CV_8UC3, img_rgb888.data).copyTo(outImage);
img_rgb888.release();
}
else if (dataPtr->description->format == colorspaces::ImageGRAY8::FORMAT_GRAY8_Z.get()->name) {
//gay compressed
size_t dest_len = dataPtr->description->width*dataPtr->description->height;
size_t source_len = dataPtr->pixelData.size();
unsigned char* origin_buf = (uchar*) malloc(dest_len);
int r = uncompress((Bytef *) origin_buf, (uLongf *) &dest_len, (const Bytef *) &(dataPtr->pixelData[0]), (uLong)source_len);
if(r != Z_OK) {
fprintf(stderr, "[CMPR] Error:\n");
switch(r) {
case Z_MEM_ERROR:
fprintf(stderr, "[CMPR] Error: Not enough memory to compress.\n");
break;
case Z_BUF_ERROR:
fprintf(stderr, "[CMPR] Error: Target buffer too small.\n");
break;
case Z_STREAM_ERROR: // Invalid compression level
fprintf(stderr, "[CMPR] Error: Invalid compression level.\n");
break;
}
}
else
{
colorspaces::Image imageGray(dataPtr->description->width,dataPtr->description->height,colorspaces::ImageGRAY8::FORMAT_GRAY8,&(origin_buf[0]));
colorspaces::ImageGRAY8 img_gray8(imageGray);//conversion will happen if needed
cv::Mat(cvSize(img_gray8.width,img_gray8.height), CV_8UC1, img_gray8.data).copyTo(outImage);
img_gray8.release();
}
if (origin_buf)
free(origin_buf);
}
else if (dataPtr->description->format == colorspaces::ImageGRAY8::FORMAT_GRAY8.get()->name){
colorspaces::Image imageGray(dataPtr->description->width,dataPtr->description->height,colorspaces::ImageGRAY8::FORMAT_GRAY8,&(dataPtr->pixelData[0]));
colorspaces::ImageGRAY8 img_gray8(imageGray);//conversion will happen if needed
cv::Mat(cvSize(img_gray8.width,img_gray8.height), CV_8UC1, img_gray8.data).copyTo(outImage);
img_gray8.release();
}
else{
LOG(ERROR) << "Unkown image format";
}
return outImage;
}
