bp::object doBPySpectralResidualSaliency(bp::object FullsizedImage)
{
NDArrayConverter cvt;
cv::Mat srcImgCPP = cvt.toMat(FullsizedImage.ptr());
std::vector<cv::Mat> foundCrops;
std::vector<std::pair<double,double>> cropGeolocations;
SpectralResidualSaliencyClass saldoer;
saldoer.ProcessSaliency(&srcImgCPP, &foundCrops, &cropGeolocations, 0);
consoleOutput.Level1() << "SpectralResidualSaliency found " << to_istring(foundCrops.size()) << " crops" << std::endl;
std::vector<bp::object> foundCropsPy;
for(int ii=0; ii<foundCrops.size(); ii++) {
foundCropsPy.append(bp::object(cvt.toNDArray()));
}
return bp::str(shapename.c_str());
}
void ShapeRecModuleAlgorithm_SingleImage_Turning::CompareTargetContourToAllReferenceShapes(std::vector<ShapeRec_Result> & returned_results,
std::vector<cv::Point> & target_contour,
std::vector<cv::Point> & target_polynomial_contour,
const std::string& folder_containing_reference_images)
{
returned_results.clear();
if(folder_containing_reference_images.empty())
{
consoleOutput.Level0() << "ShapeRecognizer - warning - was not given a folder containing reference images!!" << std::endl;
}
bool found_at_least_one = false;
std::string ref_filename;
std::ifstream myfile;
tinydir_dir dir;
std::string filename;
std::string filename_full_path;
std::string filename_extension;
//-----------------------------------------------------------------
// open reference shape files from disk (and compare each one to the shape)
// they shouldn't be loaded from disk every single time shape recognition is done;
// the contours from reference shapes should be saved in memory when the program first starts up
//-----------------------------------------------------------------
tinydir_open(&dir, folder_containing_reference_images.c_str());
while(dir.has_next)
{
tinydir_file file;
tinydir_readfile(&dir, &file);
if(file.is_dir == false)
{
filename = std::string(file.name);
if(filename.size() > 4)
{
filename_extension = eliminate_extension_from_filename(filename);
//now "filename" has neither the extension nor the folder path
if(filename_extension_is_image_type(filename_extension))
{
filename_full_path = std::string(file.path);
cv::Mat ref_mat = cv::imread(filename_full_path, CV_LOAD_IMAGE_GRAYSCALE);
if(ref_mat.empty() == false)
{
found_at_least_one = true;
cv::threshold(ref_mat, ref_mat, 128, 255, cv::THRESH_BINARY);
returned_results.push_back(CompareReferenceMatToTargetContour(ref_mat, target_contour, target_polynomial_contour));
returned_results.rbegin()->reference_shape_name = filename;
}
}
}
}
tinydir_next(&dir);
}
tinydir_close(&dir);
#if 0
for(int aaa=0; aaa < ShapeRecognizer::ShapeNamesTable.size(); aaa++)
{
ref_filename = (folder_containing_reference_images + std::string("/") + to_istring(aaa) + std::string(".jpg"));
myfile.open(ref_filename, std::ifstream::in);
if(myfile.is_open()) //just check if the file exists
{
myfile.close();
found_at_least_one = true;
cv::Mat ref_mat = cv::imread(ref_filename, 0);
cv::threshold(ref_mat, ref_mat, 128, 255, cv::THRESH_BINARY);
returned_results.push_back(CompareReferenceMatToTargetContour(ref_mat, target_contour, target_polynomial_contour));
returned_results.rbegin()->reference_shape = aaa;
}
}
#endif
if(found_at_least_one == false)
consoleOutput.Level0() << "\nDid not find any reference shape images!!" << std::endl;
}
void SaliencyModule_C_SBD::do_saliency_with_setting(cv::Mat input_image, SaliencySettings& setting)
{
cv::Mat resized, dst, src_gray, detected_edges;
std::string filenamestr;
if(input_image.cols < 20 || input_image.rows < 20)
return;
/// Step 1: resize to 100x smaller (start w/ 18MP)
cv::resize(input_image, resized, cv::Size(0,0), 0.1, 0.1);
/// Create a matrix of the same type and size as resized (for dst)
dst.create( resized.size(), resized.type() );
ConvertMat_UsingSettings(resized, resized, setting.ColorConvert_CV_color_space, setting.ColorConvert_desired_CV_color_channels, false);
if(write_output_to_folder && write_ALL_output_not_just_the_crops)
{
cv::imwrite(output_folder_to_save_crops + std::string("/z_saliency") + to_istring(write_output_to_folder_name_incrementer) + std::string("_step0_after_cielab.jpg"), resized);
}
/// Convert the image to grayscale
///////////cv::vector<cv::Mat> spl;
std::vector<cv::Mat> spl(3);
cv::split(resized, spl);
int channel;
dst = cv::Scalar::all(0);
for(channel=0; channel < 3; ++channel)
{
/////////////src_gray = spl[channel];
spl[channel].copyTo(src_gray);
/// Noise reduction
cv::blur(src_gray, src_gray, cv::Size(3,3) );
/// Canny detector
cv::Canny(src_gray, detected_edges, setting.Canny_low_threshold, setting.Canny_high_threshold, setting.Canny_kernel_size);
if(write_output_to_folder && write_ALL_output_not_just_the_crops)
{
filenamestr = output_folder_to_save_crops + std::string("/z_saliency_step5_after_canny") + to_istring(channel) + std::string(".jpg");
cv::imwrite(filenamestr.c_str(), detected_edges);
}
/// Using Canny's output as a mask, we display our result
resized.copyTo(dst, detected_edges);
}
// blob detection
cv::SimpleBlobDetector::Params params;
params.minDistBetweenBlobs = 10.0f;
params.filterByInertia = false;
params.filterByConvexity = false;
params.filterByColor = true;
params.filterByCircularity = false;
params.filterByArea = true;
params.minArea = 0.1f;
params.maxArea = 100.0f;
cv::Ptr<cv::FeatureDetector> blob_detector = new cv::SimpleBlobDetector(params);
blob_detector->create("SimpleBlob");
cv::vector<cv::KeyPoint> keypoints;
blob_detector->detect(dst, keypoints);
cv::Mat out;
out.create(dst.size(), dst.type());
cv::drawKeypoints( dst, keypoints, out, CV_RGB(0,255,0), cv::DrawMatchesFlags::DEFAULT);
cv::Rect toCrop;
cv::Mat cropped;
//crop size should depend on expected target size... or be determined by the size of the blob!
int cropsize_x, cropsize_y;
int startx, starty;
// crop from the full res image
for(int i=0; i<keypoints.size(); i++)
{
//-------------------------------------------------------
if(write_output_to_folder)
{
consoleOutput.Level1() << std::string("z_saliency") << to_istring(write_output_to_folder_name_incrementer) << std::string("_roi_")
<< to_istring(i) << std::string(" size: ") << keypoints[i].size << std::endl;
}
//-------------------------------------------------------
//formula for crop size:
// sqrt(area) converts units from (length)^2 to (length)... (best for squares, okay for anything else)
// then multiply this (length) unit by a scale... roughly 130 is a good starting point
cropsize_y = cropsize_x = static_cast<int>(sqrt(keypoints[i].size) * 160.0);
if((cropsize_x % 2) != 0)
{
cropsize_y = (++cropsize_x);
}
//if input is 17.92 megapixel, this allows crops up to 1000x1000 pixels
//the largest crops from 2013 that were actual targets were like 450x450
double max_percentage_of_fullsize_image_to_allow_crop = 5.582;
if(cropsize_x*cropsize_y > RoundDoubleToInt(0.01 * max_percentage_of_fullsize_image_to_allow_crop * static_cast<double>(input_image.cols*input_image.rows)))
return;
cropped.create(cv::Size(cropsize_x,cropsize_y), out.type());
startx = (int)keypoints[i].pt.x * 10 - (cropsize_x/2);
starty = (int)keypoints[i].pt.y * 10 - (cropsize_y/2);
if(startx < 0)
startx = 0;
if(starty < 0)
starty = 0;
if(startx > input_image.cols - cropsize_x)
startx = input_image.cols - cropsize_x;
if(starty > input_image.rows - cropsize_y)
starty = input_image.rows - cropsize_y;
toCrop = cv::Rect(startx, starty, cropsize_x, cropsize_y);
cropped = cv::Mat(input_image, toCrop);
returned_cropped_images.push_back(cv::Mat());
cropped.copyTo(*returned_cropped_images.rbegin());
}
if(write_output_to_folder && write_ALL_output_not_just_the_crops)
{
filenamestr = output_folder_to_save_crops + std::string("/z_saliency") + to_istring(write_output_to_folder_name_incrementer) + std::string("_out.jpg");
cv::imwrite(filenamestr.c_str(), out);
}
}
void Segmentation_SSEG_MultiReturn::DoModule(cv::Mat cropped_target_image,
std::vector<cv::Mat>* returned_shape_segmentations,
std::vector<cv::Scalar>* returned_shape_colors,
std::string* folder_path_of_output_saved_images/*=nullptr*/,
bool save_images_and_results/*=false*/,
std::string* name_of_target_image/*=nullptr*/,
std::vector<test_data_results_segmentation*>* optional_results_info_vec/*=nullptr*/,
test_data_results_segmentation* master_results_info_segmentation_only/*=nullptr*/,
std::string* correct_shape_name/*=nullptr*/,
const char* correct_ocr_character/*=nullptr*/)
{
if(returned_shape_segmentations != nullptr && my_segmenter_singleimage_algorithm != nullptr && returned_shape_colors != nullptr)
{
//=======================================================================
bool I_was_given_a_real_character_and_shape_name_to_compare = (correct_shape_name != nullptr && correct_ocr_character != nullptr);
if(I_was_given_a_real_character_and_shape_name_to_compare)
I_was_given_a_real_character_and_shape_name_to_compare = (*correct_ocr_character != 0 && correct_shape_name->empty()==false);
int test_number = 0;
if(optional_results_info_vec == nullptr || optional_results_info_vec->size() < settings.size())
{
test_number = -1; //don't save to optional results info
}
test_data_results_segmentation* all_segmentations_test_data_checker = (master_results_info_segmentation_only != nullptr) ? new test_data_results_segmentation() : nullptr;
//=======================================================================
std::vector<Segmenter_Module_Settings>::iterator settings_iter = settings.begin();
for(; settings_iter != settings.end(); settings_iter++)
{
//=======================================================================
cv::Mat foundshape_blob_returned_mask;
cv::Mat foundshape_histogrambins_returned_for_saving;
//=======================================================================
cv::Scalar returned_blob_color;
cv::Mat foundshape_filled_binary = my_segmenter_singleimage_algorithm->findShape(cropped_target_image,
*settings_iter,
nullptr, //this would be a set of input colors to avoid, but we don't know what to avoid (this is for CSEG)
&returned_blob_color,
&foundshape_blob_returned_mask,
std::string("Shape Segmentation (SSEG)"),
&foundshape_histogrambins_returned_for_saving);
if(foundshape_filled_binary.empty() == false)
{
returned_shape_segmentations->push_back(foundshape_filled_binary);
returned_shape_colors->push_back(returned_blob_color);
consoleOutput.Level3() << std::string("--color found by SSEG setting: ")
<< to_sstring((*returned_shape_colors->rbegin())[0]) << std::string(",")
<< to_sstring((*returned_shape_colors->rbegin())[1]) << std::string(",")
<< to_sstring((*returned_shape_colors->rbegin())[2]) << std::string(",")
<< to_sstring((*returned_shape_colors->rbegin())[3]) << std::string(",")
<< std::endl;
}
//=======================================================================
if(test_number >= 0)
UpdateResultsAttemptsData_SSEG(&consoleOutput.Level2(), (test_number < 0) ? nullptr : ((*optional_results_info_vec)[test_number]), I_was_given_a_real_character_and_shape_name_to_compare);
UpdateResultsAttemptsData_SSEG(&consoleOutput.Level2(), all_segmentations_test_data_checker, I_was_given_a_real_character_and_shape_name_to_compare);
if(foundshape_filled_binary.empty() == false)
{
if(test_number >= 0)
CheckValidityOfResults_SSEG(&consoleOutput.Level2(), (test_number < 0) ? nullptr : ((*optional_results_info_vec)[test_number]),
foundshape_filled_binary.empty()==false,
I_was_given_a_real_character_and_shape_name_to_compare,
name_of_target_image);
CheckValidityOfResults_SSEG(&consoleOutput.Level2(), all_segmentations_test_data_checker,
foundshape_filled_binary.empty()==false,
I_was_given_a_real_character_and_shape_name_to_compare,
name_of_target_image);
}
if(test_number >= 0)
test_number++;
#if 0
if(save_images_and_results && folder_path_of_output_saved_images != nullptr)
{
saveImage(foundshape_histogrambins_returned_for_saving,
*folder_path_of_output_saved_images + std::string("/SSEG_HISTBINS__") + (*name_of_target_image) + std::string("___SegSetting") + to_istring(test_number) + std::string(".bmp"));
if(foundshape_filled_binary.empty() == false)
{
saveImage(foundshape_filled_binary,
*folder_path_of_output_saved_images + std::string("/SSEG__") + (*name_of_target_image) + std::string("___SegSetting") + to_istring(test_number) + std::string(".jpg"));
}
}
#endif
//=======================================================================
}
//=======================================================================
if(master_results_info_segmentation_only != nullptr)
master_results_info_segmentation_only->Update_My_SSEG_Data_DueToOther(all_segmentations_test_data_checker);
//=======================================================================
//new: average all of the shapes, to get a smoother, cleaner shape, with less noise ("signal averaging")
//
if(returned_shape_segmentations->empty()==false && returned_shape_colors->empty()==false)
{
cv::Scalar avg_color = Average_Several_CVColors(returned_shape_colors);
cv::Mat avg_shape = Average_Several_SingleChannel_CVMats(returned_shape_segmentations, 0.15f);
if(avg_shape.empty()) //the averager found that the average wasn't very accurate!
//this means segmentation wasn't consistent, so it probably didn't find anything useful
{
returned_shape_segmentations->clear();
returned_shape_colors->clear();
return;
}
cv::threshold(avg_shape, avg_shape, 100, 255, cv::THRESH_BINARY);
avg_shape.convertTo(avg_shape, CV_8U);
returned_shape_segmentations->clear();
returned_shape_colors->clear();
returned_shape_segmentations->push_back(avg_shape);
returned_shape_colors->push_back(avg_color);
//=======================================================================
#if 0
if(save_images_and_results && folder_path_of_output_saved_images != nullptr)
{
saveImage(avg_shape,
*folder_path_of_output_saved_images + std::string("/SSEG__") + (*name_of_target_image) + std::string("___avgshape___SegSetting") + to_istring(test_number) + std::string(".jpg"));
}
#endif
//=======================================================================
}
}
else
consoleOutput.Level0() << "SHAPE SEGMENTER WARNING: some stuff was nullptr, so didn't return any images!" << std::endl;
}
std::vector<cv::Mat>& SaliencyModule_C_SBD::do_saliency(cv::Mat input_image)
{
returned_cropped_images.clear();
std::vector<SaliencySettings>::iterator siter = settings.begin();
for(; siter != settings.end(); siter++)
{
do_saliency_with_setting(input_image, *siter);
}
if(write_output_to_folder && returned_cropped_images.empty() == false)
{
int img_num=0;
for(std::vector<cv::Mat>::iterator rciter = returned_cropped_images.begin(); rciter != returned_cropped_images.end(); rciter++)
{
std::string filename = output_folder_to_save_crops + std::string("/zsal_") + to_istring(write_output_to_folder_name_incrementer) + std::string("_roi_") + to_istring(img_num) + std::string(".jpg");
cv::imwrite(filename.c_str(), *rciter);
consoleOutput.Level1() << std::endl << filename << std::endl << std::endl;
img_num++;
}
}
write_output_to_folder_name_incrementer++;
return returned_cropped_images;
}
