cv::Mat getSyntheticImage(const Mat& image) {
int rand_type = rand();
Mat result = image.clone();
if (rand_type % 2 == 0) {
int ran_x = rand() % 5 - 2;
int ran_y = rand() % 5 - 2;
result = translateImg(result, ran_x, ran_y);
}
else if (rand_type % 2 != 0) {
float angle = float(rand() % 15 - 7);
result = rotateImg(result, angle);
}
return result;
}
bool SpatialAverageSpotter::train(string dirPath)
{
int count=0;
vector<vector<tuple<int,Point2f> > > features;
featureAverages.resize(codebook->size());
for (int i =0; i<codebook->size(); i++)
featureAverages[i]=Mat(BASE_SIZE,BASE_SIZE,CV_32F,Scalar(0));
DIR *dir;
struct dirent *ent;
if ((dir = opendir (dirPath.c_str())) != NULL) {
/* print all the files and directories within directory */
// Mat img;
while ((ent = readdir (dir)) != NULL) {
string fileName(ent->d_name);
// cout << "examining " << fileName << endl;
if (fileName[0] == '.' || fileName[fileName.size()-1]!='G')
continue;
Mat img = imread(dirPath+fileName, CV_LOAD_IMAGE_GRAYSCALE);
// resize(img,img,Size(0,0),2,2);
threshold(img,img,120.0,255,THRESH_BINARY);
// windowWidth += img.cols;
// windowHeight += img.rows;
// int avg=0;
// for (int x=0; x<img.cols; x++)
// for (int y=0; y<img.rows; y++)
// avg += (int)img.at<unsigned char>(y,x);
//// cout << "avg="<<avg<<"/"<<img.cols*img.rows<<" = "<<avg/(img.cols*img.rows)<<endl;
// avg /= img.cols*img.rows;
resize(img,img,Size(PRE_BASE_SIZE,PRE_BASE_SIZE*((0.0+img.rows)/img.cols)));
copyMakeBorder( img, img, BORDER_SIZE, BORDER_SIZE, BORDER_SIZE, BORDER_SIZE, BORDER_CONSTANT, 255 );
assert(img.cols > 1 && img.rows > 1);
adjustedTrainingImages.push_back(img.clone());
Point2f centerOfMass = findCenterOfMass(img);
int offsetx=(img.cols/2)-centerOfMass.x;
int offsety=(img.rows/2)-centerOfMass.y;
translateImg(img,offsetx,offsety);
vector<KeyPoint> keypoints;
Mat desc;
detectKeypoints( img,keypoints, desc);
Mat out;
cvtColor(img,out,CV_GRAY2RGB);
circle(out,centerOfMass,1,Scalar(0,0,255));
features.resize(count+1);
//double scaling = BASE_SIZE/img
for (int r=0; r<desc.rows; r++)
{
int f = codebook->quantize(desc.row(r));
Point2f offsetPoint(keypoints[r].pt.x - centerOfMass.x, keypoints[r].pt.y - centerOfMass.y);
features[count].push_back(make_tuple(f,offsetPoint));//we're ignoring the keypoint scale..
// circle(out,keypoints[r].pt,keypoints[r].size,Scalar(colorTable[f]));
Rect rec(keypoints[r].pt.x-(keypoints[r].size/2),keypoints[r].pt.y-(keypoints[r].size/2),keypoints[r].size,keypoints[r].size);
rectangle(out,rec,Scalar(colorTable[f]));
}
guassColorIn(features[count]);
imshow("learning keypoints",out);
cout << "image "<<count<<endl;
waitKey(5);
count++;
// img.release();
}
closedir (dir);
} else {
/* could not open directory */
perror ("");
return false;
}
//We now step through adjusting the scales of the various images so the guass coloring is maximized
//But we may still want to look at tfidf to see which ones should be weighted more, etc.
maximizeAlignment(features);
float max=0;
float min =99999;
float avg_max=0;
int firstx=9999;
int lastx=0;
int firsty=9999;
int lasty=0;
for (int f=0; f<codebook->size(); f++)
{
float local_max=0;
bool hitFirst=false;
for (int x=0; x<featureAverages[f].cols; x++)
for (int y=0; y<featureAverages[f].rows; y++)
{
float val = featureAverages[f].at<float>(y,x);
if (val > 300 || val < -300)
cout << "val (" << x <<","<<y<<") " << val << endl;
if (val>max) max=val;
if (val<min) min=val;
if (val>local_max) local_max=val;
if (val>WINDOW_THRESH)
{
if (!hitFirst)
{
hitFirst=true;
if (x<firstx) firstx=x;
if (y<firsty) firsty=y;
}
if (x>lastx) lastx=x;
if (y>lasty) lasty=y;
}
}
avg_max+=local_max;
}
// penalty=min+(max-min)*.2;
avg_max /= codebook->size();
penalty=avg_max*.15;//.2
// windowWidth/=count;
// windowHeight/=count;
windowWidth = lastx-firstx;
windowHeight = lasty-firsty;
cout << "window size is "<<windowWidth<<"x"<<windowHeight<<endl;
//show averages
showAverages();
return true;
}
void SpatialAverageSpotter::maximizeAlignment(vector<vector<tuple<int,Point2f> > > &features)
{
int stayCount=0;
int iters=0;
vector<bool> adjusted(features.size(),false);
while (stayCount<STAY_THRESH*features.size() && iters<MAX_REF_ITER*features.size())//20
{
iters++;
int imageNum= rand()%features.size();
int imageNumOld=imageNum;
while(1)
{
if (!adjusted[imageNum])
{
adjusted[imageNum]=true;
break;
}
imageNum = (imageNum+1)%features.size();
if ((imageNum) == imageNumOld)
{
adjusted.assign(features.size(),false);
adjusted[imageNum]=true;
// stayCount /= 2;
stayCount=0;
break;
}
}
double shiftStep = 2*(1.0-iters/(0.0+MAX_REF_ITER*features.size()));//3* DD 2* TT
if (shiftStep<1) shiftStep=1.0; //1.0 DD
double scaleStep = .03*(1.0-iters/(0.0+MAX_REF_ITER*features.size()));//.01 DD
double scaleStep2 = .06*(1.0-iters/(0.0+MAX_REF_ITER*features.size()));// .015 DD
if (scaleStep<1) scaleStep=.003; //DD remove
if (scaleStep2<1) scaleStep2=.006; //DD remove
//examine gradient at each f point (discluding self)
//if gradients agrre, make scale step
guassColorOut(features[imageNum]);
double bestScore=0;
ExemplarTransform bestMove=STAY;
double leftScore=0;
double rightScore=0;
double upScore=0;
double downScore=0;
double smallerScore=0;
double biggerScore=0;
double shortenScore=0;
double hightenScore=0;
double slimScore=0;
double fattenScore=0;
// double leftSmallerScore=0;
// double rightSmallerScore=0;
// double upSmallerScore=0;
// double downSmallerScore=0;
// double leftBiggerScore=0;
// double rightBiggerScore=0;
// double upBiggerScore=0;
// double downBiggerScore=0;
for (tuple<int,Point2f> f : features[imageNum])
{
bestScore += getUsingOffset(featureAverages[get<0>(f)],get<1>(f).x,get<1>(f).y);
leftScore += getUsingOffset(featureAverages[get<0>(f)], get<1>(f).x-shiftStep, get<1>(f).y);
rightScore += getUsingOffset(featureAverages[get<0>(f)], get<1>(f).x+shiftStep, get<1>(f).y);
upScore += getUsingOffset(featureAverages[get<0>(f)], get<1>(f).x, get<1>(f).y-shiftStep);
downScore += getUsingOffset(featureAverages[get<0>(f)], get<1>(f).x, get<1>(f).y+shiftStep);
smallerScore += getUsingOffset(featureAverages[get<0>(f)], get<1>(f).x*(1-scaleStep), get<1>(f).y*(1-scaleStep));
biggerScore += getUsingOffset(featureAverages[get<0>(f)], get<1>(f).x*(1+scaleStep), get<1>(f).y*(1+scaleStep));
shortenScore += getUsingOffset(featureAverages[get<0>(f)], get<1>(f).x,get<1>(f).y*(1-scaleStep2));
hightenScore += getUsingOffset(featureAverages[get<0>(f)], get<1>(f).x, get<1>(f).y*(1+scaleStep2));
slimScore += getUsingOffset(featureAverages[get<0>(f)], get<1>(f).x*(1-scaleStep2), get<1>(f).y);
fattenScore += getUsingOffset(featureAverages[get<0>(f)], get<1>(f).x*(1+scaleStep2), get<1>(f).y);
// leftSmallerScore += getUsingOffset(featureAverages[get<0>(f)], (get<1>(f).x-shiftStep)*(1-scaleStep), get<1>(f).y*(1-scaleStep));
// rightSmallerScore += getUsingOffset(featureAverages[get<0>(f)], (get<1>(f).x+shiftStep)*(1-scaleStep), get<1>(f).y*(1-scaleStep));
// upSmallerScore += getUsingOffset(featureAverages[get<0>(f)], (get<1>(f).x)*(1-scaleStep), (get<1>(f).y-shiftStep)*(1-scaleStep));
// downSmallerScore += getUsingOffset(featureAverages[get<0>(f)], (get<1>(f).x)*(1-scaleStep), (get<1>(f).y+shiftStep)*(1-scaleStep));
// leftBiggerScore += getUsingOffset(featureAverages[get<0>(f)], (get<1>(f).x-shiftStep)*(1+scaleStep), get<1>(f).y*(1+scaleStep));
// rightBiggerScore += getUsingOffset(featureAverages[get<0>(f)], (get<1>(f).x+shiftStep)*(1+scaleStep), get<1>(f).y*(1+scaleStep));
// upBiggerScore += getUsingOffset(featureAverages[get<0>(f)], get<1>(f).x*(1+scaleStep), (get<1>(f).y-shiftStep)*(1+scaleStep));
// downBiggerScore += getUsingOffset(featureAverages[get<0>(f)], get<1>(f).x*(1+scaleStep), (get<1>(f).y+shiftStep)*(1+scaleStep));
}
double test=bestScore;
if (leftScore > bestScore)
{
bestScore=leftScore; bestMove = LEFT;
}
if (rightScore > bestScore)
{
bestScore=rightScore; bestMove = RIGHT;
}
if (upScore > bestScore)
{
bestScore=upScore; bestMove = UP;
}
if (downScore > bestScore)
{
bestScore=downScore; bestMove = DOWN;
}
if (smallerScore > bestScore)
{
bestScore=smallerScore; bestMove = SHRINK;
}
if (biggerScore > bestScore)
{
bestScore=biggerScore; bestMove = GROW;
}
if (shortenScore > bestScore)
{
bestScore=shortenScore; bestMove = SHORTEN;
}
if (hightenScore > bestScore)
{
bestScore=hightenScore; bestMove = HIGHTEN;
}
if (slimScore > bestScore)
{
bestScore=slimScore; bestMove = SLIM;
}
if (fattenScore > bestScore)
{
bestScore=fattenScore; bestMove = FATTEN;
}
// if (leftSmallerScore > bestScore)
// {
// bestScore=leftSmallerScore; bestMove = LEFT_SHRINK;
// }
// if (rightSmallerScore > bestScore)
// {
// bestScore=rightSmallerScore; bestMove = RIGHT_SHRINK;
// }
// if (upSmallerScore > bestScore)
// {
// bestScore=upSmallerScore; bestMove = UP_SHRINK;
// }
// if (downSmallerScore > bestScore)
// {
// bestScore=downSmallerScore; bestMove = DOWN_SHRINK;
// }
// if (leftBiggerScore > bestScore)
// {
// bestScore=leftBiggerScore; bestMove = LEFT_GROW;
// }
// if (rightBiggerScore > bestScore)
// {
// bestScore=rightBiggerScore; bestMove = RIGHT_GROW;
// }
// if (upBiggerScore > bestScore)
// {
// bestScore=upBiggerScore; bestMove = UP_GROW;
// }
// if (downBiggerScore > bestScore)
// {
// bestScore=downBiggerScore; bestMove = DOWN_GROW;
// }
auto fI = features[imageNum].begin();
for(; fI!=features[imageNum].end(); fI++)
{
switch(bestMove)
{
case LEFT:
get<1>(*fI) = Point2f(get<1>(*fI).x-shiftStep, get<1>(*fI).y);
break;
case RIGHT:
get<1>(*fI) = Point2f(get<1>(*fI).x+shiftStep, get<1>(*fI).y);
break;
case UP:
get<1>(*fI) = Point2f(get<1>(*fI).x, get<1>(*fI).y-shiftStep);
// cout << "upscore:"<<upScore<<" stayscore:"<<test<<endl;
break;
case DOWN:
get<1>(*fI) = Point2f(get<1>(*fI).x, get<1>(*fI).y+shiftStep);
break;
case SHRINK:
get<1>(*fI) = Point2f(get<1>(*fI).x*(1-scaleStep), get<1>(*fI).y*(1-scaleStep));
break;
case GROW:
get<1>(*fI) = Point2f(get<1>(*fI).x*(1+scaleStep), get<1>(*fI).y*(1+scaleStep));
break;
case SHORTEN:
get<1>(*fI) = Point2f(get<1>(*fI).x, get<1>(*fI).y*(1-scaleStep2));
break;
case HIGHTEN:
get<1>(*fI) = Point2f(get<1>(*fI).x, get<1>(*fI).y*(1+scaleStep2));
break;
case SLIM:
get<1>(*fI) = Point2f(get<1>(*fI).x*(1-scaleStep2), get<1>(*fI).y);
break;
case FATTEN:
get<1>(*fI) = Point2f(get<1>(*fI).x*(1+scaleStep2), get<1>(*fI).y);
break;
// case LEFT_SHRINK:
// get<1>(*fI) = Point2f((get<1>(*fI).x-shiftStep)*(1-scaleStep), get<1>(*fI).y*(1-scaleStep));
// break;
// case RIGHT_SHRINK:
// get<1>(*fI) = Point2f((get<1>(*fI).x+shiftStep)*(1-scaleStep), get<1>(*fI).y*(1-scaleStep));
// break;
// case UP_SHRINK:
// get<1>(*fI) = Point2f((get<1>(*fI).x)*(1-scaleStep), (get<1>(*fI).y-shiftStep)*(1-scaleStep));
// break;
// case DOWN_SHRINK:
// get<1>(*fI) = Point2f((get<1>(*fI).x)*(1-scaleStep), (get<1>(*fI).y+shiftStep)*(1-scaleStep));
// break;
// case LEFT_GROW:
// get<1>(*fI) = Point2f((get<1>(*fI).x-shiftStep)*(1+scaleStep), get<1>(*fI).y*(1+scaleStep));
// break;
// case RIGHT_GROW:
// get<1>(*fI) = Point2f((get<1>(*fI).x+shiftStep)*(1+scaleStep), get<1>(*fI).y*(1+scaleStep));
// break;
// case UP_GROW:
// get<1>(*fI) = Point2f(get<1>(*fI).x*(1+scaleStep), (get<1>(*fI).y-shiftStep)*(1+scaleStep));
// break;
// case DOWN_GROW:
// get<1>(*fI) = Point2f(get<1>(*fI).x*(1+scaleStep), (get<1>(*fI).y+shiftStep)*(1+scaleStep));
// break;
case STAY:
break;
}
}
if(bestMove == STAY)
{
stayCount+=1;
}
else if (stayCount>0)
{
stayCount--;
}
//////
switch(bestMove)
{
case LEFT:
translateImg(adjustedTrainingImages[imageNum],-shiftStep,0);
break;
case RIGHT:
translateImg(adjustedTrainingImages[imageNum],shiftStep,0);
break;
case UP:
translateImg(adjustedTrainingImages[imageNum],0,-shiftStep);
break;
case DOWN:
translateImg(adjustedTrainingImages[imageNum],0,shiftStep);
break;
case SHRINK:
strechImg(adjustedTrainingImages[imageNum],(1-scaleStep),(1-scaleStep));
break;
case GROW:
strechImg(adjustedTrainingImages[imageNum],(1+scaleStep),(1+scaleStep));
break;
case SHORTEN:
strechImg(adjustedTrainingImages[imageNum],1,(1-scaleStep2));
break;
case HIGHTEN:
strechImg(adjustedTrainingImages[imageNum],1,(1+scaleStep2));
break;
case SLIM:
strechImg(adjustedTrainingImages[imageNum],(1-scaleStep2),1);
break;
case FATTEN:
strechImg(adjustedTrainingImages[imageNum],(1+scaleStep2),1);
break;
case STAY:
break;
}
//////
guassColorIn(features[imageNum]);
cout <<"image "<<imageNum<<" move "<<bestMove<<", staycount="<<stayCount<<endl;
showAverages();
Mat averageImage(adjustedTrainingImages[0].rows,adjustedTrainingImages[0].cols,CV_8U);
for (int x=0; x< averageImage.cols; x++)
for (int y=0; y<averageImage.rows; y++)
{
int sum=0;
for (Mat image : adjustedTrainingImages)
{
sum += image.at<unsigned char>(y,x);
}
sum /= adjustedTrainingImages.size();
assert(sum>=0 && sum<256);
averageImage.at<unsigned char>(y,x) = sum;
}
imshow("average",averageImage);
waitKey(5);
// imshow("adjusted",adjustedTrainingImages[imageNum]);
// waitKey();
}
Mat averageImage(adjustedTrainingImages[0].rows,adjustedTrainingImages[0].cols,CV_8U);
for (int x=0; x< averageImage.cols; x++)
for (int y=0; y<averageImage.rows; y++)
{
int sum=0;
for (Mat image : adjustedTrainingImages)
{
sum += image.at<unsigned char>(y,x);
}
sum /= adjustedTrainingImages.size();
assert(sum>=0 && sum<256);
averageImage.at<unsigned char>(y,x) = sum;
}
int i=0;
for (Mat image : adjustedTrainingImages)
{
imshow("adjusted",image);
cout << "image "<<i++<<endl;
waitKey(5);
}
imshow("average",averageImage);
waitKey(5);
}
