// Main function
int main(const int argc, const char **argv)
{
MYLOGVERB = LOG_INFO; // suppress debug messages
// Instantiate a ModelManager:
ModelManager manager("Gist Features Extraction");
// we cannot use saveResults() on our various ModelComponent objects
// here, so let's not export the related command-line options.
manager.allowOptions(OPTEXP_ALL & (~OPTEXP_SAVE));
// Instantiate our various ModelComponents:
nub::soft_ref<SimEventQueueConfigurator>
seqc(new SimEventQueueConfigurator(manager));
manager.addSubComponent(seqc);
nub::soft_ref<InputMPEGStream>
ims(new InputMPEGStream(manager, "Input MPEG Stream", "InputMPEGStream"));
manager.addSubComponent(ims);
nub::soft_ref<StdBrain> brain(new StdBrain(manager));
manager.addSubComponent(brain);
nub::ref<SpatialMetrics> metrics(new SpatialMetrics(manager));
manager.addSubComponent(metrics);
manager.exportOptions(MC_RECURSE);
metrics->setFOAradius(30); // FIXME
metrics->setFoveaRadius(30); // FIXME
// setting up the GIST ESTIMATOR
manager.setOptionValString(&OPT_GistEstimatorType,"Std");
//manager.setOptionValString(&OPT_GistEstimatorType,"FFT");
// Request a bunch of option aliases (shortcuts to lists of options):
REQUEST_OPTIONALIAS_NEURO(manager);
// Parse command-line:
if (manager.parseCommandLine(argc, argv, "<*.mpg or *_gistList.txt>",
1, 1) == false)
return(1);
nub::soft_ref<SimEventQueue> seq = seqc->getQ();
// if the file passed ends with _gistList.txt
// we have a different protocol
bool isGistListInput = false;
int ifLen = manager.getExtraArg(0).length();
if(ifLen > 13 &&
manager.getExtraArg(0).find("_gistList.txt",ifLen - 13) !=
std::string::npos)
isGistListInput = true;
// NOTE: this could now be controlled by a command-line option
// --preload-mpeg=true
manager.setOptionValString(&OPT_InputMPEGStreamPreload, "true");
// do post-command-line configs:
std::vector<std::string> tag;
std::vector<int> start;
std::vector<int> num;
unsigned int cLine = 0; int cIndex = 0;
if(isGistListInput)
{
LINFO("we have a gistList input");
getGistFileList(manager.getExtraArg(0).c_str(), tag, start, num);
cIndex = start[0];
}
else
{
LINFO("we have an mpeg input");
ims->setFileName(manager.getExtraArg(0));
manager.setOptionValString(&OPT_InputFrameDims,
convertToString(ims->peekDims()));
}
// frame delay in seconds
//double fdelay = 33.3667/1000.0; // real time
double fdelay = 3.3667/1000.0;
// let's get all our ModelComponent instances started:
manager.start();
// get the GistEstimator
nub::soft_ref<GistEstimatorStd> ge;////// =
/////// dynCastWeak<GistEstimatorStd>(brain->getGE());
LFATAL("fixme");
if (ge.isInvalid()) LFATAL("I am useless without a GistEstimator");
// MAIN LOOP
SimTime prevstime = SimTime::ZERO();
int fNum = 0;
Image< PixRGB<byte> > inputImg; Image< PixRGB<byte> > dispImg;
Image<double> cgist;
std::string folder = "";
std::string::size_type sPos = manager.getExtraArg(0).rfind("/",ifLen);
if(sPos != std::string::npos)
folder = manager.getExtraArg(0).substr(0,sPos+1);
LINFO("let's start");
while(1)
{
// has the time come for a new frame?
if (fNum == 0 ||
(seq->now() - 0.5 * (prevstime - seq->now())).secs() - fNum * fdelay > fdelay)
{
// load new frame
std::string fName;
if(isGistListInput)
{
if (cLine >= tag.size()) break; // end of input list
// open the current file
char tNumStr[100]; sprintf(tNumStr,"%06d",cIndex);
fName = folder + tag[cLine] + std::string(tNumStr) + ".ppm";
inputImg = Raster::ReadRGB(fName);
cIndex++;
if(cIndex >= start[cLine] + num[cLine])
{
cLine++;
if (cLine < tag.size()) cIndex = start[cLine];
}
// reformat the file name to a gist name
int fNameLen = fName.length();
unsigned int uPos = fName.rfind("_",fNameLen);
fName = fName.substr(0,uPos)+ ".ppm";
}
else
{
fName = manager.getExtraArg(0);
inputImg = ims->readRGB(); //Raster::ReadRGB(manager.getExtraArg(1));
if (inputImg.initialized() == false) break; // end of input stream
// format new frame
inputImg = crop(inputImg,
Rectangle(Point2D<int>(0,25),
Dims(inputImg.getHeight(),
inputImg.getWidth()-25+1)));
cIndex = fNum+1;
}
dispImg = inputImg;
LINFO("\nnew frame :%d",fNum);
// pass input to brain:
rutz::shared_ptr<SimEventInputFrame>
e(new SimEventInputFrame(brain.get(), GenericFrame(inputImg), 0));
seq->post(e); // post the image to the brain
// get the gist feature vector
cgist = ge->getGist();
//for(uint k = 0; k < cgist.getSize(); k++) LINFO("%d: %f",k, cgist.getVal(0,k));
// // setup display at the start of stream
// if (fNum == 0)
// {
// int s = SQ_SIZE;
// inputWin = new XWinManaged(Dims(w, h), 0, 0, manager.getExtraArg(0).c_str());
// wList.add(inputWin);
// gistWin = new XWinManaged(Dims(NUM_GIST_COL * s, NUM_GIST_FEAT * s), 0,0, "Gist");
// wList.add(gistWin);
// }
// // display the input image and the gist histogram
// drawGrid(dispImg, w/4,h/4,1,1,PixRGB<byte>(255,255,255));
// inputWin->drawImage(dispImg,0,0);
// gistWin->drawImage(ge->getGistHistogram(SQ_SIZE),0,0);
// SAVE GIST FEATURES TO A FILE
saveData(cgist, fName, cIndex-1);
//LINFO("\nFrame number just saved:%d",fNum);Raster::waitForKey();
// increase frame count
fNum++;
}
// evolve brain:
prevstime = seq->now(); // time before current step
const SimStatus status = seq->evolve();
if (SIM_BREAK == status) // Brain decided it's time to quit
break;
}
// stop all our ModelComponents
manager.stop();
// all done!
return 0;
}
int main(const int argc, const char **argv)
{
MYLOGVERB = LOG_INFO;
mgr = new ModelManager("Test ObjRec");
nub::soft_ref<SimEventQueueConfigurator>
seqc(new SimEventQueueConfigurator(*mgr));
mgr->addSubComponent(seqc);
//our brain
nub::ref<StdBrain> brain(new StdBrain(*mgr));
mgr->addSubComponent(brain);
mgr->exportOptions(MC_RECURSE);
mgr->setOptionValString(&OPT_VisualCortexType, "IOC");
//mgr.setOptionValString(&OPT_VisualCortexType, "I");
//mgr->setOptionValString(&OPT_VisualCortexType, "GNO");
//mgr.setOptionValString(&OPT_VisualCortexType, "N");
//manager.setOptionValString(&OPT_UseOlderVersion, "false");
// set the FOA and fovea radii
mgr->setOptionValString(&OPT_SaliencyMapType, "Fast");
mgr->setOptionValString(&OPT_SMfastInputCoeff, "1");
mgr->setOptionValString(&OPT_WinnerTakeAllType, "Fast");
mgr->setOptionValString(&OPT_SimulationTimeStep, "0.2");
mgr->setModelParamVal("FOAradius", 50, MC_RECURSE);
mgr->setModelParamVal("FoveaRadius", 50, MC_RECURSE);
mgr->setOptionValString(&OPT_IORtype, "Disc");
if (mgr->parseCommandLine(
(const int)argc, (const char**)argv, "<Network file> <server ip>", 2, 2) == false)
return 1;
// catch signals and redirect them to terminate for clean exit:
signal(SIGHUP, terminateProc); signal(SIGINT, terminateProc);
signal(SIGQUIT, terminateProc); signal(SIGTERM, terminateProc);
signal(SIGALRM, terminateProc);
mgr->start();
ComplexChannel *cc =
&*dynCastWeak<ComplexChannel>(brain->getVC());
//Get a new descriptor vector
DescriptorVec descVec(*mgr, "Descriptor Vector", "DecscriptorVec", cc);
//Get new classifier
Bayes bayesNet(descVec.getFVSize(), 0);
//get command line options
const char *bayesNetFile = mgr->getExtraArg(0).c_str();
const char *server_ip = mgr->getExtraArg(1).c_str();
bool train = false;
int foveaRadius = mgr->getModelParamVal<int>("FoveaRadius", MC_RECURSE);
printf("Setting fovea to %i, train = %i\n", foveaRadius, train);
//load the network if testing
//if (!train)
bayesNet.load(bayesNetFile);
descVec.setFoveaSize(foveaRadius);
xwin = new XWinManaged(Dims(256,256),
-1, -1, "ILab Robot Head Demo");
server = nv2_label_server_create(9930,
server_ip,
9931);
nv2_label_server_set_verbosity(server,1); //allow warnings
int send_interval = 1;
while(!terminate)
{
double prob = 0, statSig = 0;
Point2D clickLoc = xwin->getLastMouseClick();
if (clickLoc.isValid())
train = !train;
struct nv2_image_patch p;
const enum nv2_image_patch_result res =
nv2_label_server_get_current_patch(server, &p);
std::string objName = "nomatch";
if (res == NV2_IMAGE_PATCH_END)
{
fprintf(stdout, "ok, quitting\n");
break;
}
else if (res == NV2_IMAGE_PATCH_NONE)
{
usleep(10000);
continue;
}
else if (res == NV2_IMAGE_PATCH_VALID &&
p.type == NV2_PIXEL_TYPE_RGB24)
{
printf("Valid patch %s %ix%i\n", p.training_label,
p.width, p.height);
//showimg
Image<PixRGB<byte> > img(p.width, p.height, NO_INIT);
// unsigned char *imgPtr = const_cast<unsigned char*>
// (reinterpret_cast<const unsigned char*>(img.getArrayPtr()));
memcpy(img.getArrayPtr(), p.data, p.width*p.height*3);
Image<PixRGB<byte> > objImg = rescale(img, 256, 256);
int cls = classifyImage(objImg, descVec, bayesNet, &prob, &statSig);
if (cls != -1 && prob > -150)
objName = bayesNet.getClassName(cls);
else
objName = "nomatch";
printf("This is %s: Class %i prob %f\n",
objName.c_str(), cls, prob);
// if (strcmp(p.training_label, "none") != 0 &&
// false) { //training
if (cls == -1)
{
printf("Can you tell me what this is?\n");
std::getline(std::cin, objName);
learnImage(objImg, 0, descVec, bayesNet, objName.c_str());
bayesNet.save(bayesNetFile);
} else {
printf("Is this a %s?\n", objName.c_str());
if (train)
{
std::string tmp;
std::getline(std::cin, tmp);
if (tmp != "")
objName = tmp;
LINFO("Learning %s\n", objName.c_str());
fflush(stdout);
learnImage(objImg, 0, descVec, bayesNet, objName.c_str());
bayesNet.save(bayesNetFile);
}
}
}
if (objName != "nomatch")
{
printf("Object is %s\n", objName.c_str());
struct nv2_patch_label l;
l.protocol_version = NV2_LABEL_PROTOCOL_VERSION;
l.patch_id = p.id;
snprintf(l.source, sizeof(l.source), "%s",
"ObjRec");
snprintf(l.name, sizeof(l.name), "%s", // (%ux%u #%u)",
objName.c_str());
//(unsigned int) p.width,
//(unsigned int) p.height,
//(unsigned int) p.id);
snprintf(l.extra_info, sizeof(l.extra_info),
"%i", (int)statSig);
if (l.patch_id % send_interval == 0)
{
nv2_label_server_send_label(server, &l);
fprintf(stdout, "sent label '%s (%s)'\n",
l.name, l.extra_info);
}
else
{
fprintf(stdout, "DROPPED label '%s (%s)'\n",
l.name, l.extra_info);
}
}
nv2_image_patch_destroy(&p);
}
nv2_label_server_destroy(server);
}
int main(const int argc, const char **argv)
{
MYLOGVERB = LOG_INFO;
mgr = new ModelManager("Test LabelMeSaliency");
nub::soft_ref<SimEventQueueConfigurator>
seqc(new SimEventQueueConfigurator(*mgr));
mgr->addSubComponent(seqc);
//our brain
nub::ref<StdBrain> brain(new StdBrain(*mgr));
mgr->addSubComponent(brain);
mgr->exportOptions(MC_RECURSE);
mgr->setOptionValString(&OPT_RawVisualCortexChans, "IOC");
//mgr.setOptionValString(&OPT_RawVisualCortexChans, "I");
//mgr->setOptionValString(&OPT_RawVisualCortexChans, "GNO");
//mgr.setOptionValString(&OPT_RawVisualCortexChans, "N");
//manager.setOptionValString(&OPT_UseOlderVersion, "false");
// set the FOA and fovea radii
mgr->setOptionValString(&OPT_SaliencyMapType, "Fast");
mgr->setOptionValString(&OPT_SMfastInputCoeff, "1");
mgr->setOptionValString(&OPT_WinnerTakeAllType, "Fast");
mgr->setOptionValString(&OPT_SimulationTimeStep, "0.2");
mgr->setModelParamVal("FOAradius", 128, MC_RECURSE);
mgr->setModelParamVal("FoveaRadius", 128, MC_RECURSE);
mgr->setOptionValString(&OPT_IORtype, "Disc");
if (mgr->parseCommandLine(
(const int)argc, (const char**)argv, "<path to images>", 1, 1) == false);
nub::soft_ref<SimEventQueue> seq = seqc->getQ();
mgr->start();
//nub::ref<StdBrain> brain = dynCastWeak<StdBrain>(mgr->subComponent("Brain"));
//"/lab/ilab15/tmp/objectsDB/mit/labelMe/05june05_static_indoor",
ComplexChannel *cc =
&*dynCastWeak<ComplexChannel>(brain->getVC());
TestImages testImages(mgr->getExtraArg(0).c_str(), TestImages::MIT_LABELME);
Image<float> allObjImg = Raster::ReadFloat("allObjImg.pfm", RASFMT_PFM);
inplaceNormalize(allObjImg, 0.0F, 1.0F);
printf("## \"Filename\", \"Size\",\"fovea Radius\",\"Number of objects\",\"Salient Location\", \"Hits\",");
printf("\"Obj Saliency Max\",\"Obj Saliency Min\",\"Obj Saliency Sum\",\"Obj Saliency Area\"");
printf("\"Dist Saliency Max\",\"Dist Saliency Min\",\"Dist Saliency Sum\",\"Dist Saliency Area\"");
printf("\n");
for(uint scene=0; scene<testImages.getNumScenes(); scene++)
{
//get the image
LINFO("Get scene %i", scene);
Image<PixRGB<byte> > img = testImages.getScene(scene);
std::string sceneFile = testImages.getSceneFilename(scene);
LINFO("Size %ix%i", img.getWidth(), img.getHeight());
//set the fovea and foa radius to be 1/4 the size of the image width
int fRadius = 128; //(img.getWidth()/16);
//TODO: fixme
//brain->getSM()->setFoveaRadius(fRadius);
//brain->getSM()->setFOAradius(fRadius);
initRandomNumbers();
if (testImages.getNumObj() > 0) //if we have any labled objects
{
//bias the vc
Image<float> mask = rescale(allObjImg, img.getDims());
biasVC(*cc, mask);
//evolve the brain
Image<float> SMap;
rutz::shared_ptr<SimEventInputFrame> //place the image in the queue
e(new SimEventInputFrame(brain.get(), GenericFrame(img), 0));
seq->post(e);
//set the task relevance map
Point2D<int> winner;
float interestLevel=100.0F;
int nHits=0;
int nTimes=95;
printf("[ ");
Point2D<int> lastLoc(-1,-1);
while(interestLevel > 0.01F && nTimes < 100) //do until no more activation
{
nTimes++;
LINFO("InterestLevel %f", interestLevel);
Point2D<int> currentWinner = evolveBrain(img, SMap, &interestLevel, seq);
if (debug)
{
if (lastLoc.isValid())
{
drawLine(img, lastLoc, currentWinner,
PixRGB<byte>(0, 255, 0), 4);
} else {
drawCircle(img, currentWinner, fRadius-10, PixRGB<byte>(255,0,0), 3);
}
lastLoc = currentWinner;
drawCircle(img, currentWinner, fRadius, PixRGB<byte>(0,255,0), 3);
}
//check if the winner is inside an object (all objects)
int hit = -1;
for (uint obj=0; obj<testImages.getNumObj(); obj++)
{
int lineWidth = int(img.getWidth()*0.003);
std::vector<Point2D<int> > objPoly = testImages.getObjPolygon(obj);
if (debug)
{
Point2D<int> p1 = objPoly[0];
for(uint i=1; i<objPoly.size(); i++)
{
drawLine(img, p1, objPoly[i], PixRGB<byte>(255, 0, 0), lineWidth);
p1 = objPoly[i];
}
drawLine(img, p1, objPoly[0], PixRGB<byte>(255, 0, 0), lineWidth); //close the polygon
}
// if (testImages.pnpoly(objPoly, winner))
// hit = 1;
if (testImages.pnpoly(objPoly, currentWinner))
{
hit = obj;
}
}
printf("%i ", hit);
if (hit != -1)
{
winner = currentWinner;
nHits++;
}
}
if (debug)
{
Raster::WriteRGB(img, "IORSaliency.ppm");
Image<PixRGB<byte> > tmp = rescale(img, 512, 512);
SHOWIMG(tmp);
}
printf("] ");
printf("\"%s\",\"%ix%i\",\"%i\",\"%i\",\"(%i,%i)\",\"%i\"",
sceneFile.c_str(), img.getWidth(), img.getHeight(), fRadius,
testImages.getNumObj(), winner.i, winner.j, nHits);
printf("\n");
if (debug)
{
Image<PixRGB<byte> > tmp = rescale(img, 512, 512);
SHOWIMG(tmp);
}
//Compute the saliency ratio
/*Image<byte> imgMask;
//get the obj mask
for (uint obj=0; obj<testImages.getNumObj(); obj++)
{
LINFO("Adding obj %i", obj);
Image<byte> objMask = testImages.getObjMask(obj);
if (imgMask.initialized())
imgMask += objMask;
else
imgMask = objMask;
}
if (debug) SHOWIMG(rescale((Image<float>)imgMask, 512, 512));
LINFO("Mask %ix%i", imgMask.getWidth(), imgMask.getHeight());
Image<float> distMask = chamfer34(imgMask, (byte)255);
Image<float> objMask = binaryReverse(distMask, 255.0F);
//normalize mask from 0 to 1
inplaceNormalize(objMask, 0.0F, 1.0F);
inplaceNormalize(distMask, 0.0F, 1.0F);
if (debug) SHOWIMG(rescale((Image<float>)objMask, 512, 512));
if (debug) SHOWIMG(rescale((Image<float>)distMask, 512, 512));
//resize the saliency map to the orig img size
SMap = rescale(SMap, imgMask.getDims());
float objMin, objMax, objSum, objArea;
getMaskedMinMaxSumArea(SMap, objMask, objMin, objMax, objSum, objArea);
float distMin, distMax, distSum, distArea;
getMaskedMinMaxSumArea(SMap, distMask, distMin, distMax, distSum, distArea);
printf("\"%f\",\"%f\",\"%f\",\"%f\",\"%f\",\"%f\",\"%f\",\"%f\"",
objMax, objMin, objSum, objArea,
distMax, distMin, distSum, distArea);
printf("\n");
*/
} else {
printf("##%s has no objects \n", sceneFile.c_str());
}
}
}
