// Main function int main(const int argc, const char **argv) { MYLOGVERB = LOG_INFO; // suppress debug messages // Instantiate a ModelManager: ModelManager manager("Gist Features Extraction"); // we cannot use saveResults() on our various ModelComponent objects // here, so let's not export the related command-line options. manager.allowOptions(OPTEXP_ALL & (~OPTEXP_SAVE)); // Instantiate our various ModelComponents: nub::soft_ref<SimEventQueueConfigurator> seqc(new SimEventQueueConfigurator(manager)); manager.addSubComponent(seqc); nub::soft_ref<InputMPEGStream> ims(new InputMPEGStream(manager, "Input MPEG Stream", "InputMPEGStream")); manager.addSubComponent(ims); nub::soft_ref<StdBrain> brain(new StdBrain(manager)); manager.addSubComponent(brain); nub::ref<SpatialMetrics> metrics(new SpatialMetrics(manager)); manager.addSubComponent(metrics); manager.exportOptions(MC_RECURSE); metrics->setFOAradius(30); // FIXME metrics->setFoveaRadius(30); // FIXME // setting up the GIST ESTIMATOR manager.setOptionValString(&OPT_GistEstimatorType,"Std"); //manager.setOptionValString(&OPT_GistEstimatorType,"FFT"); // Request a bunch of option aliases (shortcuts to lists of options): REQUEST_OPTIONALIAS_NEURO(manager); // Parse command-line: if (manager.parseCommandLine(argc, argv, "<*.mpg or *_gistList.txt>", 1, 1) == false) return(1); nub::soft_ref<SimEventQueue> seq = seqc->getQ(); // if the file passed ends with _gistList.txt // we have a different protocol bool isGistListInput = false; int ifLen = manager.getExtraArg(0).length(); if(ifLen > 13 && manager.getExtraArg(0).find("_gistList.txt",ifLen - 13) != std::string::npos) isGistListInput = true; // NOTE: this could now be controlled by a command-line option // --preload-mpeg=true manager.setOptionValString(&OPT_InputMPEGStreamPreload, "true"); // do post-command-line configs: std::vector<std::string> tag; std::vector<int> start; std::vector<int> num; unsigned int cLine = 0; int cIndex = 0; if(isGistListInput) { LINFO("we have a gistList input"); getGistFileList(manager.getExtraArg(0).c_str(), tag, start, num); cIndex = start[0]; } else { LINFO("we have an mpeg input"); ims->setFileName(manager.getExtraArg(0)); manager.setOptionValString(&OPT_InputFrameDims, convertToString(ims->peekDims())); } // frame delay in seconds //double fdelay = 33.3667/1000.0; // real time double fdelay = 3.3667/1000.0; // let's get all our ModelComponent instances started: manager.start(); // get the GistEstimator nub::soft_ref<GistEstimatorStd> ge;////// = /////// dynCastWeak<GistEstimatorStd>(brain->getGE()); LFATAL("fixme"); if (ge.isInvalid()) LFATAL("I am useless without a GistEstimator"); // MAIN LOOP SimTime prevstime = SimTime::ZERO(); int fNum = 0; Image< PixRGB<byte> > inputImg; Image< PixRGB<byte> > dispImg; Image<double> cgist; std::string folder = ""; std::string::size_type sPos = manager.getExtraArg(0).rfind("/",ifLen); if(sPos != std::string::npos) folder = manager.getExtraArg(0).substr(0,sPos+1); LINFO("let's start"); while(1) { // has the time come for a new frame? if (fNum == 0 || (seq->now() - 0.5 * (prevstime - seq->now())).secs() - fNum * fdelay > fdelay) { // load new frame std::string fName; if(isGistListInput) { if (cLine >= tag.size()) break; // end of input list // open the current file char tNumStr[100]; sprintf(tNumStr,"%06d",cIndex); fName = folder + tag[cLine] + std::string(tNumStr) + ".ppm"; inputImg = Raster::ReadRGB(fName); cIndex++; if(cIndex >= start[cLine] + num[cLine]) { cLine++; if (cLine < tag.size()) cIndex = start[cLine]; } // reformat the file name to a gist name int fNameLen = fName.length(); unsigned int uPos = fName.rfind("_",fNameLen); fName = fName.substr(0,uPos)+ ".ppm"; } else { fName = manager.getExtraArg(0); inputImg = ims->readRGB(); //Raster::ReadRGB(manager.getExtraArg(1)); if (inputImg.initialized() == false) break; // end of input stream // format new frame inputImg = crop(inputImg, Rectangle(Point2D<int>(0,25), Dims(inputImg.getHeight(), inputImg.getWidth()-25+1))); cIndex = fNum+1; } dispImg = inputImg; LINFO("\nnew frame :%d",fNum); // pass input to brain: rutz::shared_ptr<SimEventInputFrame> e(new SimEventInputFrame(brain.get(), GenericFrame(inputImg), 0)); seq->post(e); // post the image to the brain // get the gist feature vector cgist = ge->getGist(); //for(uint k = 0; k < cgist.getSize(); k++) LINFO("%d: %f",k, cgist.getVal(0,k)); // // setup display at the start of stream // if (fNum == 0) // { // int s = SQ_SIZE; // inputWin = new XWinManaged(Dims(w, h), 0, 0, manager.getExtraArg(0).c_str()); // wList.add(inputWin); // gistWin = new XWinManaged(Dims(NUM_GIST_COL * s, NUM_GIST_FEAT * s), 0,0, "Gist"); // wList.add(gistWin); // } // // display the input image and the gist histogram // drawGrid(dispImg, w/4,h/4,1,1,PixRGB<byte>(255,255,255)); // inputWin->drawImage(dispImg,0,0); // gistWin->drawImage(ge->getGistHistogram(SQ_SIZE),0,0); // SAVE GIST FEATURES TO A FILE saveData(cgist, fName, cIndex-1); //LINFO("\nFrame number just saved:%d",fNum);Raster::waitForKey(); // increase frame count fNum++; } // evolve brain: prevstime = seq->now(); // time before current step const SimStatus status = seq->evolve(); if (SIM_BREAK == status) // Brain decided it's time to quit break; } // stop all our ModelComponents manager.stop(); // all done! return 0; }
int main(const int argc, const char **argv) { MYLOGVERB = LOG_INFO; mgr = new ModelManager("Test ObjRec"); nub::soft_ref<SimEventQueueConfigurator> seqc(new SimEventQueueConfigurator(*mgr)); mgr->addSubComponent(seqc); //our brain nub::ref<StdBrain> brain(new StdBrain(*mgr)); mgr->addSubComponent(brain); mgr->exportOptions(MC_RECURSE); mgr->setOptionValString(&OPT_VisualCortexType, "IOC"); //mgr.setOptionValString(&OPT_VisualCortexType, "I"); //mgr->setOptionValString(&OPT_VisualCortexType, "GNO"); //mgr.setOptionValString(&OPT_VisualCortexType, "N"); //manager.setOptionValString(&OPT_UseOlderVersion, "false"); // set the FOA and fovea radii mgr->setOptionValString(&OPT_SaliencyMapType, "Fast"); mgr->setOptionValString(&OPT_SMfastInputCoeff, "1"); mgr->setOptionValString(&OPT_WinnerTakeAllType, "Fast"); mgr->setOptionValString(&OPT_SimulationTimeStep, "0.2"); mgr->setModelParamVal("FOAradius", 50, MC_RECURSE); mgr->setModelParamVal("FoveaRadius", 50, MC_RECURSE); mgr->setOptionValString(&OPT_IORtype, "Disc"); if (mgr->parseCommandLine( (const int)argc, (const char**)argv, "<Network file> <server ip>", 2, 2) == false) return 1; // catch signals and redirect them to terminate for clean exit: signal(SIGHUP, terminateProc); signal(SIGINT, terminateProc); signal(SIGQUIT, terminateProc); signal(SIGTERM, terminateProc); signal(SIGALRM, terminateProc); mgr->start(); ComplexChannel *cc = &*dynCastWeak<ComplexChannel>(brain->getVC()); //Get a new descriptor vector DescriptorVec descVec(*mgr, "Descriptor Vector", "DecscriptorVec", cc); //Get new classifier Bayes bayesNet(descVec.getFVSize(), 0); //get command line options const char *bayesNetFile = mgr->getExtraArg(0).c_str(); const char *server_ip = mgr->getExtraArg(1).c_str(); bool train = false; int foveaRadius = mgr->getModelParamVal<int>("FoveaRadius", MC_RECURSE); printf("Setting fovea to %i, train = %i\n", foveaRadius, train); //load the network if testing //if (!train) bayesNet.load(bayesNetFile); descVec.setFoveaSize(foveaRadius); xwin = new XWinManaged(Dims(256,256), -1, -1, "ILab Robot Head Demo"); server = nv2_label_server_create(9930, server_ip, 9931); nv2_label_server_set_verbosity(server,1); //allow warnings int send_interval = 1; while(!terminate) { double prob = 0, statSig = 0; Point2D clickLoc = xwin->getLastMouseClick(); if (clickLoc.isValid()) train = !train; struct nv2_image_patch p; const enum nv2_image_patch_result res = nv2_label_server_get_current_patch(server, &p); std::string objName = "nomatch"; if (res == NV2_IMAGE_PATCH_END) { fprintf(stdout, "ok, quitting\n"); break; } else if (res == NV2_IMAGE_PATCH_NONE) { usleep(10000); continue; } else if (res == NV2_IMAGE_PATCH_VALID && p.type == NV2_PIXEL_TYPE_RGB24) { printf("Valid patch %s %ix%i\n", p.training_label, p.width, p.height); //showimg Image<PixRGB<byte> > img(p.width, p.height, NO_INIT); // unsigned char *imgPtr = const_cast<unsigned char*> // (reinterpret_cast<const unsigned char*>(img.getArrayPtr())); memcpy(img.getArrayPtr(), p.data, p.width*p.height*3); Image<PixRGB<byte> > objImg = rescale(img, 256, 256); int cls = classifyImage(objImg, descVec, bayesNet, &prob, &statSig); if (cls != -1 && prob > -150) objName = bayesNet.getClassName(cls); else objName = "nomatch"; printf("This is %s: Class %i prob %f\n", objName.c_str(), cls, prob); // if (strcmp(p.training_label, "none") != 0 && // false) { //training if (cls == -1) { printf("Can you tell me what this is?\n"); std::getline(std::cin, objName); learnImage(objImg, 0, descVec, bayesNet, objName.c_str()); bayesNet.save(bayesNetFile); } else { printf("Is this a %s?\n", objName.c_str()); if (train) { std::string tmp; std::getline(std::cin, tmp); if (tmp != "") objName = tmp; LINFO("Learning %s\n", objName.c_str()); fflush(stdout); learnImage(objImg, 0, descVec, bayesNet, objName.c_str()); bayesNet.save(bayesNetFile); } } } if (objName != "nomatch") { printf("Object is %s\n", objName.c_str()); struct nv2_patch_label l; l.protocol_version = NV2_LABEL_PROTOCOL_VERSION; l.patch_id = p.id; snprintf(l.source, sizeof(l.source), "%s", "ObjRec"); snprintf(l.name, sizeof(l.name), "%s", // (%ux%u #%u)", objName.c_str()); //(unsigned int) p.width, //(unsigned int) p.height, //(unsigned int) p.id); snprintf(l.extra_info, sizeof(l.extra_info), "%i", (int)statSig); if (l.patch_id % send_interval == 0) { nv2_label_server_send_label(server, &l); fprintf(stdout, "sent label '%s (%s)'\n", l.name, l.extra_info); } else { fprintf(stdout, "DROPPED label '%s (%s)'\n", l.name, l.extra_info); } } nv2_image_patch_destroy(&p); } nv2_label_server_destroy(server); }
int main(const int argc, const char **argv) { MYLOGVERB = LOG_INFO; mgr = new ModelManager("Test LabelMeSaliency"); nub::soft_ref<SimEventQueueConfigurator> seqc(new SimEventQueueConfigurator(*mgr)); mgr->addSubComponent(seqc); //our brain nub::ref<StdBrain> brain(new StdBrain(*mgr)); mgr->addSubComponent(brain); mgr->exportOptions(MC_RECURSE); mgr->setOptionValString(&OPT_RawVisualCortexChans, "IOC"); //mgr.setOptionValString(&OPT_RawVisualCortexChans, "I"); //mgr->setOptionValString(&OPT_RawVisualCortexChans, "GNO"); //mgr.setOptionValString(&OPT_RawVisualCortexChans, "N"); //manager.setOptionValString(&OPT_UseOlderVersion, "false"); // set the FOA and fovea radii mgr->setOptionValString(&OPT_SaliencyMapType, "Fast"); mgr->setOptionValString(&OPT_SMfastInputCoeff, "1"); mgr->setOptionValString(&OPT_WinnerTakeAllType, "Fast"); mgr->setOptionValString(&OPT_SimulationTimeStep, "0.2"); mgr->setModelParamVal("FOAradius", 128, MC_RECURSE); mgr->setModelParamVal("FoveaRadius", 128, MC_RECURSE); mgr->setOptionValString(&OPT_IORtype, "Disc"); if (mgr->parseCommandLine( (const int)argc, (const char**)argv, "<path to images>", 1, 1) == false); nub::soft_ref<SimEventQueue> seq = seqc->getQ(); mgr->start(); //nub::ref<StdBrain> brain = dynCastWeak<StdBrain>(mgr->subComponent("Brain")); //"/lab/ilab15/tmp/objectsDB/mit/labelMe/05june05_static_indoor", ComplexChannel *cc = &*dynCastWeak<ComplexChannel>(brain->getVC()); TestImages testImages(mgr->getExtraArg(0).c_str(), TestImages::MIT_LABELME); Image<float> allObjImg = Raster::ReadFloat("allObjImg.pfm", RASFMT_PFM); inplaceNormalize(allObjImg, 0.0F, 1.0F); printf("## \"Filename\", \"Size\",\"fovea Radius\",\"Number of objects\",\"Salient Location\", \"Hits\","); printf("\"Obj Saliency Max\",\"Obj Saliency Min\",\"Obj Saliency Sum\",\"Obj Saliency Area\""); printf("\"Dist Saliency Max\",\"Dist Saliency Min\",\"Dist Saliency Sum\",\"Dist Saliency Area\""); printf("\n"); for(uint scene=0; scene<testImages.getNumScenes(); scene++) { //get the image LINFO("Get scene %i", scene); Image<PixRGB<byte> > img = testImages.getScene(scene); std::string sceneFile = testImages.getSceneFilename(scene); LINFO("Size %ix%i", img.getWidth(), img.getHeight()); //set the fovea and foa radius to be 1/4 the size of the image width int fRadius = 128; //(img.getWidth()/16); //TODO: fixme //brain->getSM()->setFoveaRadius(fRadius); //brain->getSM()->setFOAradius(fRadius); initRandomNumbers(); if (testImages.getNumObj() > 0) //if we have any labled objects { //bias the vc Image<float> mask = rescale(allObjImg, img.getDims()); biasVC(*cc, mask); //evolve the brain Image<float> SMap; rutz::shared_ptr<SimEventInputFrame> //place the image in the queue e(new SimEventInputFrame(brain.get(), GenericFrame(img), 0)); seq->post(e); //set the task relevance map Point2D<int> winner; float interestLevel=100.0F; int nHits=0; int nTimes=95; printf("[ "); Point2D<int> lastLoc(-1,-1); while(interestLevel > 0.01F && nTimes < 100) //do until no more activation { nTimes++; LINFO("InterestLevel %f", interestLevel); Point2D<int> currentWinner = evolveBrain(img, SMap, &interestLevel, seq); if (debug) { if (lastLoc.isValid()) { drawLine(img, lastLoc, currentWinner, PixRGB<byte>(0, 255, 0), 4); } else { drawCircle(img, currentWinner, fRadius-10, PixRGB<byte>(255,0,0), 3); } lastLoc = currentWinner; drawCircle(img, currentWinner, fRadius, PixRGB<byte>(0,255,0), 3); } //check if the winner is inside an object (all objects) int hit = -1; for (uint obj=0; obj<testImages.getNumObj(); obj++) { int lineWidth = int(img.getWidth()*0.003); std::vector<Point2D<int> > objPoly = testImages.getObjPolygon(obj); if (debug) { Point2D<int> p1 = objPoly[0]; for(uint i=1; i<objPoly.size(); i++) { drawLine(img, p1, objPoly[i], PixRGB<byte>(255, 0, 0), lineWidth); p1 = objPoly[i]; } drawLine(img, p1, objPoly[0], PixRGB<byte>(255, 0, 0), lineWidth); //close the polygon } // if (testImages.pnpoly(objPoly, winner)) // hit = 1; if (testImages.pnpoly(objPoly, currentWinner)) { hit = obj; } } printf("%i ", hit); if (hit != -1) { winner = currentWinner; nHits++; } } if (debug) { Raster::WriteRGB(img, "IORSaliency.ppm"); Image<PixRGB<byte> > tmp = rescale(img, 512, 512); SHOWIMG(tmp); } printf("] "); printf("\"%s\",\"%ix%i\",\"%i\",\"%i\",\"(%i,%i)\",\"%i\"", sceneFile.c_str(), img.getWidth(), img.getHeight(), fRadius, testImages.getNumObj(), winner.i, winner.j, nHits); printf("\n"); if (debug) { Image<PixRGB<byte> > tmp = rescale(img, 512, 512); SHOWIMG(tmp); } //Compute the saliency ratio /*Image<byte> imgMask; //get the obj mask for (uint obj=0; obj<testImages.getNumObj(); obj++) { LINFO("Adding obj %i", obj); Image<byte> objMask = testImages.getObjMask(obj); if (imgMask.initialized()) imgMask += objMask; else imgMask = objMask; } if (debug) SHOWIMG(rescale((Image<float>)imgMask, 512, 512)); LINFO("Mask %ix%i", imgMask.getWidth(), imgMask.getHeight()); Image<float> distMask = chamfer34(imgMask, (byte)255); Image<float> objMask = binaryReverse(distMask, 255.0F); //normalize mask from 0 to 1 inplaceNormalize(objMask, 0.0F, 1.0F); inplaceNormalize(distMask, 0.0F, 1.0F); if (debug) SHOWIMG(rescale((Image<float>)objMask, 512, 512)); if (debug) SHOWIMG(rescale((Image<float>)distMask, 512, 512)); //resize the saliency map to the orig img size SMap = rescale(SMap, imgMask.getDims()); float objMin, objMax, objSum, objArea; getMaskedMinMaxSumArea(SMap, objMask, objMin, objMax, objSum, objArea); float distMin, distMax, distSum, distArea; getMaskedMinMaxSumArea(SMap, distMask, distMin, distMax, distSum, distArea); printf("\"%f\",\"%f\",\"%f\",\"%f\",\"%f\",\"%f\",\"%f\",\"%f\"", objMax, objMin, objSum, objArea, distMax, distMin, distSum, distArea); printf("\n"); */ } else { printf("##%s has no objects \n", sceneFile.c_str()); } } }