LabelImage RandomForestImage::predict(const RGBDImage& image,
cuv::ndarray<float, cuv::host_memory_space>* probabilities, const bool onGPU, bool useDepthImages) const {
LabelImage prediction(image.getWidth(), image.getHeight());
const LabelType numClasses = getNumClasses();
if (treeData.size() != ensemble.size()) {
throw std::runtime_error((boost::format("tree data size: %d, ensemble size: %d. histograms normalized?")
% treeData.size() % ensemble.size()).str());
}
cuv::ndarray<float, cuv::host_memory_space> hostProbabilities(
cuv::extents[numClasses][image.getHeight()][image.getWidth()],
m_predictionAllocator);
if (onGPU) {
cuv::ndarray<float, cuv::dev_memory_space> deviceProbabilities(
cuv::extents[numClasses][image.getHeight()][image.getWidth()],
m_predictionAllocator);
cudaSafeCall(cudaMemset(deviceProbabilities.ptr(), 0, static_cast<size_t>(deviceProbabilities.size() * sizeof(float))));
{
utils::Profile profile("classifyImagesGPU");
for (const boost::shared_ptr<const TreeNodes>& data : treeData) {
classifyImage(treeData.size(), deviceProbabilities, image, numClasses, data, useDepthImages);
}
}
normalizeProbabilities(deviceProbabilities);
cuv::ndarray<LabelType, cuv::dev_memory_space> output(image.getHeight(), image.getWidth(),
m_predictionAllocator);
determineMaxProbabilities(deviceProbabilities, output);
hostProbabilities = deviceProbabilities;
cuv::ndarray<LabelType, cuv::host_memory_space> outputHost(image.getHeight(), image.getWidth(),
m_predictionAllocator);
outputHost = output;
{
utils::Profile profile("setLabels");
for (int y = 0; y < image.getHeight(); ++y) {
for (int x = 0; x < image.getWidth(); ++x) {
prediction.setLabel(x, y, static_cast<LabelType>(outputHost(y, x)));
}
}
}
} else {
utils::Profile profile("classifyImagesCPU");
tbb::parallel_for(tbb::blocked_range<size_t>(0, image.getHeight()),
[&](const tbb::blocked_range<size_t>& range) {
for(size_t y = range.begin(); y != range.end(); y++) {
for(int x=0; x < image.getWidth(); x++) {
for (LabelType label = 0; label < numClasses; label++) {
hostProbabilities(label, y, x) = 0.0f;
}
for (const auto& tree : ensemble) {
const auto& t = tree->getTree();
PixelInstance pixel(&image, 0, x, y);
const auto& hist = t->classifySoft(pixel);
assert(hist.size() == numClasses);
for(LabelType label = 0; label<hist.size(); label++) {
hostProbabilities(label, y, x) += hist[label];
}
}
double sum = 0.0f;
for (LabelType label = 0; label < numClasses; label++) {
sum += hostProbabilities(label, y, x);
}
float bestProb = -1.0f;
for (LabelType label = 0; label < numClasses; label++) {
hostProbabilities(label, y, x) /= sum;
float prob = hostProbabilities(label, y, x);
if (prob > bestProb) {
prediction.setLabel(x, y, label);
bestProb = prob;
}
}
}
}
});
}
if (probabilities) {
*probabilities = hostProbabilities;
}
return prediction;
}
int main(const int argc, const char **argv)
{
MYLOGVERB = LOG_INFO;
mgr = new ModelManager("Test ObjRec");
nub::soft_ref<SimEventQueueConfigurator>
seqc(new SimEventQueueConfigurator(*mgr));
mgr->addSubComponent(seqc);
//our brain
nub::ref<StdBrain> brain(new StdBrain(*mgr));
mgr->addSubComponent(brain);
mgr->exportOptions(MC_RECURSE);
mgr->setOptionValString(&OPT_VisualCortexType, "IOC");
//mgr.setOptionValString(&OPT_VisualCortexType, "I");
//mgr->setOptionValString(&OPT_VisualCortexType, "GNO");
//mgr.setOptionValString(&OPT_VisualCortexType, "N");
//manager.setOptionValString(&OPT_UseOlderVersion, "false");
// set the FOA and fovea radii
mgr->setOptionValString(&OPT_SaliencyMapType, "Fast");
mgr->setOptionValString(&OPT_SMfastInputCoeff, "1");
mgr->setOptionValString(&OPT_WinnerTakeAllType, "Fast");
mgr->setOptionValString(&OPT_SimulationTimeStep, "0.2");
mgr->setModelParamVal("FOAradius", 50, MC_RECURSE);
mgr->setModelParamVal("FoveaRadius", 50, MC_RECURSE);
mgr->setOptionValString(&OPT_IORtype, "Disc");
if (mgr->parseCommandLine(
(const int)argc, (const char**)argv, "<Network file> <server ip>", 2, 2) == false)
return 1;
// catch signals and redirect them to terminate for clean exit:
signal(SIGHUP, terminateProc); signal(SIGINT, terminateProc);
signal(SIGQUIT, terminateProc); signal(SIGTERM, terminateProc);
signal(SIGALRM, terminateProc);
mgr->start();
ComplexChannel *cc =
&*dynCastWeak<ComplexChannel>(brain->getVC());
//Get a new descriptor vector
DescriptorVec descVec(*mgr, "Descriptor Vector", "DecscriptorVec", cc);
//Get new classifier
Bayes bayesNet(descVec.getFVSize(), 0);
//get command line options
const char *bayesNetFile = mgr->getExtraArg(0).c_str();
const char *server_ip = mgr->getExtraArg(1).c_str();
bool train = false;
int foveaRadius = mgr->getModelParamVal<int>("FoveaRadius", MC_RECURSE);
printf("Setting fovea to %i, train = %i\n", foveaRadius, train);
//load the network if testing
//if (!train)
bayesNet.load(bayesNetFile);
descVec.setFoveaSize(foveaRadius);
xwin = new XWinManaged(Dims(256,256),
-1, -1, "ILab Robot Head Demo");
server = nv2_label_server_create(9930,
server_ip,
9931);
nv2_label_server_set_verbosity(server,1); //allow warnings
int send_interval = 1;
while(!terminate)
{
double prob = 0, statSig = 0;
Point2D clickLoc = xwin->getLastMouseClick();
if (clickLoc.isValid())
train = !train;
struct nv2_image_patch p;
const enum nv2_image_patch_result res =
nv2_label_server_get_current_patch(server, &p);
std::string objName = "nomatch";
if (res == NV2_IMAGE_PATCH_END)
{
fprintf(stdout, "ok, quitting\n");
break;
}
else if (res == NV2_IMAGE_PATCH_NONE)
{
usleep(10000);
continue;
}
else if (res == NV2_IMAGE_PATCH_VALID &&
p.type == NV2_PIXEL_TYPE_RGB24)
{
printf("Valid patch %s %ix%i\n", p.training_label,
p.width, p.height);
//showimg
Image<PixRGB<byte> > img(p.width, p.height, NO_INIT);
// unsigned char *imgPtr = const_cast<unsigned char*>
// (reinterpret_cast<const unsigned char*>(img.getArrayPtr()));
memcpy(img.getArrayPtr(), p.data, p.width*p.height*3);
Image<PixRGB<byte> > objImg = rescale(img, 256, 256);
int cls = classifyImage(objImg, descVec, bayesNet, &prob, &statSig);
if (cls != -1 && prob > -150)
objName = bayesNet.getClassName(cls);
else
objName = "nomatch";
printf("This is %s: Class %i prob %f\n",
objName.c_str(), cls, prob);
// if (strcmp(p.training_label, "none") != 0 &&
// false) { //training
if (cls == -1)
{
printf("Can you tell me what this is?\n");
std::getline(std::cin, objName);
learnImage(objImg, 0, descVec, bayesNet, objName.c_str());
bayesNet.save(bayesNetFile);
} else {
printf("Is this a %s?\n", objName.c_str());
if (train)
{
std::string tmp;
std::getline(std::cin, tmp);
if (tmp != "")
objName = tmp;
LINFO("Learning %s\n", objName.c_str());
fflush(stdout);
learnImage(objImg, 0, descVec, bayesNet, objName.c_str());
bayesNet.save(bayesNetFile);
}
}
}
if (objName != "nomatch")
{
printf("Object is %s\n", objName.c_str());
struct nv2_patch_label l;
l.protocol_version = NV2_LABEL_PROTOCOL_VERSION;
l.patch_id = p.id;
snprintf(l.source, sizeof(l.source), "%s",
"ObjRec");
snprintf(l.name, sizeof(l.name), "%s", // (%ux%u #%u)",
objName.c_str());
//(unsigned int) p.width,
//(unsigned int) p.height,
//(unsigned int) p.id);
snprintf(l.extra_info, sizeof(l.extra_info),
"%i", (int)statSig);
if (l.patch_id % send_interval == 0)
{
nv2_label_server_send_label(server, &l);
fprintf(stdout, "sent label '%s (%s)'\n",
l.name, l.extra_info);
}
else
{
fprintf(stdout, "DROPPED label '%s (%s)'\n",
l.name, l.extra_info);
}
}
nv2_image_patch_destroy(&p);
}
nv2_label_server_destroy(server);
}
LabelImage RandomForestImage::improveHistograms(const RGBDImage& image, const LabelImage& labelImage, const bool onGPU, bool useDepthImages) const {
LabelImage prediction(image.getWidth(), image.getHeight());
const LabelType numClasses = getNumClasses();
if (treeData.size() != ensemble.size()) {
throw std::runtime_error((boost::format("tree data size: %d, ensemble size: %d. histograms normalized?")
% treeData.size() % ensemble.size()).str());
}
cuv::ndarray<float, cuv::host_memory_space> hostProbabilities(
cuv::extents[numClasses][image.getHeight()][image.getWidth()],
m_predictionAllocator);
//These offsets should have been used instead of traversing to the leaf again
/* cuv::ndarray<unsigned int, cuv::dev_memory_space> nodeOffsets(
cuv::extents[image.getHeight()][image.getWidth()],
m_predictionAllocator);
*/
if (onGPU) {
cuv::ndarray<float, cuv::dev_memory_space> deviceProbabilities(
cuv::extents[numClasses][image.getHeight()][image.getWidth()],
m_predictionAllocator);
cudaSafeCall(cudaMemset(deviceProbabilities.ptr(), 0, static_cast<size_t>(deviceProbabilities.size() * sizeof(float))));
{
utils::Profile profile("classifyImagesGPU");
for (const boost::shared_ptr<const TreeNodes>& data : treeData) {
classifyImage(treeData.size(), deviceProbabilities, image, numClasses, data, useDepthImages);
bool found_tree = false;
//should be change to parallel for and add lock
for (size_t treeNr = 0; treeNr < ensemble.size(); treeNr++) {
if (data->getTreeId() == ensemble[treeNr]->getId()) {
found_tree =true;
const boost::shared_ptr<RandomTree<PixelInstance, ImageFeatureFunction> >& tree = ensemble[treeNr]->getTree();
//this should have been used and done before trying to classify the images, since it doesn't change
//std::vector<size_t> leafSet;
//tree->collectLeafNodes(leafSet);
for (int y = 0; y < image.getHeight(); y++)
for (int x = 0; x < image.getWidth(); x++) {
LabelType label = labelImage.getLabel(x,y);
if (!shouldIgnoreLabel(label)) {
PixelInstance pixel(&image, label, x, y);
//This should be changed. When classifying the image, the nodeoffsets should be returned and those used directly
//instead of traversing again to the leaves. As a test, can check if the nodeoffset is the same as the one returned
//by travertoleaf
tree->setAllPixelsHistogram(pixel);
}
}
}
if (found_tree)
break;
}
}
}
}
//should also add the CPU code!
return prediction;
}
