void InitializeVertex(VertexDescriptorType v, TGraph& g) const override
{
//std::cout << "Initializing " << v[0] << " " << v[1] << std::endl;
// Create the patch object and associate with the node
itk::Index<2> index;
index[0] = v[0];
index[1] = v[1];
unsigned int pointId = 0;
unsigned int numberOfMissingPoints = 0;
// Look for 'index' in the map
CoordinateMapType::const_iterator iter = this->CoordinateMap.find(index);
if(iter != this->CoordinateMap.end())
{
pointId = iter->second;
float descriptorValues[this->FeatureArray->GetNumberOfComponents()];
this->FeatureArray->GetTupleValue(pointId, descriptorValues);
std::vector<float> featureVector(descriptorValues, descriptorValues + sizeof(descriptorValues) / sizeof(float) );
DescriptorType descriptor(featureVector);
descriptor.SetVertex(v);
descriptor.SetStatus(PixelDescriptor::SOURCE_NODE);
put(this->DescriptorMap, v, descriptor);
}
else
{
//std::cout << index << " not found in the map!" << std::endl;
numberOfMissingPoints++;
std::vector<float> featureVector(this->FeatureArray->GetNumberOfComponents(), 0);
DescriptorType descriptor(featureVector);
descriptor.SetVertex(v);
descriptor.SetStatus(PixelDescriptor::INVALID);
put(this->DescriptorMap, v, descriptor);
}
//std::cout << "There were " << numberOfMissingPoints << " missing points when computing the descriptor for node " << index << std::endl;
}
int classifyMulticlassSvm(CvSVM& classifier, Mat image)
{
/* Extract features */
Mat featureVector(FEAT_SIZE, 1, CV_32FC1);
extractFeatures(image, featureVector);
/* Classify */
float classLabel = classifier.predict(featureVector, true);
/* Integer class label */
return (int)classLabel;
}
int classifyOneAgainstAllSvm(vector<CvSVM> classifiers, Mat image)
{
/* Extract features */
Mat featureVector(FEAT_SIZE, 1, CV_32FC1);
extractFeatures(image, featureVector);
/* Responses from the classifiers */
vector<float> responses(classifiers.size());
for (int i = 0; i < classifiers.size(); i++) {
responses[i] = classifiers[i].predict(featureVector, true);
}
/* Argmax of responses */
int argmax = 0;
for (int i = 1; i < responses.size(); i++) {
if (responses[i] > responses[argmax]) {
argmax = i;
}
}
/* Got a label or a rejection? */
return (responses[argmax] > 0) ? (argmax+1) : 0;
}
//--------------------------------------------------------------
void ofApp::keyPressed(int key){
infoText = "";
bool buildTexture = false;
switch ( key) {
case 'r':
record = true;
break;
case '1':
trainingClassLabel = 1;
break;
case '2':
trainingClassLabel = 2;
break;
case '3':
trainingClassLabel = 3;
break;
case 't':
if( pipeline.train( trainingData ) ){
infoText = "Pipeline Trained";
buildTexture = true;
}else infoText = "WARNING: Failed to train pipeline";
break;
case 's':
if( trainingData.save("TrainingData.grt") ){
infoText = "Training data saved to file";
}else infoText = "WARNING: Failed to save training data to file";
break;
case 'l':
if( trainingData.load("TrainingData.grt") ){
infoText = "Training data saved to file";
}else infoText = "WARNING: Failed to load training data from file";
break;
case 'c':
trainingData.clear();
infoText = "Training data cleared";
break;
default:
break;
}
if( buildTexture ){
const unsigned int rows = TEXTURE_RESOLUTION;
const unsigned int cols = TEXTURE_RESOLUTION;
const unsigned int size = rows*cols*4;
vector<float> pixelData( size );
ofFloatPixels pixels;
unsigned int index = 0;
unsigned int classLabel = 0;
VectorFloat featureVector(2);
VectorFloat likelihoods;
float r,g,b,a;
float maximumLikelihood;
for(unsigned int j=0; j<cols; j++){
for(unsigned int i=0; i<rows; i++){
featureVector[0] = i/double(rows);
featureVector[1] = j/double(cols);
if( pipeline.predict( featureVector ) ){
classLabel = pipeline.getPredictedClassLabel();
maximumLikelihood = pipeline.getMaximumLikelihood();
likelihoods = pipeline.getClassLikelihoods();
switch( classLabel ){
case 1:
r = 1.0;
g = 0.0;
b = 0.0;
a = maximumLikelihood;
break;
case 2:
r = 0.0;
g = 1.0;
b = 0.0;
a = maximumLikelihood;
break;
case 3:
r = 0.0;
g = 0.0;
b = 1.0;
a = maximumLikelihood;
break;
break;
default:
r = 0;
g = 0;
b = 0;
a = 1;
break;
}
pixelData[ index++ ] = r;
pixelData[ index++ ] = g;
pixelData[ index++ ] = b;
pixelData[ index++ ] = a;
}
}
}
pixels.setFromExternalPixels(&pixelData[0],rows,cols,OF_PIXELS_RGBA);
if(!texture.isAllocated()){
texture.allocate( pixels, false );
texture.setRGToRGBASwizzles(true);
}
texture.loadData( pixels );
texture.setTextureMinMagFilter( GL_LINEAR, GL_LINEAR );
}
}
int HolisticRecognizer::loadModelData(void)
{
LOG( LTKLogger::LTK_LOGLEVEL_DEBUG) << "Entered HolisticRecognizer::loadTrainingData" << endl;
int numShapes; // number of shapes in the problem set
int shapeId; // temp variable for storing the shape ID of incoming tracegroup
int numFeatures; // total number of features used
int numStrokes; // number of strokes in the character
HolisticShapeModel shapeModel; // a shape model variable
string algoVersion; //Algorithm version.
char versionInfo[VERSION_STR_LEN];
string version;
int iMajor, iMinor, iBugfix;
stringStringMap headerSequence;
float2DVector::iterator strokeIter; // Iterator to loop over number of strokes
float2DVector::iterator strokeIterEnd;
floatVector::iterator loopIter; // Iterator to loop over all the features
floatVector::iterator loopIterEnd;
// load the training file for the mapping
if(m_confMapFile != "")
{
m_confMapper.readConfigMapping(m_confMapFile);
}
else
{
LOG( LTKLogger::LTK_LOGLEVEL_DEBUG) << "Warning: Mapping file is not set in the config" << endl;
}
if((errorCode = cheSumGen.readMDTHeader(m_referenceModelFile,headerSequence)) != 0)
{
return errorCode;
//throw LTKException("ModelData File has been corrupted" + m_referenceModelFile);
}
algoVersion = headerSequence[RECVERSION].c_str();
getCurrentVersion(&iMajor, &iMinor, &iBugfix);
sprintf(versionInfo, "%c", algoVersion[0]);
int ver = atoi(versionInfo);
if(ver != iMajor)
{
LOG( LTKLogger::LTK_LOGLEVEL_ERR) << "Incompatible algorithm version : " + version << endl;
LTKReturnError(EINCOMPATIBLE_VERSION);
}
// open the reference model file for loading the training data
ifstream infile(m_referenceModelFile.c_str(),ios::in);
if(!infile)
{
LOG( LTKLogger::LTK_LOGLEVEL_ERR) << "Unable to open model data file : " + m_referenceModelFile << endl;
LTKReturnError(EMODEL_DATA_FILE_OPEN);
}
infile.seekg(atoi(headerSequence[HEADERLEN].c_str()),ios::beg);
LOG( LTKLogger::LTK_LOGLEVEL_DEBUG) << "Reference model file read" << endl;
// reading reference model file header
infile >> numShapes;
LOG( LTKLogger::LTK_LOGLEVEL_DEBUG) << "number of shapes read from reference model file = " << numShapes << endl;
if(m_numShapes != numShapes)
{
LOG( LTKLogger::LTK_LOGLEVEL_ERR) << "Invalid value for number of shapes" << endl;
LOG( LTKLogger::LTK_LOGLEVEL_ERR) << "NumShapes from cofiguration file ( or default value ) : " << m_numShapes << endl;
LOG( LTKLogger::LTK_LOGLEVEL_ERR) << "NumShapes from model data file : " << numShapes << endl;
LTKReturnError(EINVALID_NUM_OF_SHAPES);
}
infile >> numFeatures;
LOG( LTKLogger::LTK_LOGLEVEL_DEBUG) << "number of features read from reference model file = " << numFeatures << endl;
floatVector tempFloatVector(numFeatures,0); // temporary float vector
// reading the prototype models
while(infile)
{
infile >> shapeId;
infile >> numStrokes;
float2DVector featureVector(numStrokes,tempFloatVector);
// reading from the dat file and populating the featureVector
strokeIterEnd = featureVector.end();
for(strokeIter = featureVector.begin(); strokeIter != strokeIterEnd; ++strokeIter)
{
loopIterEnd = (*strokeIter).end();
for(loopIter = (*strokeIter).begin(); loopIter != loopIterEnd; ++loopIter)
{
infile >> (*loopIter);
}
}
shapeModel.setShapeId(shapeId);
shapeModel.setModel(featureVector);
m_referenceModels.push_back(shapeModel); // populating the reference models
}
cout << "Total number of reference models read - " << m_referenceModels.size() << endl;
infile.close();
LOG( LTKLogger::LTK_LOGLEVEL_DEBUG) << "Exiting HolisticRecognizer::loadTrainingData" << endl;
return SUCCESS;
}
void CaffeClassifier::predictHeatMap(cv::Mat& inputImage, int label, string predictionLayer, string dataLayer, cv::Mat& heatMap) {
const int IMAGE_SIZE = 227;
const int BATCH_SIZE = 64;
heatMap = cv::Mat(IMAGE_SIZE, IMAGE_SIZE, CV_32FC1, Scalar(26.932154));
const int STEP_SIZE = 9;
const int START_OFFSET = STEP_SIZE / 2;
const int FILLER_SIZE = 50;
cv::Scalar mean = cv::mean(inputImage);
std::vector<Point> middlePoints;
for (int i = START_OFFSET; i < IMAGE_SIZE; i += STEP_SIZE) {
for (int j = START_OFFSET; j < IMAGE_SIZE; j += STEP_SIZE) {
middlePoints.push_back(Point(i, j));
}
}
for (int i = 0; i < middlePoints.size(); i += BATCH_SIZE) {
std::cout << (i * 100) / middlePoints.size() << "% " << std::flush;
vector<Datum> vecDatum;
for (int j = 0; j < BATCH_SIZE; ++j) {
// do not go over the last middle point
int index = min(static_cast<int>(middlePoints.size() - 1), i + j);
Point p = middlePoints[index];
cv::Mat image = inputImage.clone();
// cv::Rect rect(Point(max(0, p.x - FILLER_SIZE), max(0, p.y - FILLER_SIZE)), Point(min(IMAGE_SIZE - 1, p.x + FILLER_SIZE), min(IMAGE_SIZE - 1, p.y + FILLER_SIZE)));
// cv::Mat subMat = image(rect);
// cv::Scalar mean = cv::mean(subMat);
circle(image,
p,
FILLER_SIZE,
mean,
CV_FILLED);
// rectangle(image,
// Point(max(0, p.x - FILLER_SIZE), max(0, p.y - FILLER_SIZE)),
// Point(min(IMAGE_SIZE - 1, p.x + FILLER_SIZE), min(IMAGE_SIZE - 1, p.y + FILLER_SIZE)),
// Scalar(0, 0, 0),
// CV_FILLED);
std::ostringstream o;
o << "/home/knub/Repositories/video-classification/nets/activity_recognition/caffenet/";
o << index;
o << "_heat.png";
cv::imwrite(o.str(), image);
// check channels
if (channels != image.channels()) {
cerr << "Error: the channel number of input image is invalid for CNN classifier!" << endl;
exit(1);
}
// mat to datum
Datum datum;
CVMatToDatum(image, &datum);
vecDatum.push_back(datum);
image.release();
}
// get the data layer
const caffe::shared_ptr<MemoryDataLayer<float>> memDataLayer = boost::static_pointer_cast<MemoryDataLayer<float>>(caffeNet->layer_by_name(dataLayer));
// push new image data
memDataLayer->AddDatumVector(vecDatum);
// do forward pass
vector<Blob<float>*> inputVec;
caffeNet->Forward(inputVec);
// get results
const caffe::shared_ptr<Blob<float> > featureBlob = caffeNet->blob_by_name(predictionLayer);
int dimFeatures = featureBlob->count() / BATCH_SIZE; // 101
assert(dimFeatures == 101);
// get output from each input image
for (int j = 0; j < BATCH_SIZE; ++j) {
int index = min(static_cast<int>(middlePoints.size() - 1), i + j);
Point p = middlePoints[index];
// std::cout << "Channels: " << featureBlob->channels() << ", Count: " << featureBlob->count() << ", Width: " << featureBlob->width() << ", Height: " << featureBlob->height() << std::endl;
// featureBlob = 64 x 101 matrix
float* fs = featureBlob->mutable_cpu_data() + featureBlob->offset(j);
vector<float> featureVector(fs, fs + dimFeatures);
// std::vector<float>::iterator result = std::max_element(featureVector.begin(), featureVector.end());
// int predicted = result - featureVector.begin();
// std::cout << "Predicted: " << predicted << ", Actual: " << label << std::endl;
// assert(predicted == label);
float confidence = featureVector[label];
rectangle(heatMap,
Point(p.x - START_OFFSET, p.y - START_OFFSET),
Point(p.x + START_OFFSET, p.y + START_OFFSET),
Scalar(confidence),
CV_FILLED);
}
}
}