RegressionData ClassificationData::reformatAsRegressionData() const{
//Turns the classification into a regression data to enable regression algorithms like the MLP to be used as a classifier
//This sets the number of targets in the regression data equal to the number of classes in the classification data
//The output of each regression training sample will then be all 0's, except for the index matching the classLabel, which will be 1
//For this to work, the labelled classification data cannot have any samples with a classLabel of 0!
RegressionData regressionData;
if( totalNumSamples == 0 ){
return regressionData;
}
const UINT numInputDimensions = numDimensions;
const UINT numTargetDimensions = getNumClasses();
regressionData.setInputAndTargetDimensions(numInputDimensions, numTargetDimensions);
for(UINT i=0; i<totalNumSamples; i++){
VectorDouble targetVector(numTargetDimensions,0);
//Set the class index in the target vector to 1 and all other values in the target vector to 0
UINT classLabel = data[i].getClassLabel();
if( classLabel > 0 ){
targetVector[ classLabel-1 ] = 1;
}else{
regressionData.clear();
return regressionData;
}
regressionData.addSample(data[i].getSample(),targetVector);
}
return regressionData;
}
//Compute the regression data that will be stored at this node
bool RegressionTree::computeNodeRegressionData( const RegressionData &trainingData, VectorFloat ®ressionData ){
const UINT M = trainingData.getNumSamples();
const UINT N = trainingData.getNumInputDimensions();
const UINT T = trainingData.getNumTargetDimensions();
if( M == 0 ){
Regressifier::errorLog << "computeNodeRegressionData(...) - Failed to compute regression data, there are zero training samples!" << std::endl;
return false;
}
//Make sure the regression data is the correct size
regressionData.clear();
regressionData.resize( T, 0 );
//The regression data at this node is simply an average over all the training data at this node
for(unsigned int j=0; j<N; j++){
for(unsigned int i=0; i<M; i++){
regressionData[j] += trainingData[i].getTargetVector()[j];
}
regressionData[j] /= M;
}
return true;
}
void NeRegSuf::Update(const RegressionData &rdp){
++n_;
int p = rdp.xdim();
if(xtx_.nrow()==0 || xtx_.ncol()==0)
xtx_ = SpdMatrix(p,0.0);
if(xty_.size()==0) xty_ = Vector(p, 0.0);
const Vector & tmpx(rdp.x()); // add_intercept(rdp.x());
double y = rdp.y();
xty_.axpy(tmpx, y);
if(!xtx_is_fixed_) {
xtx_.add_outer(tmpx, 1.0, false);
needs_to_reflect_ = true;
}
sumsqy+= y*y;
sumy_ += y;
x_column_sums_.axpy(tmpx, 1.0);
}
bool RegressionTree::train_(RegressionData &trainingData){
//Clear any previous model
clear();
const unsigned int M = trainingData.getNumSamples();
const unsigned int N = trainingData.getNumInputDimensions();
const unsigned int T = trainingData.getNumTargetDimensions();
if( M == 0 ){
Regressifier::errorLog << "train_(RegressionData &trainingData) - Training data has zero samples!" << std::endl;
return false;
}
numInputDimensions = N;
numOutputDimensions = T;
inputVectorRanges = trainingData.getInputRanges();
targetVectorRanges = trainingData.getTargetRanges();
//Scale the training data if needed
if( useScaling ){
//Scale the training data between 0 and 1
trainingData.scale(0, 1);
}
//Setup the valid features - at this point all features can be used
Vector< UINT > features(N);
for(UINT i=0; i<N; i++){
features[i] = i;
}
//Build the tree
UINT nodeID = 0;
tree = buildTree( trainingData, NULL, features, nodeID );
if( tree == NULL ){
clear();
Regressifier::errorLog << "train_(RegressionData &trainingData) - Failed to build tree!" << std::endl;
return false;
}
//Flag that the algorithm has been trained
trained = true;
return true;
}
int main (int argc, const char * argv[])
{
//Turn on the training log so we can print the training status of the LinearRegression to the screen
TrainingLog::enableLogging( true );
//Load the training data
RegressionData trainingData;
RegressionData testData;
if( !trainingData.loadDatasetFromFile("LinearRegressionTrainingData.txt") ){
cout << "ERROR: Failed to load training data!\n";
return EXIT_FAILURE;
}
if( !testData.loadDatasetFromFile("LinearRegressionTestData.txt") ){
cout << "ERROR: Failed to load test data!\n";
return EXIT_FAILURE;
}
//Make sure the dimensionality of the training and test data matches
if( trainingData.getNumInputDimensions() != testData.getNumInputDimensions() ){
cout << "ERROR: The number of input dimensions in the training data (" << trainingData.getNumInputDimensions() << ")";
cout << " does not match the number of input dimensions in the test data (" << testData.getNumInputDimensions() << ")\n";
return EXIT_FAILURE;
}
if( testData.getNumTargetDimensions() != testData.getNumTargetDimensions() ){
cout << "ERROR: The number of target dimensions in the training data (" << testData.getNumTargetDimensions() << ")";
cout << " does not match the number of target dimensions in the test data (" << testData.getNumTargetDimensions() << ")\n";
return EXIT_FAILURE;
}
cout << "Training and Test datasets loaded\n";
//Print the stats of the datasets
cout << "Training data stats:\n";
trainingData.printStats();
cout << "Test data stats:\n";
testData.printStats();
//Create a new gesture recognition pipeline
GestureRecognitionPipeline pipeline;
//Add a LinearRegression instance to the pipeline
pipeline.setRegressifier( LinearRegression() );
//Train the LinearRegression model
cout << "Training LogisticRegression model...\n";
if( !pipeline.train( trainingData ) ){
cout << "ERROR: Failed to train LinearRegression model!\n";
return EXIT_FAILURE;
}
cout << "Model trained.\n";
//Test the model
cout << "Testing LinearRegression model...\n";
if( !pipeline.test( testData ) ){
cout << "ERROR: Failed to test LinearRegression model!\n";
return EXIT_FAILURE;
}
cout << "Test complete. Test RMS error: " << pipeline.getTestRMSError() << endl;
//Run back over the test data again and output the results to a file
fstream file;
file.open("LinearRegressionResultsData.txt", fstream::out);
for(UINT i=0; i<testData.getNumSamples(); i++){
vector< double > inputVector = testData[i].getInputVector();
vector< double > targetVector = testData[i].getTargetVector();
//Map the input vector using the trained regression model
if( !pipeline.predict( inputVector ) ){
cout << "ERROR: Failed to map test sample " << i << endl;
return EXIT_FAILURE;
}
//Get the mapped regression data
vector< double > outputVector = pipeline.getRegressionData();
//Write the mapped value and also the target value to the file
for(UINT j=0; j<outputVector.size(); j++){
file << outputVector[j] << "\t";
}
for(UINT j=0; j<targetVector.size(); j++){
file << targetVector[j] << "\t";
}
file << endl;
}
//Close the file
file.close();
return EXIT_SUCCESS;
}
bool train( CommandLineParser &parser ){
infoLog << "Training regression model..." << endl;
string trainDatasetFilename = "";
string modelFilename = "";
string defaultFilename = "linear-regression-model.grt";
bool removeFeatures = false;
bool defaultRemoveFeatures = false;
//Get the filename
if( !parser.get("filename",trainDatasetFilename) ){
errorLog << "Failed to parse filename from command line! You can set the filename using the -f." << endl;
printUsage();
return false;
}
//Get the model filename
parser.get("model-filename",modelFilename,defaultFilename);
//Load the training data to train the model
RegressionData trainingData;
infoLog << "- Loading Training Data..." << endl;
if( !trainingData.load( trainDatasetFilename ) ){
errorLog << "Failed to load training data!\n";
return false;
}
const unsigned int N = trainingData.getNumInputDimensions();
const unsigned int T = trainingData.getNumTargetDimensions();
infoLog << "- Num training samples: " << trainingData.getNumSamples() << endl;
infoLog << "- Num input dimensions: " << N << endl;
infoLog << "- Num target dimensions: " << T << endl;
//Create a new regression instance
LogisticRegression regression;
regression.setMaxNumEpochs( 500 );
regression.setMinChange( 1.0e-5 );
regression.setUseValidationSet( true );
regression.setValidationSetSize( 20 );
regression.setRandomiseTrainingOrder( true );
regression.enableScaling( true );
//Create a new pipeline that will hold the regression algorithm
GestureRecognitionPipeline pipeline;
//Add a multidimensional regression instance and set the regression algorithm to Linear Regression
pipeline.setRegressifier( MultidimensionalRegression( regression, true ) );
infoLog << "- Training model...\n";
//Train the classifier
if( !pipeline.train( trainingData ) ){
errorLog << "Failed to train model!" << endl;
return false;
}
infoLog << "- Model trained!" << endl;
infoLog << "- Saving model to: " << modelFilename << endl;
//Save the pipeline
if( pipeline.save( modelFilename ) ){
infoLog << "- Model saved." << endl;
}else warningLog << "Failed to save model to file: " << modelFilename << endl;
infoLog << "- TrainingTime: " << pipeline.getTrainingTime() << endl;
return true;
}
bool RegressionTree::computeBestSpiltBestIterativeSpilt( const RegressionData &trainingData, const Vector< UINT > &features, UINT &featureIndex, Float &threshold, Float &minError ){
const UINT M = trainingData.getNumSamples();
const UINT N = (UINT)features.size();
if( N == 0 ) return false;
minError = grt_numeric_limits< Float >::max();
UINT bestFeatureIndex = 0;
UINT groupID = 0;
Float bestThreshold = 0;
Float error = 0;
Float minRange = 0;
Float maxRange = 0;
Float step = 0;
Vector< UINT > groupIndex(M);
VectorFloat groupCounter(2,0);
VectorFloat groupMean(2,0);
VectorFloat groupMSE(2,0);
Vector< MinMax > ranges = trainingData.getInputRanges();
//Loop over each feature and try and find the best split point
for(UINT n=0; n<N; n++){
minRange = ranges[n].minValue;
maxRange = ranges[n].maxValue;
step = (maxRange-minRange)/Float(numSplittingSteps);
threshold = minRange;
featureIndex = features[n];
while( threshold <= maxRange ){
//Iterate over each sample and work out what group it falls into
for(UINT i=0; i<M; i++){
groupID = trainingData[i].getInputVector()[featureIndex] >= threshold ? 1 : 0;
groupIndex[i] = groupID;
groupMean[ groupID ] += trainingData[i].getInputVector()[featureIndex];
groupCounter[ groupID ]++;
}
groupMean[0] /= groupCounter[0] > 0 ? groupCounter[0] : 1;
groupMean[1] /= groupCounter[1] > 0 ? groupCounter[1] : 1;
//Compute the MSE for each group
for(UINT i=0; i<M; i++){
groupMSE[ groupIndex[i] ] += grt_sqr( groupMean[ groupIndex[i] ] - trainingData[ i ].getInputVector()[features[n]] );
}
groupMSE[0] /= groupCounter[0] > 0 ? groupCounter[0] : 1;
groupMSE[1] /= groupCounter[1] > 0 ? groupCounter[1] : 1;
error = sqrt( groupMSE[0] + groupMSE[1] );
//Store the best threshold and feature index
if( error < minError ){
minError = error;
bestThreshold = threshold;
bestFeatureIndex = featureIndex;
}
//Update the threshold
threshold += step;
}
}
//Set the best feature index and threshold
featureIndex = bestFeatureIndex;
threshold = bestThreshold;
return true;
}
bool LogisticRegression::train_(RegressionData &trainingData){
const unsigned int M = trainingData.getNumSamples();
const unsigned int N = trainingData.getNumInputDimensions();
const unsigned int K = trainingData.getNumTargetDimensions();
trained = false;
trainingResults.clear();
if( M == 0 ){
errorLog << "train_(RegressionData trainingData) - Training data has zero samples!" << endl;
return false;
}
if( K == 0 ){
errorLog << "train_(RegressionData trainingData) - The number of target dimensions is not 1!" << endl;
return false;
}
numInputDimensions = N;
numOutputDimensions = 1; //Logistic Regression will have 1 output
inputVectorRanges.clear();
targetVectorRanges.clear();
//Scale the training and validation data, if needed
if( useScaling ){
//Find the ranges for the input data
inputVectorRanges = trainingData.getInputRanges();
//Find the ranges for the target data
targetVectorRanges = trainingData.getTargetRanges();
//Scale the training data
trainingData.scale(inputVectorRanges,targetVectorRanges,0.0,1.0);
}
//Reset the weights
Random rand;
w0 = rand.getRandomNumberUniform(-0.1,0.1);
w.resize(N);
for(UINT j=0; j<N; j++){
w[j] = rand.getRandomNumberUniform(-0.1,0.1);
}
double error = 0;
double lastSquaredError = 0;
double delta = 0;
UINT iter = 0;
bool keepTraining = true;
Random random;
vector< UINT > randomTrainingOrder(M);
TrainingResult result;
trainingResults.reserve(M);
//In most cases, the training data is grouped into classes (100 samples for class 1, followed by 100 samples for class 2, etc.)
//This can cause a problem for stochastic gradient descent algorithm. To avoid this issue, we randomly shuffle the order of the
//training samples. This random order is then used at each epoch.
for(UINT i=0; i<M; i++){
randomTrainingOrder[i] = i;
}
std::random_shuffle(randomTrainingOrder.begin(), randomTrainingOrder.end());
//Run the main stochastic gradient descent training algorithm
while( keepTraining ){
//Run one epoch of training using stochastic gradient descent
totalSquaredTrainingError = 0;
for(UINT m=0; m<M; m++){
//Select the random sample
UINT i = randomTrainingOrder[m];
//Compute the error, given the current weights
VectorDouble x = trainingData[i].getInputVector();
VectorDouble y = trainingData[i].getTargetVector();
double h = w0;
for(UINT j=0; j<N; j++){
h += x[j] * w[j];
}
error = y[0] - sigmoid( h );
totalSquaredTrainingError += SQR(error);
//Update the weights
for(UINT j=0; j<N; j++){
w[j] += learningRate * error * x[j];
}
w0 += learningRate * error;
}
//Compute the error
delta = fabs( totalSquaredTrainingError-lastSquaredError );
lastSquaredError = totalSquaredTrainingError;
//Check to see if we should stop
if( delta <= minChange ){
keepTraining = false;
}
if( ++iter >= maxNumEpochs ){
keepTraining = false;
}
if( std::isinf( totalSquaredTrainingError ) || std::isnan( totalSquaredTrainingError ) ){
errorLog << "train_(RegressionData &trainingData) - Training failed! Total squared error is NAN. If scaling is not enabled then you should try to scale your data and see if this solves the issue." << endl;
return false;
}
//Store the training results
rootMeanSquaredTrainingError = sqrt( totalSquaredTrainingError / double(M) );
result.setRegressionResult(iter,totalSquaredTrainingError,rootMeanSquaredTrainingError,this);
trainingResults.push_back( result );
//Notify any observers of the new result
trainingResultsObserverManager.notifyObservers( result );
trainingLog << "Epoch: " << iter << " SSE: " << totalSquaredTrainingError << " Delta: " << delta << endl;
}
//Flag that the algorithm has been trained
regressionData.resize(1,0);
trained = true;
return trained;
}
bool MultidimensionalRegression::train_(RegressionData &trainingData){
const unsigned int M = trainingData.getNumSamples();
const unsigned int N = trainingData.getNumInputDimensions();
const unsigned int K = trainingData.getNumTargetDimensions();
trained = false;
trainingResults.clear();
deleteRegressionModules();
if( !getIsRegressionModuleSet() ){
errorLog << "train_(RegressionData &trainingData) - The regression module has not been set!" << std::endl;
return false;
}
if( M == 0 ){
errorLog << "train_(RegressionData &trainingData) - Training data has zero samples!" << std::endl;
return false;
}
numInputDimensions = N;
numOutputDimensions = K;
inputVectorRanges.clear();
targetVectorRanges.clear();
//Scale the training and validation data, if needed
if( useScaling ){
//Find the ranges for the input data
inputVectorRanges = trainingData.getInputRanges();
//Find the ranges for the target data
targetVectorRanges = trainingData.getTargetRanges();
//Scale the training data
trainingData.scale(inputVectorRanges,targetVectorRanges,0.0,1.0);
}
//Setup the regression modules
regressionModules.resize( K, NULL );
//Any scaling will happpen at the meta level, not the regression module letter, so ensure scaling is turned off for the modules
regressifier->enableScaling( false );
for(UINT k=0; k<K; k++){
regressionModules[k] = regressifier->deepCopy();
if( regressionModules[k] == NULL ){
errorLog << "train(LabelledRegressionData &trainingData) - Failed to deep copy module " << k << std::endl;
return false;
}
}
//Train each regression module
for(UINT k=0; k<K; k++){
trainingLog << "Training regression module: " << k << std::endl;
//We need to create a 1 dimensional training dataset for the k'th target dimension
RegressionData data;
data.setInputAndTargetDimensions(N, 1);
for(UINT i=0; i<M; i++){
if( !data.addSample(trainingData[i].getInputVector(), VectorFloat(1,trainingData[i].getTargetVector()[k]) ) ){
errorLog << "train_(RegressionData &trainingData) - Failed to add sample to dataset for regression module " << k << std::endl;
return false;
}
}
if( !regressionModules[k]->train( data ) ){
errorLog << "train_(RegressionData &trainingData) - Failed to train regression module " << k << std::endl;
return false;
}
}
//Flag that the algorithm has been trained
regressionData.resize(K,0);
trained = true;
return trained;
}
bool train( CommandLineParser &parser ){
infoLog << "Training regression model..." << endl;
string trainDatasetFilename = "";
string modelFilename = "";
float learningRate = 0;
float minChange = 0;
unsigned int maxEpoch = 0;
unsigned int batchSize = 0;
//Get the filename
if( !parser.get("filename",trainDatasetFilename) ){
errorLog << "Failed to parse filename from command line! You can set the filename using the -f." << endl;
printHelp();
return false;
}
//Get the parameters from the parser
parser.get("model-filename",modelFilename);
parser.get( "learning-rate", learningRate );
parser.get( "min-change", minChange );
parser.get( "max-epoch", maxEpoch );
parser.get( "batch-size", batchSize );
infoLog << "settings: learning-rate: " << learningRate << " min-change: " << minChange << " max-epoch: " << maxEpoch << " batch-size: " << batchSize << endl;
//Load the training data to train the model
RegressionData trainingData;
//Try and parse the input and target dimensions
unsigned int numInputDimensions = 0;
unsigned int numTargetDimensions = 0;
if( parser.get("num-inputs",numInputDimensions) && parser.get("num-targets",numTargetDimensions) ){
infoLog << "num input dimensions: " << numInputDimensions << " num target dimensions: " << numTargetDimensions << endl;
trainingData.setInputAndTargetDimensions( numInputDimensions, numTargetDimensions );
}
if( (numInputDimensions == 0 || numTargetDimensions == 0) && Util::stringEndsWith( trainDatasetFilename, ".csv" ) ){
errorLog << "Failed to parse num input dimensions and num target dimensions from input arguments. You must supply the input and target dimensions if the data format is CSV!" << endl;
printHelp();
return false;
}
infoLog << "- Loading Training Data..." << endl;
if( !trainingData.load( trainDatasetFilename ) ){
errorLog << "Failed to load training data!\n";
return false;
}
const unsigned int N = trainingData.getNumInputDimensions();
const unsigned int T = trainingData.getNumTargetDimensions();
infoLog << "- Num training samples: " << trainingData.getNumSamples() << endl;
infoLog << "- Num input dimensions: " << N << endl;
infoLog << "- Num target dimensions: " << T << endl;
//Create a new regression instance
LogisticRegression regression;
regression.setMaxNumEpochs( maxEpoch );
regression.setMinChange( minChange );
regression.setUseValidationSet( true );
regression.setValidationSetSize( 20 );
regression.setRandomiseTrainingOrder( true );
regression.enableScaling( true );
//Create a new pipeline that will hold the regression algorithm
GestureRecognitionPipeline pipeline;
//Add a multidimensional regression instance and set the regression algorithm to Linear Regression
pipeline.setRegressifier( MultidimensionalRegression( regression, true ) );
infoLog << "- Training model...\n";
//Train the classifier
if( !pipeline.train( trainingData ) ){
errorLog << "Failed to train model!" << endl;
return false;
}
infoLog << "- Model trained!" << endl;
infoLog << "- Saving model to: " << modelFilename << endl;
//Save the pipeline
if( pipeline.save( modelFilename ) ){
infoLog << "- Model saved." << endl;
}else warningLog << "Failed to save model to file: " << modelFilename << endl;
infoLog << "- TrainingTime: " << pipeline.getTrainingTime() << endl;
return true;
}
