RegressionData ClassificationData::reformatAsRegressionData() const{ //Turns the classification into a regression data to enable regression algorithms like the MLP to be used as a classifier //This sets the number of targets in the regression data equal to the number of classes in the classification data //The output of each regression training sample will then be all 0's, except for the index matching the classLabel, which will be 1 //For this to work, the labelled classification data cannot have any samples with a classLabel of 0! RegressionData regressionData; if( totalNumSamples == 0 ){ return regressionData; } const UINT numInputDimensions = numDimensions; const UINT numTargetDimensions = getNumClasses(); regressionData.setInputAndTargetDimensions(numInputDimensions, numTargetDimensions); for(UINT i=0; i<totalNumSamples; i++){ VectorDouble targetVector(numTargetDimensions,0); //Set the class index in the target vector to 1 and all other values in the target vector to 0 UINT classLabel = data[i].getClassLabel(); if( classLabel > 0 ){ targetVector[ classLabel-1 ] = 1; }else{ regressionData.clear(); return regressionData; } regressionData.addSample(data[i].getSample(),targetVector); } return regressionData; }
//Compute the regression data that will be stored at this node bool RegressionTree::computeNodeRegressionData( const RegressionData &trainingData, VectorFloat ®ressionData ){ const UINT M = trainingData.getNumSamples(); const UINT N = trainingData.getNumInputDimensions(); const UINT T = trainingData.getNumTargetDimensions(); if( M == 0 ){ Regressifier::errorLog << "computeNodeRegressionData(...) - Failed to compute regression data, there are zero training samples!" << std::endl; return false; } //Make sure the regression data is the correct size regressionData.clear(); regressionData.resize( T, 0 ); //The regression data at this node is simply an average over all the training data at this node for(unsigned int j=0; j<N; j++){ for(unsigned int i=0; i<M; i++){ regressionData[j] += trainingData[i].getTargetVector()[j]; } regressionData[j] /= M; } return true; }
void NeRegSuf::Update(const RegressionData &rdp){ ++n_; int p = rdp.xdim(); if(xtx_.nrow()==0 || xtx_.ncol()==0) xtx_ = SpdMatrix(p,0.0); if(xty_.size()==0) xty_ = Vector(p, 0.0); const Vector & tmpx(rdp.x()); // add_intercept(rdp.x()); double y = rdp.y(); xty_.axpy(tmpx, y); if(!xtx_is_fixed_) { xtx_.add_outer(tmpx, 1.0, false); needs_to_reflect_ = true; } sumsqy+= y*y; sumy_ += y; x_column_sums_.axpy(tmpx, 1.0); }
bool RegressionTree::train_(RegressionData &trainingData){ //Clear any previous model clear(); const unsigned int M = trainingData.getNumSamples(); const unsigned int N = trainingData.getNumInputDimensions(); const unsigned int T = trainingData.getNumTargetDimensions(); if( M == 0 ){ Regressifier::errorLog << "train_(RegressionData &trainingData) - Training data has zero samples!" << std::endl; return false; } numInputDimensions = N; numOutputDimensions = T; inputVectorRanges = trainingData.getInputRanges(); targetVectorRanges = trainingData.getTargetRanges(); //Scale the training data if needed if( useScaling ){ //Scale the training data between 0 and 1 trainingData.scale(0, 1); } //Setup the valid features - at this point all features can be used Vector< UINT > features(N); for(UINT i=0; i<N; i++){ features[i] = i; } //Build the tree UINT nodeID = 0; tree = buildTree( trainingData, NULL, features, nodeID ); if( tree == NULL ){ clear(); Regressifier::errorLog << "train_(RegressionData &trainingData) - Failed to build tree!" << std::endl; return false; } //Flag that the algorithm has been trained trained = true; return true; }
int main (int argc, const char * argv[]) { //Turn on the training log so we can print the training status of the LinearRegression to the screen TrainingLog::enableLogging( true ); //Load the training data RegressionData trainingData; RegressionData testData; if( !trainingData.loadDatasetFromFile("LinearRegressionTrainingData.txt") ){ cout << "ERROR: Failed to load training data!\n"; return EXIT_FAILURE; } if( !testData.loadDatasetFromFile("LinearRegressionTestData.txt") ){ cout << "ERROR: Failed to load test data!\n"; return EXIT_FAILURE; } //Make sure the dimensionality of the training and test data matches if( trainingData.getNumInputDimensions() != testData.getNumInputDimensions() ){ cout << "ERROR: The number of input dimensions in the training data (" << trainingData.getNumInputDimensions() << ")"; cout << " does not match the number of input dimensions in the test data (" << testData.getNumInputDimensions() << ")\n"; return EXIT_FAILURE; } if( testData.getNumTargetDimensions() != testData.getNumTargetDimensions() ){ cout << "ERROR: The number of target dimensions in the training data (" << testData.getNumTargetDimensions() << ")"; cout << " does not match the number of target dimensions in the test data (" << testData.getNumTargetDimensions() << ")\n"; return EXIT_FAILURE; } cout << "Training and Test datasets loaded\n"; //Print the stats of the datasets cout << "Training data stats:\n"; trainingData.printStats(); cout << "Test data stats:\n"; testData.printStats(); //Create a new gesture recognition pipeline GestureRecognitionPipeline pipeline; //Add a LinearRegression instance to the pipeline pipeline.setRegressifier( LinearRegression() ); //Train the LinearRegression model cout << "Training LogisticRegression model...\n"; if( !pipeline.train( trainingData ) ){ cout << "ERROR: Failed to train LinearRegression model!\n"; return EXIT_FAILURE; } cout << "Model trained.\n"; //Test the model cout << "Testing LinearRegression model...\n"; if( !pipeline.test( testData ) ){ cout << "ERROR: Failed to test LinearRegression model!\n"; return EXIT_FAILURE; } cout << "Test complete. Test RMS error: " << pipeline.getTestRMSError() << endl; //Run back over the test data again and output the results to a file fstream file; file.open("LinearRegressionResultsData.txt", fstream::out); for(UINT i=0; i<testData.getNumSamples(); i++){ vector< double > inputVector = testData[i].getInputVector(); vector< double > targetVector = testData[i].getTargetVector(); //Map the input vector using the trained regression model if( !pipeline.predict( inputVector ) ){ cout << "ERROR: Failed to map test sample " << i << endl; return EXIT_FAILURE; } //Get the mapped regression data vector< double > outputVector = pipeline.getRegressionData(); //Write the mapped value and also the target value to the file for(UINT j=0; j<outputVector.size(); j++){ file << outputVector[j] << "\t"; } for(UINT j=0; j<targetVector.size(); j++){ file << targetVector[j] << "\t"; } file << endl; } //Close the file file.close(); return EXIT_SUCCESS; }
bool train( CommandLineParser &parser ){ infoLog << "Training regression model..." << endl; string trainDatasetFilename = ""; string modelFilename = ""; string defaultFilename = "linear-regression-model.grt"; bool removeFeatures = false; bool defaultRemoveFeatures = false; //Get the filename if( !parser.get("filename",trainDatasetFilename) ){ errorLog << "Failed to parse filename from command line! You can set the filename using the -f." << endl; printUsage(); return false; } //Get the model filename parser.get("model-filename",modelFilename,defaultFilename); //Load the training data to train the model RegressionData trainingData; infoLog << "- Loading Training Data..." << endl; if( !trainingData.load( trainDatasetFilename ) ){ errorLog << "Failed to load training data!\n"; return false; } const unsigned int N = trainingData.getNumInputDimensions(); const unsigned int T = trainingData.getNumTargetDimensions(); infoLog << "- Num training samples: " << trainingData.getNumSamples() << endl; infoLog << "- Num input dimensions: " << N << endl; infoLog << "- Num target dimensions: " << T << endl; //Create a new regression instance LogisticRegression regression; regression.setMaxNumEpochs( 500 ); regression.setMinChange( 1.0e-5 ); regression.setUseValidationSet( true ); regression.setValidationSetSize( 20 ); regression.setRandomiseTrainingOrder( true ); regression.enableScaling( true ); //Create a new pipeline that will hold the regression algorithm GestureRecognitionPipeline pipeline; //Add a multidimensional regression instance and set the regression algorithm to Linear Regression pipeline.setRegressifier( MultidimensionalRegression( regression, true ) ); infoLog << "- Training model...\n"; //Train the classifier if( !pipeline.train( trainingData ) ){ errorLog << "Failed to train model!" << endl; return false; } infoLog << "- Model trained!" << endl; infoLog << "- Saving model to: " << modelFilename << endl; //Save the pipeline if( pipeline.save( modelFilename ) ){ infoLog << "- Model saved." << endl; }else warningLog << "Failed to save model to file: " << modelFilename << endl; infoLog << "- TrainingTime: " << pipeline.getTrainingTime() << endl; return true; }
bool RegressionTree::computeBestSpiltBestIterativeSpilt( const RegressionData &trainingData, const Vector< UINT > &features, UINT &featureIndex, Float &threshold, Float &minError ){ const UINT M = trainingData.getNumSamples(); const UINT N = (UINT)features.size(); if( N == 0 ) return false; minError = grt_numeric_limits< Float >::max(); UINT bestFeatureIndex = 0; UINT groupID = 0; Float bestThreshold = 0; Float error = 0; Float minRange = 0; Float maxRange = 0; Float step = 0; Vector< UINT > groupIndex(M); VectorFloat groupCounter(2,0); VectorFloat groupMean(2,0); VectorFloat groupMSE(2,0); Vector< MinMax > ranges = trainingData.getInputRanges(); //Loop over each feature and try and find the best split point for(UINT n=0; n<N; n++){ minRange = ranges[n].minValue; maxRange = ranges[n].maxValue; step = (maxRange-minRange)/Float(numSplittingSteps); threshold = minRange; featureIndex = features[n]; while( threshold <= maxRange ){ //Iterate over each sample and work out what group it falls into for(UINT i=0; i<M; i++){ groupID = trainingData[i].getInputVector()[featureIndex] >= threshold ? 1 : 0; groupIndex[i] = groupID; groupMean[ groupID ] += trainingData[i].getInputVector()[featureIndex]; groupCounter[ groupID ]++; } groupMean[0] /= groupCounter[0] > 0 ? groupCounter[0] : 1; groupMean[1] /= groupCounter[1] > 0 ? groupCounter[1] : 1; //Compute the MSE for each group for(UINT i=0; i<M; i++){ groupMSE[ groupIndex[i] ] += grt_sqr( groupMean[ groupIndex[i] ] - trainingData[ i ].getInputVector()[features[n]] ); } groupMSE[0] /= groupCounter[0] > 0 ? groupCounter[0] : 1; groupMSE[1] /= groupCounter[1] > 0 ? groupCounter[1] : 1; error = sqrt( groupMSE[0] + groupMSE[1] ); //Store the best threshold and feature index if( error < minError ){ minError = error; bestThreshold = threshold; bestFeatureIndex = featureIndex; } //Update the threshold threshold += step; } } //Set the best feature index and threshold featureIndex = bestFeatureIndex; threshold = bestThreshold; return true; }
bool LogisticRegression::train_(RegressionData &trainingData){ const unsigned int M = trainingData.getNumSamples(); const unsigned int N = trainingData.getNumInputDimensions(); const unsigned int K = trainingData.getNumTargetDimensions(); trained = false; trainingResults.clear(); if( M == 0 ){ errorLog << "train_(RegressionData trainingData) - Training data has zero samples!" << endl; return false; } if( K == 0 ){ errorLog << "train_(RegressionData trainingData) - The number of target dimensions is not 1!" << endl; return false; } numInputDimensions = N; numOutputDimensions = 1; //Logistic Regression will have 1 output inputVectorRanges.clear(); targetVectorRanges.clear(); //Scale the training and validation data, if needed if( useScaling ){ //Find the ranges for the input data inputVectorRanges = trainingData.getInputRanges(); //Find the ranges for the target data targetVectorRanges = trainingData.getTargetRanges(); //Scale the training data trainingData.scale(inputVectorRanges,targetVectorRanges,0.0,1.0); } //Reset the weights Random rand; w0 = rand.getRandomNumberUniform(-0.1,0.1); w.resize(N); for(UINT j=0; j<N; j++){ w[j] = rand.getRandomNumberUniform(-0.1,0.1); } double error = 0; double lastSquaredError = 0; double delta = 0; UINT iter = 0; bool keepTraining = true; Random random; vector< UINT > randomTrainingOrder(M); TrainingResult result; trainingResults.reserve(M); //In most cases, the training data is grouped into classes (100 samples for class 1, followed by 100 samples for class 2, etc.) //This can cause a problem for stochastic gradient descent algorithm. To avoid this issue, we randomly shuffle the order of the //training samples. This random order is then used at each epoch. for(UINT i=0; i<M; i++){ randomTrainingOrder[i] = i; } std::random_shuffle(randomTrainingOrder.begin(), randomTrainingOrder.end()); //Run the main stochastic gradient descent training algorithm while( keepTraining ){ //Run one epoch of training using stochastic gradient descent totalSquaredTrainingError = 0; for(UINT m=0; m<M; m++){ //Select the random sample UINT i = randomTrainingOrder[m]; //Compute the error, given the current weights VectorDouble x = trainingData[i].getInputVector(); VectorDouble y = trainingData[i].getTargetVector(); double h = w0; for(UINT j=0; j<N; j++){ h += x[j] * w[j]; } error = y[0] - sigmoid( h ); totalSquaredTrainingError += SQR(error); //Update the weights for(UINT j=0; j<N; j++){ w[j] += learningRate * error * x[j]; } w0 += learningRate * error; } //Compute the error delta = fabs( totalSquaredTrainingError-lastSquaredError ); lastSquaredError = totalSquaredTrainingError; //Check to see if we should stop if( delta <= minChange ){ keepTraining = false; } if( ++iter >= maxNumEpochs ){ keepTraining = false; } if( std::isinf( totalSquaredTrainingError ) || std::isnan( totalSquaredTrainingError ) ){ errorLog << "train_(RegressionData &trainingData) - Training failed! Total squared error is NAN. If scaling is not enabled then you should try to scale your data and see if this solves the issue." << endl; return false; } //Store the training results rootMeanSquaredTrainingError = sqrt( totalSquaredTrainingError / double(M) ); result.setRegressionResult(iter,totalSquaredTrainingError,rootMeanSquaredTrainingError,this); trainingResults.push_back( result ); //Notify any observers of the new result trainingResultsObserverManager.notifyObservers( result ); trainingLog << "Epoch: " << iter << " SSE: " << totalSquaredTrainingError << " Delta: " << delta << endl; } //Flag that the algorithm has been trained regressionData.resize(1,0); trained = true; return trained; }
bool MultidimensionalRegression::train_(RegressionData &trainingData){ const unsigned int M = trainingData.getNumSamples(); const unsigned int N = trainingData.getNumInputDimensions(); const unsigned int K = trainingData.getNumTargetDimensions(); trained = false; trainingResults.clear(); deleteRegressionModules(); if( !getIsRegressionModuleSet() ){ errorLog << "train_(RegressionData &trainingData) - The regression module has not been set!" << std::endl; return false; } if( M == 0 ){ errorLog << "train_(RegressionData &trainingData) - Training data has zero samples!" << std::endl; return false; } numInputDimensions = N; numOutputDimensions = K; inputVectorRanges.clear(); targetVectorRanges.clear(); //Scale the training and validation data, if needed if( useScaling ){ //Find the ranges for the input data inputVectorRanges = trainingData.getInputRanges(); //Find the ranges for the target data targetVectorRanges = trainingData.getTargetRanges(); //Scale the training data trainingData.scale(inputVectorRanges,targetVectorRanges,0.0,1.0); } //Setup the regression modules regressionModules.resize( K, NULL ); //Any scaling will happpen at the meta level, not the regression module letter, so ensure scaling is turned off for the modules regressifier->enableScaling( false ); for(UINT k=0; k<K; k++){ regressionModules[k] = regressifier->deepCopy(); if( regressionModules[k] == NULL ){ errorLog << "train(LabelledRegressionData &trainingData) - Failed to deep copy module " << k << std::endl; return false; } } //Train each regression module for(UINT k=0; k<K; k++){ trainingLog << "Training regression module: " << k << std::endl; //We need to create a 1 dimensional training dataset for the k'th target dimension RegressionData data; data.setInputAndTargetDimensions(N, 1); for(UINT i=0; i<M; i++){ if( !data.addSample(trainingData[i].getInputVector(), VectorFloat(1,trainingData[i].getTargetVector()[k]) ) ){ errorLog << "train_(RegressionData &trainingData) - Failed to add sample to dataset for regression module " << k << std::endl; return false; } } if( !regressionModules[k]->train( data ) ){ errorLog << "train_(RegressionData &trainingData) - Failed to train regression module " << k << std::endl; return false; } } //Flag that the algorithm has been trained regressionData.resize(K,0); trained = true; return trained; }
bool train( CommandLineParser &parser ){ infoLog << "Training regression model..." << endl; string trainDatasetFilename = ""; string modelFilename = ""; float learningRate = 0; float minChange = 0; unsigned int maxEpoch = 0; unsigned int batchSize = 0; //Get the filename if( !parser.get("filename",trainDatasetFilename) ){ errorLog << "Failed to parse filename from command line! You can set the filename using the -f." << endl; printHelp(); return false; } //Get the parameters from the parser parser.get("model-filename",modelFilename); parser.get( "learning-rate", learningRate ); parser.get( "min-change", minChange ); parser.get( "max-epoch", maxEpoch ); parser.get( "batch-size", batchSize ); infoLog << "settings: learning-rate: " << learningRate << " min-change: " << minChange << " max-epoch: " << maxEpoch << " batch-size: " << batchSize << endl; //Load the training data to train the model RegressionData trainingData; //Try and parse the input and target dimensions unsigned int numInputDimensions = 0; unsigned int numTargetDimensions = 0; if( parser.get("num-inputs",numInputDimensions) && parser.get("num-targets",numTargetDimensions) ){ infoLog << "num input dimensions: " << numInputDimensions << " num target dimensions: " << numTargetDimensions << endl; trainingData.setInputAndTargetDimensions( numInputDimensions, numTargetDimensions ); } if( (numInputDimensions == 0 || numTargetDimensions == 0) && Util::stringEndsWith( trainDatasetFilename, ".csv" ) ){ errorLog << "Failed to parse num input dimensions and num target dimensions from input arguments. You must supply the input and target dimensions if the data format is CSV!" << endl; printHelp(); return false; } infoLog << "- Loading Training Data..." << endl; if( !trainingData.load( trainDatasetFilename ) ){ errorLog << "Failed to load training data!\n"; return false; } const unsigned int N = trainingData.getNumInputDimensions(); const unsigned int T = trainingData.getNumTargetDimensions(); infoLog << "- Num training samples: " << trainingData.getNumSamples() << endl; infoLog << "- Num input dimensions: " << N << endl; infoLog << "- Num target dimensions: " << T << endl; //Create a new regression instance LogisticRegression regression; regression.setMaxNumEpochs( maxEpoch ); regression.setMinChange( minChange ); regression.setUseValidationSet( true ); regression.setValidationSetSize( 20 ); regression.setRandomiseTrainingOrder( true ); regression.enableScaling( true ); //Create a new pipeline that will hold the regression algorithm GestureRecognitionPipeline pipeline; //Add a multidimensional regression instance and set the regression algorithm to Linear Regression pipeline.setRegressifier( MultidimensionalRegression( regression, true ) ); infoLog << "- Training model...\n"; //Train the classifier if( !pipeline.train( trainingData ) ){ errorLog << "Failed to train model!" << endl; return false; } infoLog << "- Model trained!" << endl; infoLog << "- Saving model to: " << modelFilename << endl; //Save the pipeline if( pipeline.save( modelFilename ) ){ infoLog << "- Model saved." << endl; }else warningLog << "Failed to save model to file: " << modelFilename << endl; infoLog << "- TrainingTime: " << pipeline.getTrainingTime() << endl; return true; }