LabelledRegressionData LabelledRegressionData::getTrainingFoldData(const UINT foldIndex) const{
LabelledRegressionData trainingData;
if( !crossValidationSetup ){
errorLog << "getTrainingFoldData(UINT foldIndex) - Cross Validation has not been setup! You need to call the spiltDataIntoKFolds(UINT K,bool useStratifiedSampling) function first before calling this function!" << endl;
return trainingData;
}
if( foldIndex >= kFoldValue ) return trainingData;
trainingData.setInputAndTargetDimensions(numInputDimensions, numTargetDimensions);
//Add the data to the training set, this will consist of all the data that is NOT in the foldIndex
UINT index = 0;
for(UINT k=0; k<kFoldValue; k++){
if( k != foldIndex ){
for(UINT i=0; i<crossValidationIndexs[k].size(); i++){
index = crossValidationIndexs[k][i];
trainingData.addSample( data[ index ].getInputVector(), data[ index ].getTargetVector() );
}
}
}
return trainingData;
}
LabelledRegressionData LabelledClassificationData::reformatAsLabelledRegressionData() const{
//Turns the classification into a regression data to enable regression algorithms like the MLP to be used as a classifier
//This sets the number of targets in the regression data equal to the number of classes in the classification data
//The output of each regression training sample will then be all 0's, except for the index matching the classLabel, which will be 1
//For this to work, the labelled classification data cannot have any samples with a classLabel of 0!
LabelledRegressionData regressionData;
if( totalNumSamples == 0 ){
return regressionData;
}
const UINT numInputDimensions = numDimensions;
const UINT numTargetDimensions = getNumClasses();
regressionData.setInputAndTargetDimensions(numInputDimensions, numTargetDimensions);
for(UINT i=0; i<totalNumSamples; i++){
VectorDouble targetVector(numTargetDimensions,0);
//Set the class index in the target vector to 1 and all other values in the target vector to 0
UINT classLabel = data[i].getClassLabel();
if( classLabel > 0 ){
targetVector[ classLabel-1 ] = 1;
}else{
regressionData.clear();
return regressionData;
}
regressionData.addSample(data[i].getSample(),targetVector);
}
return regressionData;
}
LabelledRegressionData LabelledRegressionData::getTestFoldData(const UINT foldIndex) const{
LabelledRegressionData testData;
if( !crossValidationSetup ) return testData;
if( foldIndex >= kFoldValue ) return testData;
//Add the data to the training
testData.setInputAndTargetDimensions(numInputDimensions, numTargetDimensions);
UINT index = 0;
for(UINT i=0; i<crossValidationIndexs[ foldIndex ].size(); i++){
index = crossValidationIndexs[ foldIndex ][i];
testData.addSample( data[ index ].getInputVector(), data[ index ].getTargetVector() );
}
return testData;
}
bool LabelledRegressionData::merge(const LabelledRegressionData ®ressionData){
if( regressionData.getNumInputDimensions() != numInputDimensions ){
errorLog << "merge(LabelledRegressionData ®ressionData) - The number of input dimensions in the regressionData (" << regressionData.getNumInputDimensions() << ") does not match the number of input dimensions of this dataset (" << numInputDimensions << ")" << endl;
return false;
}
if( regressionData.getNumTargetDimensions() != numTargetDimensions ){
errorLog << "merge(LabelledRegressionData ®ressionData) - The number of target dimensions in the regressionData (" << regressionData.getNumTargetDimensions() << ") does not match the number of target dimensions of this dataset (" << numTargetDimensions << ")" << endl;
return false;
}
//Add the data from the labelledData to this instance
for(UINT i=0; i<regressionData.getNumSamples(); i++){
addSample(regressionData[i].getInputVector(), regressionData[i].getTargetVector());
}
//The dataset has changed so flag that any previous cross validation setup will now not work
crossValidationSetup = false;
crossValidationIndexs.clear();
return true;
}
bool LinearRegression::train(LabelledRegressionData &trainingData){
const unsigned int M = trainingData.getNumSamples();
const unsigned int N = trainingData.getNumInputDimensions();
const unsigned int K = trainingData.getNumTargetDimensions();
trained = false;
if( M == 0 ){
errorLog << "train(LabelledRegressionData &trainingData) - Training data has zero samples!" << endl;
return false;
}
if( K == 0 ){
errorLog << "train(LabelledRegressionData &trainingData) - The number of target dimensions is not 1!" << endl;
return false;
}
numFeatures = N;
numOutputDimensions = 1; //Logistic Regression will have 1 output
inputVectorRanges.clear();
targetVectorRanges.clear();
//Scale the training and validation data, if needed
if( useScaling ){
//Find the ranges for the input data
inputVectorRanges = trainingData.getInputRanges();
//Find the ranges for the target data
targetVectorRanges = trainingData.getTargetRanges();
//Scale the training data
trainingData.scale(inputVectorRanges,targetVectorRanges,0.0,1.0);
}
//Reset the weights
Random rand;
w0 = rand.getRandomNumberUniform(-0.1,0.1);
w.resize(N);
for(UINT j=0; j<N; j++){
w[j] = rand.getRandomNumberUniform(-0.1,0.1);
}
double error = 0;
double errorSum = 0;
double lastErrorSum = 0;
double delta = 0;
UINT iter = 0;
bool keepTraining = true;
Random random;
vector< UINT > randomTrainingOrder(M);
//In most cases, the training data is grouped into classes (100 samples for class 1, followed by 100 samples for class 2, etc.)
//This can cause a problem for stochastic gradient descent algorithm. To avoid this issue, we randomly shuffle the order of the
//training samples. This random order is then used at each epoch.
for(UINT i=0; i<M; i++){
randomTrainingOrder[i] = i;
}
std::random_shuffle(randomTrainingOrder.begin(), randomTrainingOrder.end());
//Run the main stochastic gradient descent training algorithm
while( keepTraining ){
//Run one epoch of training using stochastic gradient descent
errorSum = 0;
for(UINT m=0; m<M; m++){
//Select the random sample
UINT i = randomTrainingOrder[m];
//Compute the error, given the current weights
VectorDouble x = trainingData[i].getInputVector();
VectorDouble y = trainingData[i].getTargetVector();
double h = w0;
for(UINT j=0; j<N; j++){
h += x[j] * w[j];
}
error = y[0] - sigmoid( h );
errorSum += error;
//Update the weights
for(UINT j=0; j<N; j++){
w[j] += learningRate * error * x[j];
}
w0 += learningRate * error;
}
//Compute the error
delta = fabs( errorSum-lastErrorSum );
lastErrorSum = errorSum;
//Check to see if we should stop
if( delta <= minChange ){
keepTraining = false;
}
if( ++iter >= maxNumIterations ){
keepTraining = false;
}
trainingLog << "Epoch: " << iter << " TotalError: " << errorSum << " Delta: " << delta << endl;
}
//Flag that the algorithm has been trained
regressionData.resize(1,0);
trained = true;
return trained;
}
bool LogisticRegression::train(LabelledRegressionData trainingData){
const unsigned int M = trainingData.getNumSamples();
const unsigned int N = trainingData.getNumInputDimensions();
const unsigned int K = trainingData.getNumTargetDimensions();
trained = false;
trainingResults.clear();
if( M == 0 ){
errorLog << "train(LabelledRegressionData trainingData) - Training data has zero samples!" << endl;
return false;
}
if( K == 0 ){
errorLog << "train(LabelledRegressionData trainingData) - The number of target dimensions is not 1!" << endl;
return false;
}
numInputDimensions = N;
numOutputDimensions = 1; //Logistic Regression will have 1 output
inputVectorRanges.clear();
targetVectorRanges.clear();
//Scale the training and validation data, if needed
if( useScaling ){
//Find the ranges for the input data
inputVectorRanges = trainingData.getInputRanges();
//Find the ranges for the target data
targetVectorRanges = trainingData.getTargetRanges();
//Scale the training data
trainingData.scale(inputVectorRanges,targetVectorRanges,0.0,1.0);
}
//Reset the weights
Random rand;
w0 = rand.getRandomNumberUniform(-0.1,0.1);
w.resize(N);
for(UINT j=0; j<N; j++){
w[j] = rand.getRandomNumberUniform(-0.1,0.1);
}
double error = 0;
double lastSquaredError = 0;
double delta = 0;
UINT iter = 0;
bool keepTraining = true;
Random random;
vector< UINT > randomTrainingOrder(M);
TrainingResult result;
trainingResults.reserve(M);
//In most cases, the training data is grouped into classes (100 samples for class 1, followed by 100 samples for class 2, etc.)
//This can cause a problem for stochastic gradient descent algorithm. To avoid this issue, we randomly shuffle the order of the
//training samples. This random order is then used at each epoch.
for(UINT i=0; i<M; i++){
randomTrainingOrder[i] = i;
}
std::random_shuffle(randomTrainingOrder.begin(), randomTrainingOrder.end());
//Run the main stochastic gradient descent training algorithm
while( keepTraining ){
//Run one epoch of training using stochastic gradient descent
totalSquaredTrainingError = 0;
for(UINT m=0; m<M; m++){
//Select the random sample
UINT i = randomTrainingOrder[m];
//Compute the error, given the current weights
VectorDouble x = trainingData[i].getInputVector();
VectorDouble y = trainingData[i].getTargetVector();
double h = w0;
for(UINT j=0; j<N; j++){
h += x[j] * w[j];
}
error = y[0] - sigmoid( h );
totalSquaredTrainingError += SQR(error);
//Update the weights
for(UINT j=0; j<N; j++){
w[j] += learningRate * error * x[j];
}
w0 += learningRate * error;
}
//Compute the error
delta = fabs( totalSquaredTrainingError-lastSquaredError );
lastSquaredError = totalSquaredTrainingError;
//Check to see if we should stop
if( delta <= minChange ){
keepTraining = false;
}
if( ++iter >= maxNumEpochs ){
keepTraining = false;
}
if( isinf( totalSquaredTrainingError ) || isnan( totalSquaredTrainingError ) ){
errorLog << "train(LabelledRegressionData &trainingData) - Training failed! Total squared error is NAN. If scaling is not enabled then you should try to scale your data and see if this solves the issue." << endl;
return false;
}
//Store the training results
rootMeanSquaredTrainingError = sqrt( totalSquaredTrainingError / double(M) );
result.setRegressionResult(iter,totalSquaredTrainingError,rootMeanSquaredTrainingError);
trainingResults.push_back( result );
//Notify any observers of the new training data
trainingResultsObserverManager.notifyObservers( result );
trainingLog << "Epoch: " << iter << " SSE: " << totalSquaredTrainingError << " Delta: " << delta << endl;
}
//Flag that the algorithm has been trained
regressionData.resize(1,0);
trained = true;
return trained;
}