bool LinearRegression::train(LabelledRegressionData &trainingData){
const unsigned int M = trainingData.getNumSamples();
const unsigned int N = trainingData.getNumInputDimensions();
const unsigned int K = trainingData.getNumTargetDimensions();
trained = false;
if( M == 0 ){
errorLog << "train(LabelledRegressionData &trainingData) - Training data has zero samples!" << endl;
return false;
}
if( K == 0 ){
errorLog << "train(LabelledRegressionData &trainingData) - The number of target dimensions is not 1!" << endl;
return false;
}
numFeatures = N;
numOutputDimensions = 1; //Logistic Regression will have 1 output
inputVectorRanges.clear();
targetVectorRanges.clear();
//Scale the training and validation data, if needed
if( useScaling ){
//Find the ranges for the input data
inputVectorRanges = trainingData.getInputRanges();
//Find the ranges for the target data
targetVectorRanges = trainingData.getTargetRanges();
//Scale the training data
trainingData.scale(inputVectorRanges,targetVectorRanges,0.0,1.0);
}
//Reset the weights
Random rand;
w0 = rand.getRandomNumberUniform(-0.1,0.1);
w.resize(N);
for(UINT j=0; j<N; j++){
w[j] = rand.getRandomNumberUniform(-0.1,0.1);
}
double error = 0;
double errorSum = 0;
double lastErrorSum = 0;
double delta = 0;
UINT iter = 0;
bool keepTraining = true;
Random random;
vector< UINT > randomTrainingOrder(M);
//In most cases, the training data is grouped into classes (100 samples for class 1, followed by 100 samples for class 2, etc.)
//This can cause a problem for stochastic gradient descent algorithm. To avoid this issue, we randomly shuffle the order of the
//training samples. This random order is then used at each epoch.
for(UINT i=0; i<M; i++){
randomTrainingOrder[i] = i;
}
std::random_shuffle(randomTrainingOrder.begin(), randomTrainingOrder.end());
//Run the main stochastic gradient descent training algorithm
while( keepTraining ){
//Run one epoch of training using stochastic gradient descent
errorSum = 0;
for(UINT m=0; m<M; m++){
//Select the random sample
UINT i = randomTrainingOrder[m];
//Compute the error, given the current weights
VectorDouble x = trainingData[i].getInputVector();
VectorDouble y = trainingData[i].getTargetVector();
double h = w0;
for(UINT j=0; j<N; j++){
h += x[j] * w[j];
}
error = y[0] - sigmoid( h );
errorSum += error;
//Update the weights
for(UINT j=0; j<N; j++){
w[j] += learningRate * error * x[j];
}
w0 += learningRate * error;
}
//Compute the error
delta = fabs( errorSum-lastErrorSum );
lastErrorSum = errorSum;
//Check to see if we should stop
if( delta <= minChange ){
keepTraining = false;
}
if( ++iter >= maxNumIterations ){
keepTraining = false;
}
trainingLog << "Epoch: " << iter << " TotalError: " << errorSum << " Delta: " << delta << endl;
}
//Flag that the algorithm has been trained
regressionData.resize(1,0);
trained = true;
return trained;
}
bool LogisticRegression::train(LabelledRegressionData trainingData){
const unsigned int M = trainingData.getNumSamples();
const unsigned int N = trainingData.getNumInputDimensions();
const unsigned int K = trainingData.getNumTargetDimensions();
trained = false;
trainingResults.clear();
if( M == 0 ){
errorLog << "train(LabelledRegressionData trainingData) - Training data has zero samples!" << endl;
return false;
}
if( K == 0 ){
errorLog << "train(LabelledRegressionData trainingData) - The number of target dimensions is not 1!" << endl;
return false;
}
numInputDimensions = N;
numOutputDimensions = 1; //Logistic Regression will have 1 output
inputVectorRanges.clear();
targetVectorRanges.clear();
//Scale the training and validation data, if needed
if( useScaling ){
//Find the ranges for the input data
inputVectorRanges = trainingData.getInputRanges();
//Find the ranges for the target data
targetVectorRanges = trainingData.getTargetRanges();
//Scale the training data
trainingData.scale(inputVectorRanges,targetVectorRanges,0.0,1.0);
}
//Reset the weights
Random rand;
w0 = rand.getRandomNumberUniform(-0.1,0.1);
w.resize(N);
for(UINT j=0; j<N; j++){
w[j] = rand.getRandomNumberUniform(-0.1,0.1);
}
double error = 0;
double lastSquaredError = 0;
double delta = 0;
UINT iter = 0;
bool keepTraining = true;
Random random;
vector< UINT > randomTrainingOrder(M);
TrainingResult result;
trainingResults.reserve(M);
//In most cases, the training data is grouped into classes (100 samples for class 1, followed by 100 samples for class 2, etc.)
//This can cause a problem for stochastic gradient descent algorithm. To avoid this issue, we randomly shuffle the order of the
//training samples. This random order is then used at each epoch.
for(UINT i=0; i<M; i++){
randomTrainingOrder[i] = i;
}
std::random_shuffle(randomTrainingOrder.begin(), randomTrainingOrder.end());
//Run the main stochastic gradient descent training algorithm
while( keepTraining ){
//Run one epoch of training using stochastic gradient descent
totalSquaredTrainingError = 0;
for(UINT m=0; m<M; m++){
//Select the random sample
UINT i = randomTrainingOrder[m];
//Compute the error, given the current weights
VectorDouble x = trainingData[i].getInputVector();
VectorDouble y = trainingData[i].getTargetVector();
double h = w0;
for(UINT j=0; j<N; j++){
h += x[j] * w[j];
}
error = y[0] - sigmoid( h );
totalSquaredTrainingError += SQR(error);
//Update the weights
for(UINT j=0; j<N; j++){
w[j] += learningRate * error * x[j];
}
w0 += learningRate * error;
}
//Compute the error
delta = fabs( totalSquaredTrainingError-lastSquaredError );
lastSquaredError = totalSquaredTrainingError;
//Check to see if we should stop
if( delta <= minChange ){
keepTraining = false;
}
if( ++iter >= maxNumEpochs ){
keepTraining = false;
}
if( isinf( totalSquaredTrainingError ) || isnan( totalSquaredTrainingError ) ){
errorLog << "train(LabelledRegressionData &trainingData) - Training failed! Total squared error is NAN. If scaling is not enabled then you should try to scale your data and see if this solves the issue." << endl;
return false;
}
//Store the training results
rootMeanSquaredTrainingError = sqrt( totalSquaredTrainingError / double(M) );
result.setRegressionResult(iter,totalSquaredTrainingError,rootMeanSquaredTrainingError);
trainingResults.push_back( result );
//Notify any observers of the new training data
trainingResultsObserverManager.notifyObservers( result );
trainingLog << "Epoch: " << iter << " SSE: " << totalSquaredTrainingError << " Delta: " << delta << endl;
}
//Flag that the algorithm has been trained
regressionData.resize(1,0);
trained = true;
return trained;
}