bool BAG::train_(ClassificationData &trainingData){ //Clear any previous models clear(); const unsigned int M = trainingData.getNumSamples(); const unsigned int N = trainingData.getNumDimensions(); const unsigned int K = trainingData.getNumClasses(); if( M == 0 ){ errorLog << "train_(ClassificationData &trainingData) - Training data has zero samples!" << endl; return false; } numInputDimensions = N; numClasses = K; classLabels.resize(K); ranges = trainingData.getRanges(); //Scale the training data if needed if( useScaling ){ //Scale the training data between 0 and 1 trainingData.scale(0, 1); } UINT ensembleSize = (UINT)ensemble.size(); if( ensembleSize == 0 ){ errorLog << "train_(ClassificationData &trainingData) - The ensemble size is zero! You need to add some classifiers to the ensemble first." << endl; return false; } for(UINT i=0; i<ensembleSize; i++){ if( ensemble[i] == NULL ){ errorLog << "train_(ClassificationData &trainingData) - The classifier at ensemble index " << i << " has not been set!" << endl; return false; } } //Train the ensemble for(UINT i=0; i<ensembleSize; i++){ ClassificationData boostedDataset = trainingData.getBootstrappedDataset(); trainingLog << "Training ensemble " << i+1 << ". Ensemble type: " << ensemble[i]->getClassType() << endl; //Train the classifier with the bootstrapped dataset if( !ensemble[i]->train( boostedDataset ) ){ errorLog << "train_(ClassificationData &trainingData) - The classifier at ensemble index " << i << " failed training!" << endl; return false; } } //Set the class labels classLabels = trainingData.getClassLabels(); //Flag that the model has been trained trained = true; return trained; }
bool AdaBoost::train_(ClassificationData &trainingData){ //Clear any previous model clear(); if( trainingData.getNumSamples() <= 1 ){ errorLog << "train_(ClassificationData &trainingData) - There are not enough training samples to train a model! Number of samples: " << trainingData.getNumSamples() << endl; return false; } numInputDimensions = trainingData.getNumDimensions(); numClasses = trainingData.getNumClasses(); const UINT M = trainingData.getNumSamples(); const UINT POSITIVE_LABEL = WEAK_CLASSIFIER_POSITIVE_CLASS_LABEL; const UINT NEGATIVE_LABEL = WEAK_CLASSIFIER_NEGATIVE_CLASS_LABEL; double alpha = 0; const double beta = 0.001; double epsilon = 0; TrainingResult trainingResult; const UINT K = (UINT)weakClassifiers.size(); if( K == 0 ){ errorLog << "train_(ClassificationData &trainingData) - No weakClassifiers have been set. You need to set at least one weak classifier first." << endl; return false; } classLabels.resize(numClasses); models.resize(numClasses); ranges = trainingData.getRanges(); //Scale the training data if needed if( useScaling ){ trainingData.scale(ranges,0,1); } //Create the weights vector VectorDouble weights(M); //Create the error matrix MatrixDouble errorMatrix(K,M); for(UINT classIter=0; classIter<numClasses; classIter++){ //Get the class label for the current class classLabels[classIter] = trainingData.getClassLabels()[classIter]; //Set the class label of the current model models[ classIter ].setClassLabel( classLabels[classIter] ); //Setup the labels for this class, POSITIVE_LABEL == 1, NEGATIVE_LABEL == 2 ClassificationData classData; classData.setNumDimensions(trainingData.getNumDimensions()); for(UINT i=0; i<M; i++){ UINT label = trainingData[i].getClassLabel()==classLabels[classIter] ? POSITIVE_LABEL : NEGATIVE_LABEL; VectorDouble trainingSample = trainingData[i].getSample(); classData.addSample(label,trainingSample); } //Setup the initial training sample weights std::fill(weights.begin(),weights.end(),1.0/M); //Run the boosting loop bool keepBoosting = true; UINT t = 0; while( keepBoosting ){ //Pick the classifier from the family of classifiers that minimizes the total error UINT bestClassifierIndex = 0; double minError = numeric_limits<double>::max(); for(UINT k=0; k<K; k++){ //Get the k'th possible classifier WeakClassifier *weakLearner = weakClassifiers[k]; //Train the current classifier if( !weakLearner->train(classData,weights) ){ errorLog << "Failed to train weakLearner!" << endl; return false; } //Compute the weighted error for this clasifier double e = 0; double positiveLabel = weakLearner->getPositiveClassLabel(); double numCorrect = 0; double numIncorrect = 0; for(UINT i=0; i<M; i++){ //Only penalize errors double prediction = weakLearner->predict( classData[i].getSample() ); if( (prediction == positiveLabel && classData[i].getClassLabel() != POSITIVE_LABEL) || //False positive (prediction != positiveLabel && classData[i].getClassLabel() == POSITIVE_LABEL) ){ //False negative e += weights[i]; //Increase the error proportional to the weight of the example errorMatrix[k][i] = 1; //Flag that there was an error numIncorrect++; }else{ errorMatrix[k][i] = 0; //Flag that there was no error numCorrect++; } } trainingLog << "PositiveClass: " << classLabels[classIter] << " Boosting Iter: " << t << " Classifier: " << k << " WeightedError: " << e << " NumCorrect: " << numCorrect/M << " NumIncorrect: " <<numIncorrect/M << endl; if( e < minError ){ minError = e; bestClassifierIndex = k; } } epsilon = minError; //Set alpha, using the M1 weight value, small weights (close to 0) will receive a strong weight in the final classifier alpha = 0.5 * log( (1.0-epsilon)/epsilon ); trainingLog << "PositiveClass: " << classLabels[classIter] << " Boosting Iter: " << t << " Best Classifier Index: " << bestClassifierIndex << " MinError: " << minError << " Alpha: " << alpha << endl; if( isinf(alpha) ){ keepBoosting = false; trainingLog << "Alpha is INF. Stopping boosting for current class" << endl; } if( 0.5 - epsilon <= beta ){ keepBoosting = false; trainingLog << "Epsilon <= Beta. Stopping boosting for current class" << endl; } if( ++t >= numBoostingIterations ) keepBoosting = false; trainingResult.setClassificationResult(t, minError, this); trainingResults.push_back(trainingResult); trainingResultsObserverManager.notifyObservers( trainingResult ); if( keepBoosting ){ //Add the best weak classifier to the committee models[ classIter ].addClassifierToCommitee( weakClassifiers[bestClassifierIndex], alpha ); //Update the weights for the next boosting iteration double reWeight = (1.0 - epsilon) / epsilon; double oldSum = 0; double newSum = 0; for(UINT i=0; i<M; i++){ oldSum += weights[i]; //Only update the weights that resulted in an incorrect prediction if( errorMatrix[bestClassifierIndex][i] == 1 ) weights[i] *= reWeight; newSum += weights[i]; } //Normalize all the weights //This results to increasing the weights of the samples that were incorrectly labelled //While decreasing the weights of the samples that were correctly classified reWeight = oldSum/newSum; for(UINT i=0; i<M; i++){ weights[i] *= reWeight; } }else{ trainingLog << "Stopping boosting training at iteration : " << t-1 << " with an error of " << epsilon << endl; if( t-1 == 0 ){ //Add the best weak classifier to the committee (we have to add it as this is the first iteration) if( isinf(alpha) ){ alpha = 1; } //If alpha is infinite then the first classifier got everything correct models[ classIter ].addClassifierToCommitee( weakClassifiers[bestClassifierIndex], alpha ); } } } } //Normalize the weights for(UINT k=0; k<numClasses; k++){ models[k].normalizeWeights(); } //Flag that the model has been trained trained = true; //Setup the data for prediction predictedClassLabel = 0; maxLikelihood = 0; classLikelihoods.resize(numClasses); classDistances.resize(numClasses); return true; }
bool RandomForests::train_(ClassificationData &trainingData){ //Clear any previous model clear(); const unsigned int M = trainingData.getNumSamples(); const unsigned int N = trainingData.getNumDimensions(); const unsigned int K = trainingData.getNumClasses(); if( M == 0 ){ errorLog << "train_(ClassificationData &trainingData) - Training data has zero samples!" << endl; return false; } if( bootstrappedDatasetWeight <= 0.0 || bootstrappedDatasetWeight > 1.0 ){ errorLog << "train_(ClassificationData &trainingData) - Bootstrapped Dataset Weight must be [> 0.0 and <= 1.0]" << endl; return false; } numInputDimensions = N; numClasses = K; classLabels = trainingData.getClassLabels(); ranges = trainingData.getRanges(); //Scale the training data if needed if( useScaling ){ //Scale the training data between 0 and 1 trainingData.scale(0, 1); } //Flag that the main algorithm has been trained encase we need to trigger any callbacks trained = true; //Train the random forest forest.reserve( forestSize ); for(UINT i=0; i<forestSize; i++){ //Get a balanced bootstrapped dataset UINT datasetSize = (UINT)(trainingData.getNumSamples() * bootstrappedDatasetWeight); ClassificationData data = trainingData.getBootstrappedDataset( datasetSize, true ); DecisionTree tree; tree.setDecisionTreeNode( *decisionTreeNode ); tree.enableScaling( false ); //We have already scaled the training data so we do not need to scale it again tree.setTrainingMode( trainingMode ); tree.setNumSplittingSteps( numRandomSplits ); tree.setMinNumSamplesPerNode( minNumSamplesPerNode ); tree.setMaxDepth( maxDepth ); tree.enableNullRejection( useNullRejection ); tree.setRemoveFeaturesAtEachSpilt( removeFeaturesAtEachSpilt ); trainingLog << "Training forest " << i+1 << "/" << forestSize << "..." << endl; //Train this tree if( !tree.train( data ) ){ errorLog << "train_(ClassificationData &labelledTrainingData) - Failed to train tree at forest index: " << i << endl; clear(); return false; } //Deep copy the tree into the forest forest.push_back( tree.deepCopyTree() ); } return true; }
bool RandomForests::train_(ClassificationData &trainingData){ //Clear any previous model clear(); const unsigned int M = trainingData.getNumSamples(); const unsigned int N = trainingData.getNumDimensions(); const unsigned int K = trainingData.getNumClasses(); if( M == 0 ){ errorLog << "train_(ClassificationData &trainingData) - Training data has zero samples!" << std::endl; return false; } if( bootstrappedDatasetWeight <= 0.0 || bootstrappedDatasetWeight > 1.0 ){ errorLog << "train_(ClassificationData &trainingData) - Bootstrapped Dataset Weight must be [> 0.0 and <= 1.0]" << std::endl; return false; } numInputDimensions = N; numClasses = K; classLabels = trainingData.getClassLabels(); ranges = trainingData.getRanges(); //Scale the training data if needed if( useScaling ){ //Scale the training data between 0 and 1 trainingData.scale(0, 1); } if( useValidationSet ){ validationSetAccuracy = 0; validationSetPrecision.resize( useNullRejection ? K+1 : K, 0 ); validationSetRecall.resize( useNullRejection ? K+1 : K, 0 ); } //Flag that the main algorithm has been trained encase we need to trigger any callbacks trained = true; //Train the random forest forest.reserve( forestSize ); for(UINT i=0; i<forestSize; i++){ //Get a balanced bootstrapped dataset UINT datasetSize = (UINT)(trainingData.getNumSamples() * bootstrappedDatasetWeight); ClassificationData data = trainingData.getBootstrappedDataset( datasetSize, true ); Timer timer; timer.start(); DecisionTree tree; tree.setDecisionTreeNode( *decisionTreeNode ); tree.enableScaling( false ); //We have already scaled the training data so we do not need to scale it again tree.setUseValidationSet( useValidationSet ); tree.setValidationSetSize( validationSetSize ); tree.setTrainingMode( trainingMode ); tree.setNumSplittingSteps( numRandomSplits ); tree.setMinNumSamplesPerNode( minNumSamplesPerNode ); tree.setMaxDepth( maxDepth ); tree.enableNullRejection( useNullRejection ); tree.setRemoveFeaturesAtEachSpilt( removeFeaturesAtEachSpilt ); trainingLog << "Training decision tree " << i+1 << "/" << forestSize << "..." << std::endl; //Train this tree if( !tree.train_( data ) ){ errorLog << "train_(ClassificationData &trainingData) - Failed to train tree at forest index: " << i << std::endl; clear(); return false; } Float computeTime = timer.getMilliSeconds(); trainingLog << "Decision tree trained in " << (computeTime*0.001)/60.0 << " minutes" << std::endl; if( useValidationSet ){ Float forestNorm = 1.0 / forestSize; validationSetAccuracy += tree.getValidationSetAccuracy(); VectorFloat precision = tree.getValidationSetPrecision(); VectorFloat recall = tree.getValidationSetRecall(); grt_assert( precision.getSize() == validationSetPrecision.getSize() ); grt_assert( recall.getSize() == validationSetRecall.getSize() ); for(UINT i=0; i<validationSetPrecision.getSize(); i++){ validationSetPrecision[i] += precision[i] * forestNorm; } for(UINT i=0; i<validationSetRecall.getSize(); i++){ validationSetRecall[i] += recall[i] * forestNorm; } } //Deep copy the tree into the forest forest.push_back( tree.deepCopyTree() ); } if( useValidationSet ){ validationSetAccuracy /= forestSize; trainingLog << "Validation set accuracy: " << validationSetAccuracy << std::endl; trainingLog << "Validation set precision: "; for(UINT i=0; i<validationSetPrecision.getSize(); i++){ trainingLog << validationSetPrecision[i] << " "; } trainingLog << std::endl; trainingLog << "Validation set recall: "; for(UINT i=0; i<validationSetRecall.getSize(); i++){ trainingLog << validationSetRecall[i] << " "; } trainingLog << std::endl; } return true; }