double FindBestGamma(dlib::krr_trainer<KernelType>& trainer,
const std::vector<SampleType>& samples,
const std::vector<LabelType>& labels)
{
// Useful method for calculating gamma value too
/*const double meanSquaredGamma = 1.0 / compute_mean_squared_distance(
/ randomly_subsample(trainingSamples, 100));*/
double bestGammaFound = 0;
double highestAccuracy = 0;
// Loop over different gamma values and uses most accurate it can find
for (double gamma = 0.000001; (gamma <= 1); gamma *= 5)
{
trainer.set_kernel(KernelType(gamma));
std::vector<double> looValues;
trainer.train(samples, labels, looValues);
const double classificationAccuracy = dlib::mean_sign_agreement(labels, looValues);
if (classificationAccuracy > highestAccuracy)
{
bestGammaFound = gamma;
highestAccuracy = classificationAccuracy;
}
}
return bestGammaFound;
}
/**
* Construct the exact kernel matrix.
*
* @param data Input data points.
* @param transformedData Matrix to output results into.
* @param eigval KPCA eigenvalues will be written to this vector.
* @param eigvec KPCA eigenvectors will be written to this matrix.
* @param rank Rank to be used for matrix approximation.
* @param kernel Kernel to be used for computation.
*/
static void ApplyKernelMatrix(const arma::mat& data,
arma::mat& transformedData,
arma::vec& eigval,
arma::mat& eigvec,
const size_t /* unused */,
KernelType kernel = KernelType())
{
// Construct the kernel matrix.
arma::mat kernelMatrix;
// Resize the kernel matrix to the right size.
kernelMatrix.set_size(data.n_cols, data.n_cols);
// Note that we only need to calculate the upper triangular part of the
// kernel matrix, since it is symmetric. This helps minimize the number of
// kernel evaluations.
for (size_t i = 0; i < data.n_cols; ++i)
{
for (size_t j = i; j < data.n_cols; ++j)
{
// Evaluate the kernel on these two points.
kernelMatrix(i, j) = kernel.Evaluate(data.unsafe_col(i),
data.unsafe_col(j));
}
}
// Copy to the lower triangular part of the matrix.
for (size_t i = 1; i < data.n_cols; ++i)
for (size_t j = 0; j < i; ++j)
kernelMatrix(i, j) = kernelMatrix(j, i);
// For PCA the data has to be centered, even if the data is centered. But it
// is not guaranteed that the data, when mapped to the kernel space, is also
// centered. Since we actually never work in the feature space we cannot
// center the data. So, we perform a "psuedo-centering" using the kernel
// matrix.
arma::rowvec rowMean = arma::sum(kernelMatrix, 0) / kernelMatrix.n_cols;
kernelMatrix.each_col() -= arma::sum(kernelMatrix, 1) / kernelMatrix.n_cols;
kernelMatrix.each_row() -= rowMean;
kernelMatrix += arma::sum(rowMean) / kernelMatrix.n_cols;
// Eigendecompose the centered kernel matrix.
arma::eig_sym(eigval, eigvec, kernelMatrix);
// Swap the eigenvalues since they are ordered backwards (we need largest to
// smallest).
for (size_t i = 0; i < floor(eigval.n_elem / 2.0); ++i)
eigval.swap_rows(i, (eigval.n_elem - 1) - i);
// Flip the coefficients to produce the same effect.
eigvec = arma::fliplr(eigvec);
transformedData = eigvec.t() * kernelMatrix;
transformedData.each_col() /= arma::sqrt(eigval);
}
void RunIrisSupervisedLearning()
{
// Load training/test dataset
Table data = ParseCSVFile("data/iris.data");
// Print out the data to ensure we've loaded it correctly
std::cout << "Loaded Data:" << std::endl;
PrintTable(data);
// Extract feature std::vectors and classes from the loaded data
std::vector<SampleType> allSamples = GetFeatureVectors(data);
std::vector<std::string> classes = GetClasses(data);
// Construct labels compatible with SVMs using the class data
// Each class is made an integer. The integers grow one number
// apart, so three classes will be assigned the labels 1, 2 and
// 3 respectively.
std::vector<LabelType> allLabels = ConstructLabels(classes);
// Randomise the samples and labels to ensure the normalisation process
// does affect the performance of cross validation
randomize_samples(allSamples, allLabels);
// Split dataset in half - one half being the training set
// and one half being the test set.
// Done AFTER randomising so half of the data set isn't one class
// and half is the other - would result in a very incorrect classifier!
unsigned int numTraining = round(allSamples.size() / 2);
unsigned int numTest = allSamples.size() - numTraining;
std::vector<SampleType> trainingSamples;
std::vector<LabelType> trainingLabels;
trainingSamples.reserve(numTraining);
trainingLabels.reserve(numTraining);
std::vector<SampleType> testSamples;
std::vector<LabelType> testLabels;
testSamples.reserve(numTest);
testLabels.reserve(numTest);
for (unsigned int i = 0; (i < numTraining); ++i)
{
trainingSamples.push_back(allSamples[i]);
trainingLabels.push_back(allLabels[i]);
}
for (unsigned int i = numTraining; (i < allSamples.size()); ++i)
{
testSamples.push_back(allSamples[i]);
testLabels.push_back(allLabels[i]);
}
// Construct a trainer for the problem
dlib::krr_trainer<KernelType> trainer;
double bestGamma = FindBestGamma(trainer, trainingSamples, trainingLabels);
trainer.set_kernel(KernelType(bestGamma));
// Actually TRAIN the classifier using the data, LEARNING the function
FunctionType learnedFunction;
learnedFunction = trainer.train(trainingSamples, trainingLabels);
// NOTE: This should just print out 1 for our training method
std::cout << "The number of support vectors in our learned function is "
<< learnedFunction.basis_vectors.nr() << std::endl;
double accuracy = CalculateAccuracy(learnedFunction, testSamples, testLabels);
std::cout << "The accuracy of this classifier is: "
<< (accuracy * 100) << "%." << std::endl;
}
