/**
* Return parameter of a Laplace distribution
*/
double SVM_SVMType_SVR::svm_svr_probability(const SVM_Problem & prob, const SVM_Parameter & param) const {
unsigned int i;
unsigned int nr_fold = 5;
std::vector<double> ymv(prob.dimension);
double mae = 0;
SVM_Parameter newparam = param;
newparam.probability = 0;
svm_cross_validation(prob, newparam, nr_fold, ymv);
for (i = 0; i < prob.dimension; i++) {
ymv[i] = prob.labels[i] - ymv[i];
mae += fabs(ymv[i]);
}
mae /= prob.dimension;
double std = sqrt(2 * mae * mae);
int count = 0;
mae = 0;
double threshold = 5 * std;
for (unsigned i = 0; i < prob.dimension; ++i) {
double fabsi = fabs(ymv[i]);
if (fabsi > threshold) {
++count;
} else {
mae += fabsi;
}
}
mae /= (prob.dimension - count);
std::cout << "Prob. model for test data: target value = predicted value + z,\nz: Laplace distribution e^(-|z|/sigma)/(2sigma),sigma= " << mae << "\n";
ymv.clear();
return mae;
}
static VALUE cModel_class_cross_validation(VALUE cls, VALUE problem, VALUE parameter, VALUE num_fold)
{
const struct svm_problem *prob;
const struct svm_parameter *param;
int nr_fold, i;
double *target_ptr;
VALUE target;
Data_Get_Struct(problem, struct svm_problem, prob);
Data_Get_Struct(parameter, struct svm_parameter, param);
nr_fold = NUM2INT(num_fold);
target = rb_ary_new2(prob->l);
target_ptr = (double *)calloc(prob->l, sizeof(double));
if(target_ptr == 0) {
rb_raise(rb_eNoMemError, "on cross-validation result allocation" " %s:%i", __FILE__,__LINE__);
}
svm_cross_validation(prob, param, nr_fold, target_ptr);
for(i = 0; i < prob->l; ++i) {
rb_ary_push(target, rb_float_new(*(target_ptr+i)));
}
free(target_ptr);
return target;
}
svm_classifier_t *svm_finish_classifier(svm_classifier_t *svm) {
int i, j, nr_fold=5, total_correct, best_correct=0, best_c=svm->param.C, best_g=svm->param.gamma;
const char *error_msg;
double *result = (double *) rs_malloc(sizeof(double)*svm->problem.l,"cross validation result");
rs_msg("Building SVM classifier...");
if (svm->finished)
rs_warning("SVM classifier is already trained!");
error_msg = svm_check_parameter(&(svm->problem),&(svm->param));
if(error_msg)
rs_error("%s",error_msg);
/* Skalierung */
_create_scaling(svm->problem,svm->feature_dim,&(svm->max),&(svm->min));
for (i=0;i<svm->problem.l;i++)
_scale_instance(&(svm->problem.x[i]),svm->feature_dim,svm->max,svm->min);
/* Cross-Validation, um C und G zu bestimmen bei RBF-Kernel */
if (svm->param.kernel_type == RBF) {
svm->param.probability = 0;
for (i=0;i<C_G_ITER;i++) {
total_correct=0;
svm->param.C=pow(2,C[i]);
svm->param.gamma=pow(2,G[i]);
svm_cross_validation(&(svm->problem),&(svm->param),nr_fold,result);
for(j=0;j<svm->problem.l;j++) {
if(result[j] == svm->problem.y[j])
++total_correct;
}
if (total_correct > best_correct) {
best_correct=total_correct;
best_c=C[i];
best_g=G[i];
}
rs_msg("C-G-Selektion-Iteration # %d: tried c=%g and g=%g => CV rate is %g; current best c=%g and g=%g with CV rate %g",i+1,pow(2,C[i]),pow(2,G[i]),total_correct*100.0/svm->problem.l,pow(2,best_c),pow(2,best_g),best_correct*100.0/svm->problem.l);
}
/* Training */
svm->param.C=pow(2,best_c);
svm->param.gamma=pow(2,best_g);
svm->param.probability = 1;
}
svm->model=svm_train(&(svm->problem),&(svm->param));
svm->finished=1;
// @begin_add_johannes
rs_free (result);
// @end_add_johannes
return svm;
}
double do_crossvalidation(struct svm_problem * p_km)
{
double rate;
int i;
int total_correct = 0;
double total_error = 0;
double sumv = 0, sumy = 0, sumvv = 0, sumyy = 0, sumvy = 0;
double *target = Malloc(double,prob.l);
svm_cross_validation(p_km,¶m,nr_fold,target);
if(param.svm_type == EPSILON_SVR ||
param.svm_type == NU_SVR)
{
for(i=0;i<prob.l;i++)
{
double y = prob.y[i];
double v = target[i];
total_error += (v-y)*(v-y);
sumv += v;
sumy += y;
sumvv += v*v;
sumyy += y*y;
sumvy += v*y;
}
printf("Cross Validation Mean squared error = %g\n",total_error/prob.l);
printf("Cross Validation Squared correlation coefficient = %g\n",
((prob.l*sumvy-sumv*sumy)*(prob.l*sumvy-sumv*sumy))/
((prob.l*sumvv-sumv*sumv)*(prob.l*sumyy-sumy*sumy))
);
}
else
{
for(i=0;i<prob.l;i++)
if(target[i] == prob.y[i])
++total_correct;
rate = (100.0*total_correct)/prob.l;
}
free(target);
return rate;
}
/*@returns: cross validation error
*
*
*/
double SVM::crossValidationSamples(struct svm_problem &strSvmProblem, struct svm_parameter &strParameters, int iNrfold)
{
int i;
int total_correct = 0;
double total_error = 0;
double sumv = 0, sumy = 0, sumvv = 0, sumyy = 0, sumvy = 0;
double *target = Malloc(double,strSvmProblem.l);
double dCrossValError = -1;
svm_cross_validation(&strSvmProblem,&strParameters,iNrfold,target);
if(strParameters.svm_type == EPSILON_SVR || strParameters.svm_type == NU_SVR)
{
for(i=0;i<strSvmProblem.l;i++)
{
double y = strSvmProblem.y[i];
double v = target[i];
total_error += (v-y)*(v-y);
sumv += v;
sumy += y;
sumvv += v*v;
sumyy += y*y;
sumvy += v*y;
}
dCrossValError = total_error/(double)strSvmProblem.l;
printf("Cross Validation Mean squared error = %g\n", dCrossValError);
printf("Cross Validation Squared correlation coefficient = %g\n",
((strSvmProblem.l*sumvy-sumv*sumy)*(strSvmProblem.l*sumvy-sumv*sumy))/
((strSvmProblem.l*sumvv-sumv*sumv)*(strSvmProblem.l*sumyy-sumy*sumy))
);
}
else
{
for(i=0;i<strSvmProblem.l;i++)
if(target[i] == strSvmProblem.y[i])
++total_correct;
dCrossValError = 1 - ((double)total_correct/(double)strSvmProblem.l);
printf("Cross Validation Accuarcy = %lf%%\n", 100*(1-dCrossValError));
}
free(target);
return dCrossValError;
}
virtual void Learn(IDataSet* data) {
vector< vector<double> > features = PreprocessFeatures(data);
vector<int> targets;
for (int i = 0; i < data->GetObjectCount(); i++)
targets.push_back(data->GetTarget(i));
train_prob = sh_ptr<svm_problem_xx>(new svm_problem_xx(features, targets));
double c_values[] = { 2048, 32768 };
double gamma_values[] = { 3.0517578125e-05, 0.0001220703125 };
svm_parameter best_param;
double best_accuracy = -1;
// select hyperparameters (regularization coefficient C and kernel parameter gamma) by 10-fold crossvalidation
for (int c_index = 0; c_index < ARRAY_SIZE(c_values); c_index++) {
for (int gamma_index = 0; gamma_index < ARRAY_SIZE(gamma_values); gamma_index++) {
svm_parameter param;
memset(¶m, 0, sizeof(param));
param.svm_type = C_SVC;
param.kernel_type = RBF;
param.C = c_values[c_index];
param.gamma = gamma_values[gamma_index];
param.cache_size = 100;
param.eps = 1e-3;
param.shrinking = 1;
vector<double> res(targets.size());
svm_cross_validation(train_prob.get(), ¶m, 10, &res[0]);
double accuracy = 0;
for (int i = 0; i < res.size(); i++)
if (res[i] == targets[i])
accuracy += 1;
accuracy /= res.size();
if (accuracy > best_accuracy) {
best_accuracy = accuracy;
best_param = param;
}
}
}
model = sh_ptr<svm_model_xx>(new svm_model_xx(svm_train(train_prob.get(), &best_param)));
}
static void cmd_cross(int nFold, double cParam, double gParam,
WindowFile &trainA, WindowFile &trainB)
{
SVMProblem problem(trainA, trainB);
SVMParam param(cParam, gParam);
double *target = new double[problem.l];
svm_cross_validation(&problem, ¶m, nFold, target);
int correct = 0;
for(int i = 0; i < problem.l; i++)
if(target[i] == problem.y[i])
++correct;
delete [] target;
fprintf(stderr, "Cross Validation Accuracy = %g%%\n",(100.*correct)/problem.l);
}
double do_cross_validation()
{
int i;
int total_correct = 0;
double total_error = 0;
double sumv = 0, sumy = 0, sumvv = 0, sumyy = 0, sumvy = 0;
double *target = Malloc(double,prob.l);
double retval = 0.0;
svm_cross_validation(&prob,¶m,nr_fold,target);
if(param.svm_type == EPSILON_SVR ||
param.svm_type == NU_SVR)
{
for(i=0;i<prob.l;i++)
{
double y = prob.y[i];
double v = target[i];
total_error += (v-y)*(v-y);
sumv += v;
sumy += y;
sumvv += v*v;
sumyy += y*y;
sumvy += v*y;
}
mexPrintf("Cross Validation Mean squared error = %g\n",total_error/prob.l);
mexPrintf("Cross Validation Squared correlation coefficient = %g\n",
((prob.l*sumvy-sumv*sumy)*(prob.l*sumvy-sumv*sumy))/
((prob.l*sumvv-sumv*sumv)*(prob.l*sumyy-sumy*sumy))
);
retval = total_error/prob.l;
}
else
{
for(i=0;i<prob.l;i++)
if(target[i] == prob.y[i])
++total_correct;
mexPrintf("Cross Validation Accuracy = %g%%\n",100.0*total_correct/prob.l);
retval = 100.0*total_correct/prob.l;
}
free(target);
return retval;
}
void svm_do_cross_validation(const svm_parameter& param, const svm_problem& prob,int nr_fold)
{
int i;
int total_correct = 0;
double total_error = 0;
double sumv = 0, sumy = 0, sumvv = 0, sumyy = 0, sumvy = 0;
double *target = new double[prob.l];
memset(target, 0, sizeof(double)*prob.l);
svm_cross_validation(&prob, ¶m, nr_fold, target);
if(param.svm_type == EPSILON_SVR ||
param.svm_type == NU_SVR)
{
for(i=0;i<prob.l;i++)
{
double y = prob.y[i];
double v = target[i];
total_error += (v-y)*(v-y);
sumv += v;
sumy += y;
sumvv += v*v;
sumyy += y*y;
sumvy += v*y;
}
printf("Cross Validation Mean squared error = %g\n",total_error/prob.l);
printf("Cross Validation Squared correlation coefficient = %g\n",
((prob.l*sumvy-sumv*sumy)*(prob.l*sumvy-sumv*sumy))/
((prob.l*sumvv-sumv*sumv)*(prob.l*sumyy-sumy*sumy))
);
}
else
{
for(i=0;i<prob.l;i++)
if(target[i] == prob.y[i])
++total_correct;
printf("Cross Validation Accuracy = %g%%\n",100.0*total_correct/prob.l);
}
delete [] target;
}
// does cross validation using the parameters obtained or default ones
double ML2::do_cross_validation()
{
int i;
int total_correct = 0;
double total_error = 0;
double sumv = 0, sumy = 0, sumvv = 0, sumyy = 0, sumvy = 0;
double *target = Malloc(double,prob.l);
//this uses the nr_folds set in the train method
svm_cross_validation(&prob,¶m,nr_fold,target);
if(param.svm_type == EPSILON_SVR ||
param.svm_type == NU_SVR)
{
for(i=0;i<prob.l;i++)
{
double y = prob.y[i];
double v = target[i];
//printf("actual y : %f, predicted value %f \n", y, v );
total_error += (v-y)*(v-y);
sumv += v;
sumy += y;
sumvv += v*v;
sumyy += y*y;
sumvy += v*y;
}
//print the error rates
cout<<"Cross Validation Mean squared error = "<<total_error/prob.l<<endl<<flush;
cout<<"Cross Validation Root Mean squared error = "<<sqrt(total_error/prob.l) <<endl<<flush;
cout<<"Cross Validation Squared correlation coefficient = "<<
((prob.l*sumvy-sumv*sumy)*(prob.l*sumvy-sumv*sumy))/
((prob.l*sumvv-sumv*sumv)*(prob.l*sumyy-sumy*sumy))
<<endl<<flush;
}
free(target);
return sqrt(total_error/prob.l);
}
void do_cross_validation(struct svm_problem *prob, struct svm_parameter *param,int nr_fold){
int i;
int total_correct = 0;
double total_error = 0;
double sumv = 0, sumy = 0, sumvv = 0, sumyy = 0, sumvy = 0;
double *target = Malloc(double,prob->l);
svm_cross_validation(prob,param,nr_fold,target);
if(param->svm_type == EPSILON_SVR ||
param->svm_type == NU_SVR)
{
for(i=0;i<prob->l;i++)
{
double y = prob->y[i];
double v = target[i];
total_error += (v-y)*(v-y);
sumv += v;
sumy += y;
sumvv += v*v;
sumyy += y*y;
sumvy += v*y;
}
elog(INFO,"Cross Validation Mean squared error = %g\n",total_error/prob->l);
elog(INFO,"Cross Validation Squared correlation coefficient = %g\n",
((prob->l*sumvy-sumv*sumy)*(prob->l*sumvy-sumv*sumy))/
((prob->l*sumvv-sumv*sumv)*(prob->l*sumyy-sumy*sumy))
);
}
else
{
for(i=0;i<prob->l;i++)
if(target[i] == prob->y[i])
++total_correct;
elog(INFO,"Cross Validation Accuracy = %g%%\n",100.0*total_correct/prob->l);
}
pgm_free(target);
}
std::vector< std::pair<size_t, size_t> > tune_params_single(
const double* divs, size_t num_bags,
const std::vector<label_type> &labels,
const Kernel * const * kernels, size_t num_kernels,
const std::vector<double> &c_vals,
svm_parameter svm_params,
size_t folds,
double *final_score)
{
typedef std::pair<size_t, size_t> config;
// this only works for int or double label_types
BOOST_STATIC_ASSERT((
boost::is_same<label_type, int>::value ||
boost::is_same<label_type, double>::value
));
// keep track of the best kernel/C combos
// keep all with same acc, to avoid bias towards ones we see earlier
std::vector<config> best_configs;
double best_score = -std::numeric_limits<double>::infinity();
if (num_kernels == 0) {
*final_score = best_score;
return best_configs;
}
// make our svm_problem that we'll be reusing throughout
size_t num_train = labels.size();
if (num_train > num_bags) {
throw std::invalid_argument("tune_params: more labels than bags");
}
svm_problem *prob = new svm_problem;
prob->l = num_train;
prob->y = new double[num_train];
for (size_t i = 0; i < num_train; i++)
prob->y[i] = (double) labels[i];
// store un-transformed divs in the problem just so it's all allocated
store_kernel_matrix(*prob, divs, true);
// make a copy of divergences so we can mangle it
double *km = new double[num_bags * num_bags];
double *km_train = (num_train < num_bags) ?
new double[num_train * num_train] : km;
// used to store labels into during CV
double *cv_labels = new double[num_train];
for (size_t k = 0; k < num_kernels; k++) {
// turn into a kernel matrix
std::copy(divs, divs + num_bags * num_bags, km);
kernels[k]->transformDivergences(km, num_bags);
project_to_symmetric_psd(km, num_bags);
// is it a constant matrix or something else awful?
if (terrible_kernel(km, num_bags)) {
FILE_LOG(logDEBUG1) << "tuning: skipping terrible kernel "
<< kernels[k]->name();
continue;
}
// copy the top-left corner into km_train if necessary
if (km_train != km)
copy_upper(km, km_train, num_bags, num_train);
// store in the svm_problem
store_kernel_matrix(*prob, km_train, false);
// TODO: add epsilon-SVR's epsilon (svm_params.p) to the grid search
// (preferably by generalizing this notion of parameter tuning)
for (size_t ci = 0; ci < c_vals.size(); ci++) {
// do SVM cross-validation with these params
svm_params.C = c_vals[ci];
svm_cross_validation(prob, &svm_params, folds, cv_labels);
double score = 0;
if (boost::is_same<label_type, int>::value) {
// integer type; get classification accuracy
for (size_t i = 0; i < num_train; i++)
if (cv_labels[i] == labels[i])
score++;
FILE_LOG(logDEBUG2) << "tuning: " <<
score << "/" << num_train
<< " by " << kernels[k]->name() << ", C = " << c_vals[ci];
} else {
// double type; get *negative* squared err (so max is best)
for (size_t i = 0; i < num_train; i++) {
double diff = cv_labels[i] - labels[i];
score -= diff * diff;
}
FILE_LOG(logDEBUG2) << "tuning: " << score
<< " by " << kernels[k]->name() << ", C = " << c_vals[ci];
}
if (score >= best_score) {
if (score > best_score) {
best_configs.clear();
best_score = score;
}
best_configs.push_back(std::make_pair(k, ci));
}
}
}
if (km_train != km)
delete[] km_train;
delete[] km;
delete[] cv_labels;
for (size_t i = 0; i < num_train; i++)
delete[] prob->x[i];
delete[] prob->x;
delete[] prob->y;
delete prob;
*final_score = best_score;
return best_configs;
}
void SVMTrainModel::do_cross_validation(double &RecRate, std::vector<int> &ConfusionTable)
{
int i;
int total_correct = 0;
double total_error = 0;
double sumv = 0, sumy = 0, sumvv = 0, sumyy = 0, sumvy = 0;
double *target = Malloc(double,prob.l);
svm_cross_validation(&prob,¶m,nr_fold,target);
if(param.svm_type == EPSILON_SVR ||
param.svm_type == NU_SVR)
{
for(i=0;i<prob.l;i++)
{
double y = prob.y[i];
double v = target[i];
total_error += (v-y)*(v-y);
sumv += v;
sumy += y;
sumvv += v*v;
sumyy += y*y;
sumvy += v*y;
}
printf("Cross Validation Mean squared error = %g\n",total_error/prob.l);
printf("Cross Validation Squared correlation coefficient = %g\n",
((prob.l*sumvy-sumv*sumy)*(prob.l*sumvy-sumv*sumy))/
((prob.l*sumvv-sumv*sumv)*(prob.l*sumyy-sumy*sumy))
);
}
else
{
ConfusionTable.at(0) = 0; ConfusionTable.at(1) = 0; ConfusionTable.at(2) = 0; ConfusionTable.at(3) = 0;
for(i=0;i<prob.l;i++)
{
if(target[i] == prob.y[i])
{
++total_correct;
if(target[i] == 0)
{
ConfusionTable.at(0) += 1;
}
else
{
ConfusionTable.at(3) += 1;
}
}
else
{
if(target[i] == 0)
{
ConfusionTable.at(2) += 1;
}
else
{
ConfusionTable.at(1) += 1;
}
}
}
//printf("Cross Validation Accuracy = %g%%\n",100.0*total_correct/prob.l);
RecRate = 100.0*total_correct/prob.l;
}
free(target);
}
void BagOfFeatures::trainSVM(int type = NU_SVC,
int kernel = RBF,
double degree = 0.05,
double gamma = 0.25,
double coef0 = 0.5,
double C = .05,
double cache = 300,
double eps = 0.000001,
double nu = 0.5,
int shrinking = 0,
int probability = 0,
int weight = 0)
{
if(SVMModel != NULL)
{
svm_destroy_model(SVMModel);
//svm_destroy_param(&SVMParam);
}
int i, j, k, l = -1;
int totalData = 0;
int size, length = dictionary->rows;
int count;
//Get the total number of training data
for(i = 0; i < numClasses; i++)
totalData += data[i].getTrainSize();
// Set up the data
struct svm_problem SVMProblem;
SVMProblem.l = totalData;
SVMProblem.y = new double [totalData];
SVMProblem.x = new struct svm_node* [totalData];
// Allocate memory
//for(i = 0; i < totalData; i++)
//{
// SVMProblem.x[i] = new struct svm_node [length+1];
//}
// For each class
for(i = 0; i < numClasses; i++)
{
// Get the number of images
size = data[i].getTrainSize();
for(j = 0; j < size; j++)
{
l++;
count = 0;
for(k = 0; k < length; k++)
{
if(trainObject[i].histogramSet.histogram[j][k] != 0)
count++;
}
SVMProblem.x[l] = new struct svm_node [count+1];
count = 0;
for(k = 0; k < length; k++)
{
if(trainObject[i].histogramSet.histogram[j][k] != 0)
{
SVMProblem.x[l][count].index = k;
SVMProblem.x[l][count].value = trainObject[i].histogramSet.histogram[j][k];
//cout << "(" << SVMProblem.x[l][count].index
// << ", " << SVMProblem.x[l][count].value << ")" << endl;
count++;
}
}
SVMProblem.x[l][count].index = -1;
//cout << endl;
//SVMProblem.x[l][count].value = -1;
// Copy the histograms
//for(k = 0; k < length; k++)
//{
// SVMProblem.x[l][k].index = k;
// SVMProblem.x[l][k].value = trainObject[i].histogramSet.histogram[j][k];
//}
// End of the data
//SVMProblem.x[l][length].index = -1;
//SVMProblem.x[l][length].value = -1;
//Attach the labels
SVMProblem.y[l] = data[i].getLabel();
//cout << "Label: " << SVMProblem.y[l] << endl;
}
}
// Types
SVMParam.svm_type = type;
SVMParam.kernel_type = kernel;
// Parameters
SVMParam.degree = degree;
SVMParam.gamma = gamma;
SVMParam.coef0 = coef0;
SVMParam.C = C;
// For training only
SVMParam.cache_size = cache;
SVMParam.eps = eps;
SVMParam.nu = nu;
SVMParam.shrinking = shrinking;
SVMParam.probability = probability;
// Don't change the weights
SVMParam.nr_weight = weight;
double* target = new double [totalData];
svm_check_parameter(&SVMProblem, &SVMParam);
svm_cross_validation(&SVMProblem, &SVMParam, 10, target);
SVMModel = svm_train(&SVMProblem, &SVMParam);
delete [] target;
classifierType = LIBSVM_CLASSIFIER;
}
bool SVM::trainSVM(){
crossValidationResult = 0;
//Erase any previous models
if( trained ){
svm_free_and_destroy_model(&model);
trained = false;
}
//Check to make sure the problem has been set
if( !problemSet ){
errorLog << "trainSVM() - Problem not set!" << endl;
return false;
}
//Verify the problem and the parameters
if( !validateProblemAndParameters() ) return false;
//Scale the training data if needed
if( useScaling ){
for(int i=0; i<prob.l; i++)
for(UINT j=0; j<numInputDimensions; j++)
prob.x[i][j].value = scale(prob.x[i][j].value,ranges[j].minValue,ranges[j].maxValue,SVM_MIN_SCALE_RANGE,SVM_MAX_SCALE_RANGE);
}
if( useCrossValidation ){
int i;
double total_correct = 0;
double total_error = 0;
double sumv = 0, sumy = 0, sumvv = 0, sumyy = 0, sumvy = 0;
double *target = new double[prob.l];
svm_cross_validation(&prob,¶m,kFoldValue,target);
if( param.svm_type == EPSILON_SVR || param.svm_type == NU_SVR )
{
for(i=0;i<prob.l;i++)
{
double y = prob.y[i];
double v = target[i];
total_error += (v-y)*(v-y);
sumv += v;
sumy += y;
sumvv += v*v;
sumyy += y*y;
sumvy += v*y;
}
crossValidationResult = total_error/prob.l;
}
else
{
for(i=0;i<prob.l;i++){
if(target[i] == prob.y[i]){
++total_correct;
}
}
crossValidationResult = total_correct/prob.l*100.0;
}
delete[] target;
}
//Train the SVM - if we are running cross validation then the CV will be run first followed by a full train
model = svm_train(&prob,¶m);
if( model == NULL ){
errorLog << "trainSVM() - Failed to train SVM Model!" << endl;
return false;
}
if( model != NULL ){
trained = true;
numClasses = getNumClasses();
classLabels.resize( getNumClasses() );
for(UINT k=0; k<getNumClasses(); k++){
classLabels[k] = model->label[k];
}
classLikelihoods.resize(numClasses,DEFAULT_NULL_LIKELIHOOD_VALUE);
classDistances.resize(numClasses,DEFAULT_NULL_DISTANCE_VALUE);
}
return trained;
}
bool CParameterSearch::SearchRange(ParameterResult* pResult, RangeParameters& Params)
{
if(!m_pProblem || !m_pSvmParam || !pResult)
return false;
float fParam1 = 0;
float fParam2 = 0;
double* target = new double[m_pProblem->l];
/* for(int i=0; i<m_pProblem->l; i++)
{
target[i] = 0;
}*/
for(fParam1=Params.fParam1Min; fParam1<=Params.fParam1Max; fParam1+=Params.fParam1Step)
{
if(Params.bParam1UseLog)
m_pSvmParam->p = ::pow(2,fParam1);
else
m_pSvmParam->p = fParam1;
for(fParam2=Params.fParam2Min; fParam2<=Params.fParam2Max; fParam2+=Params.fParam2Step)
{
if(Params.bParam2UseLog)
m_pSvmParam->C = ::pow(2,fParam2);
else
m_pSvmParam->C = fParam2;
int nFolds = 2;
svm_cross_validation(m_pProblem, m_pSvmParam, nFolds, target);
float fError = 0;
float fWrong = 0;
for(int i=0; i<m_pProblem->l; i++)
{
fError += abs(m_pProblem->y[i] - target[i]);
if( m_pProblem->y[i] >= 0.5 && target[i] < 0.5)
fWrong++;
else if( m_pProblem->y[i] < 0.5 && target[i] >= 0.5)
fWrong++;
}
fError = (float)fError/m_pProblem->l;
fWrong = (float) fWrong/m_pProblem->l;
float fStdDev = 0;
for(int i=0; i<m_pProblem->l; i++)
{
fStdDev += pow(fError - abs(m_pProblem->y[i] - target[i]), 2) ;
}
fStdDev = pow(fStdDev, (float)0.5) / m_pProblem->l;
std::cout << "\n****************" << std::endl;
std::cout << "C = 2^" << fParam2 << ", epsilon = 2^" << fParam1 << std::endl;
std::cout << "Avg Error: " << fError << " Std Dev: " << fStdDev << std::endl;
std::cout << "Percent wrong: " << fWrong << std::endl;
ParameterResult* pNewResult = new ParameterResult;
pNewResult->fError = fError;
pNewResult->fStdDev = fStdDev;
pNewResult->fWrong = fWrong;
pNewResult->fParam1 = fParam1;
pNewResult->fParam2 = fParam2;
pNewResult->nLevel = pResult->nLevel + 1;
m_searchResults.insert(pNewResult);
const ParameterResult* pConstResult = pNewResult;
*m_pOA << *pConstResult;
m_pofs->flush();
//SaveTextResults();
}
}
pResult->bRefined = true;
SerializeData();
return true;
}
Transformation SVMTrain::analyze(const DataSet* dataset) const {
G_INFO("Doing SVMTrain analysis...");
QStringList sdescs = selectDescriptors(dataset->layout(), StringType, _descriptorNames, _exclude, false);
if (!sdescs.isEmpty()) {
throw GaiaException("SVMTrain: if you want to use string descriptors for training your SVM, "
"you first need to enumerate them using the 'enumerate' transform on them. "
"String descriptors: ", sdescs.join(", "));
}
QStringList descs = selectDescriptors(dataset->layout(), UndefinedType, _descriptorNames, _exclude);
// sort descriptors in the order in which they are taken inside the libsvm dataset
sort(descs.begin(), descs.end(), DescCompare(dataset->layout()));
Region region = dataset->layout().descriptorLocation(descs);
QStringList classMapping = svm::createClassMapping(dataset, _className);
// first, convert the training data into an SVM problem structure
// NB: all checks about fixed-length and type of descriptors are done in this function
struct svm_problem prob = svm::dataSetToLibsvmProblem(dataset, _className, region, classMapping);
// also get dimension (this trick works because a vector in there is the number
// of dimensions + 1 for the sentinel, and we're not in a sparse representation)
int dimension = prob.x[1] - prob.x[0] - 1;
// default values
struct svm_parameter param;
//param.svm_type = C_SVC;
//param.kernel_type = RBF;
//param.degree = 3;
//param.gamma = 0; // 1/k
param.coef0 = 0;
param.nu = 0.5;
param.cache_size = 100;
//param.C = 1;
param.eps = 1e-3;
param.p = 0.1;
param.shrinking = 1;
//param.probability = 0;
param.nr_weight = 0;
param.weight_label = NULL;
param.weight = NULL;
// get parameters
QString svmType = _params.value("type", "C-SVC").toString().toLower();
param.svm_type = _svmTypeMap.value(svmType);
QString kernelType = _params.value("kernel", "RBF").toString().toLower();
param.kernel_type = _kernelTypeMap.value(kernelType);
param.degree = _params.value("degree", 3).toInt();
param.C = _params.value("c", 1).toDouble();
param.gamma = _params.value("gamma", 1.0/dimension).toDouble();
param.probability = _params.value("probability", false).toBool() ? 1 : 0;
const char* error_msg = svm_check_parameter(&prob, ¶m);
if (error_msg) {
throw GaiaException(error_msg);
}
// do it!
struct svm_model* model;
const bool crossValidation = false;
if (crossValidation) {
int nr_fold = 10;
int total_correct = 0;
double total_error = 0;
double sumv = 0, sumy = 0, sumvv = 0, sumyy = 0, sumvy = 0;
double* target = new double[prob.l];
svm_cross_validation(&prob, ¶m, nr_fold, target);
if (param.svm_type == EPSILON_SVR ||
param.svm_type == NU_SVR) {
for (int i=0; i<prob.l; i++) {
double y = prob.y[i];
double v = target[i];
total_error += (v-y)*(v-y);
sumv += v;
sumy += y;
sumvv += v*v;
sumyy += y*y;
sumvy += v*y;
}
G_INFO("Cross Validation Mean squared error =" << total_error/prob.l);
G_INFO("Cross Validation Squared correlation coefficient =" <<
((prob.l*sumvy - sumv*sumy) * (prob.l*sumvy - sumv*sumy)) /
((prob.l*sumvv - sumv*sumv) * (prob.l*sumyy - sumy*sumy))
);
}
else {
for (int i=0; i<prob.l; i++)
if (target[i] == prob.y[i])
++total_correct;
G_INFO("Cross Validation Accuracy =" << 100.0*total_correct/prob.l << "%");
}
}
else { // !crossValidation
model = svm_train(&prob, ¶m);
}
// save model to a temporary file (only method available from libsvm...),
// reload it and put it into a gaia2::Parameter
QTemporaryFile modelFile;
modelFile.open();
QString modelFilename = modelFile.fileName();
modelFile.close();
if (svm_save_model(modelFilename.toAscii().constData(), model) == -1) {
throw GaiaException("SVMTrain: error while saving SVM model to temp file");
}
modelFile.open();
QByteArray modelData = modelFile.readAll();
modelFile.close();
// if we asked for the model to be output specifically, also do it
if (_params.value("modelFilename", "").toString() != "") {
QString filename = _params.value("modelFilename").toString();
svm_save_model(filename.toAscii().constData(), model);
}
// destroy the model allocated by libsvm
svm_destroy_model(model);
Transformation result(dataset->layout());
result.analyzerName = "svmtrain";
result.analyzerParams = _params;
result.applierName = "svmpredict";
result.params.insert("modelData", modelData);
result.params.insert("className", _params.value("className"));
result.params.insert("descriptorNames", descs);
result.params.insert("classMapping", classMapping);
result.params.insert("probability", (param.probability == 1 && (param.svm_type == C_SVC ||
param.svm_type == NU_SVC)));
return result;
}
void SVMClassifier::train()
{
if(class_data.size() == 0){
printf("SVMClassifier::train() -- No training data available! Doing nothing.\n");
return;
}
int n_classes = class_data.size();
//Count the training data
int n_data = 0;
int dims = class_data.begin()->second[0].size();
for(ClassMap::iterator iter = class_data.begin(); iter != class_data.end(); iter++){
CPointList cpl = iter->second;
if(cpl.size() == 1)
n_data += 2; //There's a bug in libSVM for classes with only 1 data point, so we will duplicate them later
else
n_data += cpl.size();
}
//Allocate space for data in an svm_problem structure
svm_data.l = n_data;
svm_data.y = new double[n_data];
svm_data.x = new svm_node*[n_data];
for(int i=0; i<n_data; i++)
svm_data.x[i] = new svm_node[dims+1];
//Create maps between string labels and int labels
label_str_to_int.clear();
label_int_to_str.clear();
int label_n = 0;
for(ClassMap::iterator iter = class_data.begin(); iter != class_data.end(); iter++){
string cname = iter->first;
label_str_to_int[cname] = label_n;
label_int_to_str[label_n] = cname;
//cout << "MAP: " << label_n << " " << cname << " Size: " << iter->second.size() << endl;
++label_n;
}
//Find the range of the data in each dim and calc the scaling factors to scale from 0 to 1
scaling_factors = new double*[dims];
for(int i=0; i<dims; i++)
scaling_factors[i] = new double[2];
//Scale each dimension separately
for(int j=0; j<dims; j++){
//First find the min, max, and scaling factor
double minval = INFINITY;
double maxval = -INFINITY;
for(ClassMap::iterator iter = class_data.begin(); iter != class_data.end(); iter++){
CPointList cpl = iter->second;
for(size_t i=0; i<cpl.size(); i++){
if(cpl[i][j] < minval)
minval = cpl[i][j];
if(cpl[i][j] > maxval)
maxval = cpl[i][j];
}
}
double factor = maxval-minval;
double offset = minval;
//Do the scaling and save the scaling factor and offset
for(ClassMap::iterator iter = class_data.begin(); iter != class_data.end(); iter++){
for(size_t i=0; i<iter->second.size(); i++){
iter->second[i][j] = (iter->second[i][j] - offset) / factor;
}
}
scaling_factors[j][0] = offset;
scaling_factors[j][1] = factor;
}
//Put the training data into the svm_problem
int n = 0;
for(ClassMap::iterator iter = class_data.begin(); iter != class_data.end(); iter++){
string cname = iter->first;
CPointList cpl = iter->second;
//Account for bug in libSVM with classes with only 1 data point by duplicating it.
if(cpl.size() == 1){
svm_data.y[n] = label_str_to_int[cname];
svm_data.y[n+1] = label_str_to_int[cname];
for(int j=0; j<dims; j++){
svm_data.x[n][j].index = j;
svm_data.x[n][j].value = cpl[0][j] + 0.001;
svm_data.x[n+1][j].index = j;
svm_data.x[n+1][j].value = cpl[0][j] + 0.001;
}
svm_data.x[n][dims].index = -1;
svm_data.x[n+1][dims].index = -1;
n = n + 2;
}
else{
for(size_t i=0; i<cpl.size(); i++){
svm_data.y[n] = label_str_to_int[cname];
for(int j=0; j<dims; j++){
svm_data.x[n][j].index = j;
svm_data.x[n][j].value = cpl[i][j];
}
svm_data.x[n][dims].index = -1;
n = n + 1;
}
}
}
//Set the training params
svm_parameter params;
params.svm_type = C_SVC;
params.kernel_type = RBF;
params.cache_size = 100.0;
params.gamma = 1.0;
params.C = 1.0;
params.eps = 0.001;
params.shrinking = 1;
params.probability = 0;
params.degree = 0;
params.nr_weight = 0;
//params.weight_label =
//params.weight =
const char *err_str = svm_check_parameter(&svm_data, ¶ms);
if(err_str){
printf("SVMClassifier::train() -- Bad SVM parameters!\n");
printf("%s\n",err_str);
return;
}
//Grid Search for best C and gamma params
int n_folds = min(10, n_data); //Make sure there at least as many points as folds
double *resp = new double[n_data];
double best_accy = 0.0;
double best_g = 0.0;
double best_c = 0.0;
//First, do a coarse search
for(double c = -5.0; c <= 15.0; c += 2.0){
for(double g = 3.0; g >= -15.0; g -= 2.0){
params.gamma = pow(2,g);
params.C = pow(2,c);
svm_cross_validation(&svm_data, ¶ms, n_folds, resp);
//Figure out the accuracy using these params
int correct = 0;
for(int i=0; i<n_data; i++){
if(resp[i] == svm_data.y[i])
++correct;
double accy = double(correct) / double(n_data);
if(accy > best_accy){
best_accy = accy;
best_g = params.gamma;
best_c = params.C;
}
}
}
}
//Now do a finer grid search based on coarse results
double start_c = best_c - 1.0;
double end_c = best_c + 1.0;
double start_g = best_g + 1.0;
double end_g = best_g - 1.0;
for(double c = start_c; c <= end_c; c += 0.1){
for(double g = start_g; g >= end_g; g -= 0.1){
params.gamma = pow(2,g);
params.C = pow(2,c);
svm_cross_validation(&svm_data, ¶ms, n_folds, resp);
//Figure out the accuracy using these params
int correct = 0;
for(int i=0; i<n_data; i++){
if(resp[i] == svm_data.y[i])
++correct;
double accy = double(correct) / double(n_data);
if(accy > best_accy){
best_accy = accy;
best_g = params.gamma;
best_c = params.C;
}
}
}
}
// Set params to best found in grid search
params.gamma = best_g;
params.C = best_c;
printf("BEST PARAMS ncl: %i c: %f g: %f accy: %f \n\n", n_classes, best_c, best_g, best_accy);
//Train the SVM
trained_model = svm_train(&svm_data, ¶ms);
}
