double 
SvmSgd::my_evaluateEta(int imin, int imax, const xvec_t &xp, const yvec_t &yp, double eta00)
{
		SvmSgd clone(*this); // take a copy of the current state

		cout << "[my_evaluateEta: clone.wDivisor: ]" << setprecision(12) << clone.wDivisor << " clone.t: " << clone.t << " clone.eta0: " << clone.eta0 << endl; 
		cout << "Trying eta=" << eta00 ;

		assert(imin <= imax);
		double _t = 0;
		double eta = 0;
		for (int i=imin; i<=imax; i++){
				eta = eta00 / (1 + lambda * eta00 * _t);
				//cout << "[my_evaluateEta:] Eta: " << eta << endl;
				clone.trainOne(xp.at(i), yp.at(i), eta);
				_t++;
		}
		double loss = 0;
		double cost = 0;
		for (int i=imin; i<=imax; i++)
				clone.testOne(xp.at(i), yp.at(i), &loss, 0);
		loss = loss / (imax - imin + 1);
		cost = loss + 0.5 * lambda * clone.wnorm();
		cout <<" yields loss " << loss << endl;
		// cout << "Trying eta=" << eta << " yields cost " << cost << endl;
		return cost;
}
Beispiel #2
0
double
SvmAisgd::evaluateEta(int imin, int imax, const xvec_t &xp, const yvec_t &yp, double eta)
{
  SvmAisgd clone(*this); // take a copy of the current state
  assert(imin <= imax);
  for (int i=imin; i<=imax; i++)
    clone.trainOne(xp.at(i), yp.at(i), eta, 1.0);
  double loss = 0;
  double cost = 0;
  for (int i=imin; i<=imax; i++)
    clone.testOne(xp.at(i), yp.at(i), &loss, 0);
  loss = loss / (imax - imin + 1);
  cost = loss + 0.5 * lambda * clone.wnorm();
  // cout << "Trying eta=" << eta << " yields cost " << cost << endl;
  return cost;
}
Beispiel #3
0
static void
loadmult_datafile_sub(istream &f, bool binary, const char *fname, 
                  xvec_t &xp, yvec_t &yp, int &maxdim, int maxrows)
{
  cout << "# Reading file " << fname << endl;
  if (! f.good())
    assertfail("Cannot open " << fname);

  int pcount = 0;
  while (f.good() && maxrows--)
    {
	double y;
      	SVector x;
	y = (f.get());
        x.load(f);
      
      if (f.good())
        {          
          xp.push_back(x);
          yp.push_back(y);
          pcount += 1;
          if (x.size() > maxdim)
            maxdim = x.size();
        }
    }
  cout << "# Read " << pcount << " examples." << endl;
  

}
Beispiel #4
0
void 
SvmSgd::test(int imin, int imax, 
             const xvec_t &xp, const yvec_t &yp, 
             const char *prefix)

{
  cout << prefix << "Testing on [" << imin << ", " << imax << "]." << endl;
  assert(imin <= imax);
  int nerr = 0;
  double cost = 0;
  for (int i=imin; i<=imax; i++)
    {
      const SVector &x = xp.at(i);
      double y = yp.at(i);
      double wx = dot(w,x);
      double z = y * (wx + bias);
      if (z <= 0)
        nerr += 1;
#if LOSS < LOGLOSS
      if (z < 1)
#endif
        cost += loss(z);
    }
  int n = imax - imin + 1;
  double loss = cost / n;
  cost = loss + 0.5 * lambda * dot(w,w);
  cout << prefix << setprecision(4)
       << "Misclassification: " << (double)nerr * 100.0 / n << "%." << endl;
  cout << prefix << setprecision(12) 
       << "Cost: " << cost << "." << endl;
  cout << prefix << setprecision(12) 
       << "Loss: " << loss << "." << endl;
}
Beispiel #5
0
void
load(const char *fname, xvec_t &xp, yvec_t &yp)
{
    cout << "Loading " << fname << "." << endl;

    igzstream f;
    f.open(fname);
    if (! f.good())
    {
        cerr << "ERROR: cannot open " << fname << "." << endl;
        exit(10);
    }
    int pcount = 0;
    int ncount = 0;

    bool binary;
    string suffix = fname;
    if (suffix.size() >= 7)
        suffix = suffix.substr(suffix.size() - 7);
    if (suffix == ".dat.gz")
        binary = false;
    else if (suffix == ".bin.gz")
        binary = true;
    else
    {
        cerr << "ERROR: filename should end with .bin.gz or .dat.gz" << endl;
        exit(10);
    }

    while (f.good())
    {
        SVector x;
        double y;
        if (binary)
        {
            y = (f.get()) ? +1 : -1;
            x.load(f);
        }
        else
        {
            f >> y >> x;
        }
        if (f.good())
        {
            assert(y == +1 || y == -1);
            xp.push_back(x);
            yp.push_back(y);
            if (y > 0)
                pcount += 1;
            else
                ncount += 1;
            if (x.size() > dim)
                dim = x.size();
        }
    }
    cout << "Read " << pcount << "+" << ncount
         << "=" << pcount + ncount << " examples." << endl;
}
Beispiel #6
0
int main(int argc, const char **argv)
{
  parse(argc, argv);
  config(argv[0]);
  if (trainfile)
    load_datafile(trainfile, xtrain, ytrain, dims, normalize, maxtrain);
  if (testfile)
    load_datafile(testfile, xtest, ytest, dims, normalize);
  cout << "# Number of features " << dims << "." << endl;
  // prepare svm
  int imin = 0;
  int imax = xtrain.size() - 1;
  int tmin = 0;
  int tmax = xtest.size() - 1;
  // heuristic determination of averaging start point
  int avgfrom = fabs(avgstart) * (imax - imin + 1);
  avgfrom = (avgstart < 0 && dims < avgfrom) ? dims : avgfrom;
  // create
  SvmAisgd svm(dims, lambda, avgfrom);
  Timer timer;
  // determine eta0 using sample
  int smin = 0;
  int smax = imin + min(1000, imax);
  timer.start();
  svm.determineEta0(smin, smax, xtrain, ytrain);
  timer.stop();
  // train
  for(int i=0; i<epochs; i++)
    {
      cout << "--------- Epoch " << i+1 << "." << endl;
      timer.start();
      svm.train(imin, imax, xtrain, ytrain);
      timer.stop();
      cout << "Total training time " << setprecision(6)
           << timer.elapsed() << " secs." << endl;
      svm.test(imin, imax, xtrain, ytrain, "train: ");
      if (tmax >= tmin)
        svm.test(tmin, tmax, xtest, ytest, "test:  ");
    }
  svm.renorm();
  // Linear classifier is in svm.a and svm.aBias
  return 0;
}
Beispiel #7
0
int main(int argc, const char **argv)
{
  parse(argc, argv);
  config(argv[0]);
  if (trainfile)
    load_datafile(trainfile, xtrain, ytrain, dims, normalize, maxtrain);
  if (testfile)
    load_datafile(testfile, xtest, ytest, dims, normalize);
  cout << "# Number of features " << dims << "." << endl;
  // prepare svm
  int imin = 0;
  int imax = xtrain.size() - 1;
  int tmin = 0;
  int tmax = xtest.size() - 1;
  SvmSag svm(dims, lambda);
  Timer timer;
  // determine eta0 using sample
  int smin = 0;
  int smax = imin + min(1000, imax);
  timer.start();
  if (eta > 0)
    svm.setEta(eta);
  else
    svm.determineEta(smin, smax, xtrain, ytrain);
  timer.stop();
  // train
  for(int i=0; i<epochs; i++)
    {
      cout << "--------- Epoch " << i+1 << "." << endl;
      timer.start();
      if (i == 0)
        svm.trainInit(imin, imax, xtrain, ytrain);
      else
        svm.trainSag(imin, imax, xtrain, ytrain);
      timer.stop();
      cout << "Total training time " << setprecision(6) 
           << timer.elapsed() << " secs." << endl;
      svm.test(imin, imax, xtrain, ytrain, "train: ");
      if (tmax >= tmin)
        svm.test(tmin, tmax, xtest, ytest, "test:  ");
    }
  return 0;
}
Beispiel #8
0
int 
main(int argc, const char **argv)
{
  parse(argc, argv);
  cout << "Loss=" << lossname 
       << " Bias=" << BIAS 
       << " RegBias=" << REGULARIZEBIAS
       << " Lambda=" << lambda
       << endl;

  // load training set
  load(trainfile.c_str(), xtrain, ytrain);
  cout << "Number of features " << dim << "." << endl;
  int imin = 0;
  int imax = xtrain.size() - 1;
  if (trainsize > 0 && imax >= trainsize)
    imax = imin + trainsize -1;
  // prepare svm
  SvmSgd svm(dim, lambda);
  Timer timer;

  // load testing set
  if (! testfile.empty())
    load(testfile.c_str(), xtest, ytest);
  int tmin = 0;
  int tmax = xtest.size() - 1;

  svm.calibrate(imin, imax, xtrain, ytrain);
  for(int i=0; i<epochs; i++)
    {
      
      cout << "--------- Epoch " << i+1 << "." << endl;
      timer.start();
      svm.train(imin, imax, xtrain, ytrain);
      timer.stop();
      cout << "Total training time " << setprecision(6) 
           << timer.elapsed() << " secs." << endl;
      svm.test(imin, imax, xtrain, ytrain, "train: ");
      if (tmax >= tmin)
        svm.test(tmin, tmax, xtest, ytest, "test:  ");
    }
}
Beispiel #9
0
/// Perform a test pass
void
SvmAisgd::test(int imin, int imax, const xvec_t &xp, const yvec_t &yp, const char *prefix)
{
  cout << prefix << "Testing on [" << imin << ", " << imax << "]." << endl;
  assert(imin <= imax);
  double nerr = 0;
  double loss = 0;
  for (int i=imin; i<=imax; i++)
    testOne(xp.at(i), yp.at(i), &loss, &nerr);
  nerr = nerr / (imax - imin + 1);
  loss = loss / (imax - imin + 1);
  double cost = loss + 0.5 * lambda * anorm();
  cout << prefix
       << "Loss=" << setprecision(12) << loss
       << " Cost=" << setprecision(12) << cost
       << " Misclassification=" << setprecision(4) << 100 * nerr << "%."
       << endl;
}
Beispiel #10
0
/// Perform a training epoch
void
SvmAisgd::train(int imin, int imax, const xvec_t &xp, const yvec_t &yp, const char *prefix)
{
  cout << prefix << "Training on [" << imin << ", " << imax << "]." << endl;
  assert(imin <= imax);
  assert(eta0 > 0);
  for (int i=imin; i<=imax; i++)
    {
      double eta = eta0 / pow(1 + lambda * eta0 * t, 0.75);
      double mu = (t <= tstart) ? 1.0 : mu0 / (1 + mu0 * (t - tstart));
      trainOne(xp.at(i), yp.at(i), eta, mu);
      t += 1;
    }
  cout << prefix << setprecision(6) << "wNorm=" << wnorm() << " aNorm=" << anorm();
#if BIAS
  cout << " wBias=" << wBias << " aBias=" << aBias;
#endif
  cout << endl;
}
Beispiel #11
0
void 
SvmSgd::train(int imin, int imax, 
              const xvec_t &xp, const yvec_t &yp,
              const char *prefix)
{
  cout << prefix << "Training on [" << imin << ", " << imax << "]." << endl;
  assert(imin <= imax);
  count = skip;
  for (int i=imin; i<=imax; i++)
    {
      const SVector &x = xp.at(i);
      double y = yp.at(i);
      double wx = dot(w,x);
      double z = y * (wx + bias);
      double eta = 1.0 / (lambda * t);
#if LOSS < LOGLOSS
      if (z < 1)
#endif
        {
          double etd = eta * dloss(z);
          w.add(x, etd * y);
#if BIAS
#if REGULARIZEBIAS
          bias *= 1 - eta * lambda * bscale;
#endif
          bias += etd * y * bscale;
#endif
        }
      if (--count <= 0)
        {
          double r = 1 - eta * lambda * skip;
          if (r < 0.8)
            r = pow(1 - eta * lambda, skip);
          w.scale(r);
          count = skip;
        }
      t += 1;
    }
  cout << prefix << setprecision(6) 
       << "Norm: " << dot(w,w) << ", Bias: " << bias << endl;
}
Beispiel #12
0
/// Perform a SAG training epoch
void 
SvmSag::trainSag(int imin, int imax, const xvec_t &xp, const yvec_t &yp, const char *prefix)
{
  cout << prefix << "Training on [" << imin << ", " << imax << "]." << endl;
  assert(imin <= imax);
  assert(imin >= sdimin);
  assert(imax <= sdimax);
  assert(eta > 0);
  uniform_int_generator generator(imin, imax);
  for (int i=imin; i<=imax; i++)
    {
      int ii = generator(); 
      trainOne(xp.at(ii), yp.at(ii), eta, ii);
      t += 1;
    }
  cout << prefix << setprecision(6) << "wNorm=" << wnorm();
#if BIAS
  cout << " wBias=" << wBias;
#endif
  cout << endl;
}
Beispiel #13
0
/// Perform a training epoch
void
SvmSgd::train(int imin, int imax, const xvec_t &xp, const yvec_t &yp, const char *prefix)
{
#if VERBOSE
  cout << prefix << "Training on [" << imin << ", " << imax << "]." << endl;
#endif
  assert(imin <= imax);
  assert(eta0 > 0);
  for (int i=imin; i<=imax; i++)
    {
      double eta = eta0 / (1 + lambda * eta0 * t);
      trainOne(xp.at(i), yp.at(i), eta);
      t += 1;
    }
#if VERBOSE
  cout << prefix << setprecision(6) << "wNorm=" << wnorm();
#if BIAS
  cout << " wBias=" << wBias;
#endif
  cout << endl;
#endif
}
Beispiel #14
0
void
SvmSgd::train(int imin, int imax,
              const xvec_t &xp, const yvec_t &yp,
              const char *prefix)
{
    cout << prefix << "Training on [" << imin << ", " << imax << "]." << endl;
    assert(imin <= imax);
    for (int i=imin; i<=imax; i++)
    {
        double eta = 1.0 / (lambda * t);
        double s = 1 - eta * lambda;
        wscale *= s;
        if (wscale < 1e-9)
        {
            w.scale(wscale);
            wscale = 1;
        }
        const SVector &x = xp.at(i);
        double y = yp.at(i);
        double wx = dot(w,x) * wscale;
        double z = y * (wx + bias);
#if LOSS < LOGLOSS
        if (z < 1)
#endif
        {
            double etd = eta * dloss(z);
            w.add(x, etd * y / wscale);
#if BIAS
            // Slower rate on the bias because
            // it learns at each iteration.
            bias += etd * y * 0.01;
#endif
        }
        t += 1;
    }
    double wnorm =  dot(w,w) * wscale * wscale;
    cout << prefix << setprecision(6)
         << "Norm: " << wnorm << ", Bias: " << bias << endl;
}
Beispiel #15
0
/// Perform initial training epoch
void 
SvmSag::trainInit(int imin, int imax, const xvec_t &xp, const yvec_t &yp, const char *prefix)
{
  cout << prefix << "Training on [" << imin << ", " << imax << "]." << endl;
  assert(imin <= imax);
  assert(eta > 0);
  assert(m == 0);
  sd.resize(imax - imin + 1);
  sdimin = imin;
  sdimax = imax;
  for (int i=imin; i<=imax; i++)
    {
      m += 1;
      trainOne(xp.at(i), yp.at(i), eta, i);
      t += 1;
    }
  cout << prefix << setprecision(6) << "wNorm=" << wnorm();
#if BIAS
  cout << " wBias=" << wBias;
#endif
  cout << endl;
}
/// Perform a training epoch
		void 
SvmSgd::train(int imin, int imax, const xvec_t &xp, const yvec_t &yp, const char *prefix)
{
		cout << prefix << "Training on [" << imin << ", " << imax << "]." << endl;
		assert(imin <= imax);
		assert(eta0 > 0);

		//cout << "wDivisor: " << wDivisor << "  wBias: " << wBias<< endl;
		for (int i=imin; i<=imax; i++)
		{
				double eta = eta0 / (1 + lambda * eta0 * t);
				//cout << "[my_evaluateEta:] Eta: " << eta << endl;
				trainOne(xp.at(i), yp.at(i), eta);
				t += 1;
		}

		//cout << "\nAfter training: \n  wDivisor: " << wDivisor << "  wBias: " << wBias<< endl;
		cout << prefix << setprecision(6) << "wNorm=" << wnorm();
#if BIAS
		cout << " wBias=" << wBias;
#endif
		cout << endl;
}
Beispiel #17
0
void 
SvmSgd::calibrate(int imin, int imax, 
                const xvec_t &xp, const yvec_t &yp)
{
  cout << "Estimating sparsity and bscale." << endl;
  int j;

  // compute average gradient size
  double n = 0;
  double m = 0;
  double r = 0;
  FVector c(w.size());
  for (j=imin; j<=imax && m<=1000; j++,n++)
    {
      const SVector &x = xp.at(j);
      n += 1;
      r += x.npairs();
      const SVector::Pair *p = x;
      while (p->i >= 0 && p->i < c.size())
        {
          double z = c.get(p->i) + fabs(p->v);
          c.set(p->i, z);
          m = max(m, z);
          p += 1;
        }
    }

  // bias update scaling
  bscale = m/n;

  // compute weight decay skip
  skip = (int) ((8 * n * w.size()) / r);
  cout << " using " << n << " examples." << endl;
  cout << " skip: " << skip 
       << " bscale: " << setprecision(6) << bscale << endl;
}
int main(int argc, const char **argv)
{
		parse(argc, argv);
		config(argv[0]);
		if (trainfile)
				load_datafile(trainfile, xtrain, ytrain, dims, normalize, maxtrain);
		if (testfile)
				load_datafile(testfile, xtest, ytest, dims, normalize);
		cout << "# Number of features " << dims << "." << endl;
		// prepare svm
		int imin = 0;
		int imax = xtrain.size() - 1;
		int tmin = 0;
		int tmax = xtest.size() - 1;
		SvmSgd svm(dims, lambda);
		Timer timer;
		// determine eta0 using sample
		int smin = 0;
		int smax = imin + min(1000, imax);
		// train

		Timer totalTimer, overheadtimer, exetimer;

		totalTimer.start();
		//winnie, initial wDivisor and wBias

		timeval t1, t4, t5, t6, t7, t8;
		overheadtimer.start();
		if(sample_file){
				int sample_size = 500;
				int bin_num = 20;
				int num_compare = 49;
				int dimension = dims - 1; //The first feature is the classification, does not count in sampling.


				gettimeofday(&t1, NULL);

				Sampling<double> selector(imax, 0 ,
								dimension, sample_size, bin_num, num_compare);

				selector.do_sampling(sample_file);
				gettimeofday(&t4, NULL);
				selector.calc_ecdf();		
				gettimeofday(&t5, NULL);
				//	std::cout << "test init kmeans " << std::endl;
				//step3a, do database search
				if(num_compare <= 0){
						cerr << "Number of comparison is less than or equal to 0" << endl;
						return -1;
				}
				else{
						//TODO: make the comparison choose the second best result, when we use data base that contains dataset itself

						//winnie, prerun several iterations, and log some information
						int prerun_iters = 3;
						FVector old_w(dimension);

						svm.determineEta0(smin, smax, xtrain, ytrain);

						for(int i = 0; i < prerun_iters; i ++ ){

								svm.get_w(old_w);
								svm.train(imin, imax, xtrain, ytrain);
								//svm.test(imin, imax, xtrain, ytrain, "train: ");

						}
						//get the idx of dimensions	

						map <double, int> delta_w;
						svm.get_delta_w(old_w, delta_w);
						//int reducedDimNum[8] = {1, 2, 3, 4, 8, 16, 32, 50};
						int reducedDimNum[8] = {1, 3, 8, 16, 32, 50, 64, 128};
						int selected_id[9];
						multimap <double, int> error_dim;  //error value and dimension
						for(int iout = 0; iout < 8; iout++){

								vector<int> reducedDimIdx(reducedDimNum[iout]);
								map<double, int>::reverse_iterator rmit = delta_w.rbegin();
								double sum_value = 0;
								for(int i = 0; i < reducedDimNum[iout]; i ++){
										reducedDimIdx[i] = rmit->second;
										sum_value += rmit->first;
										//cout << "reduceDim: " << reducedDimIdx[i] << " value " << rmit->first << endl;
										rmit++;
								}
								cout << "sum_value: " << sum_value << " with dim num: " << reducedDimNum[iout] << endl;


								//cout << "distancetype: " << distancetype << endl;
								selected_id[iout] = selector.search_database('b', database_file, distancetype, reducedDimIdx);

								gettimeofday(&t6, NULL);
								if(selected_id[iout] < 0)
										return -1;
								else{
										std::cout << "selected id: " << selected_id[iout] << std::endl;
										//Do data customization, use norm.cpp program, 
										stringstream ss;//create a stringstream
										ss << setfill('0') << setw(2) << selected_id[iout];
										string filename = string(database_dir) + "/" + ss.str() + ".txt";
										//						double value1, value2;
										//						selector.load_memo(filename.c_str(), &value1, &value2);
										cout << "Init with file: " << filename << endl;
										svm.init_w(filename.c_str(), dims);
										gettimeofday(&t7, NULL);

										//run one iter use current w

										double pre_loss_1; //, pre_loss_2;
										//double pre_loss_delta_1; //, pre_loss_delta_2; 
										//svm.train(imin, imax, xtrain, ytrain);
										svm.test(tmin, tmax, xtest, ytest, "test:  ");
										pre_loss_1 = svm.get_cost();
										/*
										   svm.train(imin, imax, xtrain, ytrain);
										   svm.test(tmin, tmax, xtest, ytest, "test:  ");
										   pre_loss_2 = svm.get_cost();

										   pre_loss_delta_1 = pre_loss_1 - pre_loss_2; */
										double pre_loss_delta_ratio = pre_loss_1;  //method A
										//double pre_loss_delta_ratio = pre_loss_delta_1; //method B
										//double pre_loss_delta_ratio = -svm.t; //method C
										cout << "-svm.t: " << -svm.t << endl; 

										//svm.train(imin, imax, xtrain, ytrain);
										//svm.test(tmin, tmax, xtest, ytest, "test:  ");
										//pre_loss_3 = svm.get_cost();
										//pre_loss_delta_2 = pre_loss_2 - pre_loss_3;

										/*
										   double pre_loss_delta_ratio;
										   if(pre_loss_delta_1 < 0 || pre_loss_delta_2 < 0){
										   pre_loss_delta_ratio = 1000;
										   }
										   else
										   pre_loss_delta_ratio = predict_iter(pre_loss_delta_1, pre_loss_delta_2, pre_loss_2);
										   cout << "pre_loss_1: " << pre_loss_1 << endl;
										   cout << "pre_loss_2: " << pre_loss_2 << endl;
										   cout << "pre_loss_3: " << pre_loss_3 << endl;
										   cout << "pre_loss_delta_1: " << pre_loss_delta_1 << endl;
										   cout << "pre_loss_delta_2: " << pre_loss_delta_2 << endl;
										   */
										cout << "insert pre_loss_ratio " << pre_loss_delta_ratio << " at dim " << reducedDimNum[iout] << endl;
										error_dim.insert(pair<double, int>(pre_loss_delta_ratio, iout));
										//error_dim.insert(pair<double, int>(fabs(pre_loss_new - pre_loss_old), iout));


								}



						}
						stringstream ss;//create a stringstream
						ss << setfill('0') << setw(2) << selected_id[error_dim.begin()->second];
						string filename = string(database_dir) + "/" + ss.str() + ".txt";
						cout << "Init with: " << filename << endl;

						svm.init_w(filename.c_str(), dims);

						cout << "chosen dim num: " << reducedDimNum[error_dim.begin()->second] << " id: " << selected_id[error_dim.begin()->second] << endl;
						cout << "error of each dim_reduction: " << endl;
						cout << error_dim.size() << "in total\n" ;

						for(map<double, int>::iterator mit = error_dim.begin(); mit != error_dim.end(); mit++){
								cout << mit->second << " " << mit->first << endl;
						}

						//winnie, end of prerun


				}



		}
		else{
				if(!initpara_file){
						cerr << "initpara_file does not exist\n";	
						
				}
				else
						svm.init_w(initpara_file, dims);
		}

		overheadtimer.stop();

		//winnie, end of initial wDivisor and wBias
		exetimer.start();
		timer.start();
		//double temp_t = 1;

		//Original version
		//svm.determineEta0(imin, imax, xtrain, ytrain);
		//svm.determineEta0(smin, smax, xtrain, ytrain);
		//svm.t = 0;


		//Version 3 --------------------------------------------------------
		svm.my_determineEta0_v3(imin, imax, xtrain, ytrain);
		//svm.my_determineEta0_v3(smin, smax, xtrain, ytrain);
		//svm.t = 0;
		//svm.t = 500000;

		//Version 2 --------------------------------------------------------
		//svm.my_determineEta0_v2(smin, smax, xtrain, ytrain, svm.t, 4);
		//svm.my_determineEta0_v2(imin, imax, xtrain, ytrain, svm.t, 4);


		//svm.my_determineEta0(imin, imax, xtrain, ytrain, svm.t);

		timer.stop();


		//svm.eta0 = svm.eta0 / 10;
		//svm.t = 0;
		cout << "New eta0: " << svm.eta0 << " t: " << svm.t << endl;
		cout << "eta: " << setprecision(12) << svm.eta0 / (1 + lambda * svm.eta0 * svm.t) << endl;

		//winnie, modify code end condition from number of epochs to misclassification rate
		double loss_new = 1, loss_old = 1;

		int i = 0;
		//  for(int i=0; i<epochs; i++)
		if(revalue == 0){
				svm.test(imin, imax, xtrain, ytrain);
				loss_new = svm.get_cost();

				while(loss_old - loss_new > 1e-5 || loss_new - loss_old > 1e-5)
				{

						cout << "--------- Epoch " << ++i << endl;

						//timer.start();
						svm.train(imin, imax, xtrain, ytrain);
						//timer.stop();
						//cout << "Total training time " << setprecision(6) 
						//		<< timer.elapsed() << " secs." << endl;
						svm.test(imin, imax, xtrain, ytrain, "train: ");

						if (tmax >= tmin)
								svm.test(tmin, tmax, xtest, ytest, "test:  ");

						loss_old = loss_new;
						loss_new = svm.get_cost();
				}	

		}
		else{
				cout<< "[Main: svm.wDivisor: ]" << setprecision(12) <<svm.wDivisor << " svm.t: " << svm.t << "svm.eta0: " << svm.eta0 << endl;
				while(loss_new > revalue)
				{

						cout << "--------- Epoch " << ++i << endl;

						//timer.start();
						svm.train(imin, imax, xtrain, ytrain);
						//timer.stop();
						//cout << "Total training time " << setprecision(6) 
						//		<< timer.elapsed() << " secs." << endl;
						svm.test(imin, imax, xtrain, ytrain, "train: ");

						loss_old = loss_new;
						loss_new = svm.get_cost();
						if (tmax >= tmin)
								svm.test(tmin, tmax, xtest, ytest, "test:  ");

						if(i >= 200){
								cout << "Exit because reach 2 epochs\n";	
								break;
						}
				}

		}

		cout << "Loss_new: " << loss_new << " Loss_old:" << loss_old << endl;

		exetimer.stop();
		totalTimer.stop();

		cout << "Total time: " << setprecision(6)
				<< totalTimer.elapsed() << " secs." << endl;

		cout << "Total exe time: " << setprecision(6)
				<< exetimer.elapsed() << " secs." << endl;

		cout << "Total overhead time: " << setprecision(6)
				<< overheadtimer.elapsed() << " secs." << endl;
		if(sample_file){


				double elapsedTime;
				elapsedTime =(t4.tv_sec - t1.tv_sec) * 1000.0 + (t4.tv_usec - t1.tv_usec) / 1000.0;   // sec and us to ms
				std::cout <<"Overhead s1: "<< elapsedTime << " ms.\n";

				elapsedTime =(t5.tv_sec - t4.tv_sec) * 1000.0 + (t5.tv_usec - t4.tv_usec) / 1000.0;   // sec and us to ms
				std::cout <<"Overhead s2: "<< elapsedTime << " ms.\n";

				elapsedTime =(t6.tv_sec - t5.tv_sec) * 1000.0 + (t6.tv_usec - t5.tv_usec) / 1000.0;   // sec and us to ms
				std::cout <<"Overhead s3: "<< elapsedTime << " ms.\n";

				elapsedTime =(t7.tv_sec - t6.tv_sec) * 1000.0 + (t7.tv_usec - t6.tv_usec) / 1000.0;   // sec and us to ms
				std::cout <<"Overhead s4: "<< elapsedTime << " ms.\n";

				elapsedTime =(t8.tv_sec - t7.tv_sec) * 1000.0 + (t8.tv_usec - t7.tv_usec) / 1000.0;   // sec and us to ms
				std::cout <<"Overhead s5: "<< elapsedTime << " ms.\n";


		}
		//winnie, output parameter wDivisor and wBias:
		cout << "wDivisor: " << svm.get_wDivisor() << endl;
		cout << "wBias: " << svm.get_wBias() << endl;

		svm.output_w(outputfile);

		return 0;
}