TEST(Services, do_bfgs_optimize__bfgs) {
typedef stan::optimization::BFGSLineSearch<Model,stan::optimization::BFGSUpdate_HInv<> > Optimizer_BFGS;
std::vector<double> cont_vector(2);
cont_vector[0] = -1; cont_vector[1] = 1;
std::vector<int> disc_vector;
static const std::string DATA("");
std::stringstream data_stream(DATA);
stan::io::dump dummy_context(data_stream);
Model model(dummy_context);
Optimizer_BFGS bfgs(model, cont_vector, disc_vector, &std::cout);
double lp = 0;
bool save_iterations = true;
int refresh = 0;
int return_code;
unsigned int random_seed = 0;
rng_t base_rng(random_seed);
std::fstream* output_stream = 0;
mock_callback callback;
return_code = stan::services::optimization::do_bfgs_optimize(model,bfgs, base_rng,
lp, cont_vector, disc_vector,
output_stream, &std::cout,
save_iterations, refresh,
callback);
EXPECT_FLOAT_EQ(return_code, 0);
EXPECT_EQ(33, callback.n);
}
void main()
{
double eps, fopt, x[3], xm[3], g[3];
int itmax = 50, n = 3, it;
double f();
void df();
eps = 1e-7;
x[0] = x[1] = x[2] = 0.0;
it = bfgs(x, xm, n, &fopt, f, df, eps, itmax);
printf("iterations = %d:\nx0=%f,x1=%f,x2=%f\nfopt=%f\n", it, xm[0], xm[1],
xm[2], fopt);
df(xm, g, 3);
printf("df[0]=%f, df[1]=%f, df[2]=%f\n", g[0], g[1], g[2]);
}
//////////////////////////////////////////////////////////////////////
// worker thread
void PredictaEngine::loop()
{
setStatus("Waiting..");
thread_idle = true;
while(running){
while(!optimize && running){
thread_idle = true;
setStatus("Waiting..");
usleep(100000); // 100ms (waits for action
train.clear();
scoring.clear();
results.clear();
}
if(!running) continue;
thread_idle = false;
time_t executionStartedTime = time(0);
train.clear();
scoring.clear();
results.clear();
//////////////////////////////////////////////////////////////////////////
// loads at most 100.000 = 100k lines of data to memory
setStatus("Loading data (examples)..");
if(train.importAscii(trainingFile, 100000) == false){
std::string error = "Cannot load file: " + trainingFile;
setError(error);
optimize = false;
continue;
}
setStatus("Loading data (to be scored data)..");
if(scoring.importAscii(scoringFile, 100000) == false){
std::string error = "Cannot load file: " + scoringFile;
train.clear();
scoring.clear();
setError(error);
optimize = false;
continue;
}
setStatus("Checking data validity..");
train.removeBadData();
scoring.removeBadData();
// if number of data points in training is smaller than 2*dim(input)
// then the optimizer fails
if(train.getNumberOfClusters() < 0 || scoring.getNumberOfClusters() < 0){
setError("No data in input files");
optimize = false;
continue;
}
if(train.size(0) < 10){
setError("Not enough data (at least 10 examples) in input file");
optimize = false;
continue;
}
if(train.size(0) < 2*train.dimension(0)){
setError("Not enough data (at least 2*DIMENSION examples) in input file");
optimize = false;
continue;
}
if(train.dimension(0) != (scoring.dimension(0) + 1)){
setError("Incorrect dimensions in training or scoring files");
optimize = false;
continue;
}
if(train.dimension(0) < 2){
setError("Incorrect dimensions in training or scoring files");
optimize = false;
continue;
}
//////////////////////////////////////////////////////////////////////////
// preprocess data using PCA (if PCA cannot be calculated the whole process fails)
// separates training data into input and output clusters
whiteice::dataset< whiteice::math::blas_real<double> > tmp;
if(tmp.createCluster("input", train.dimension(0)-1) == false ||
tmp.createCluster("output", 1) == false){
setError("Internal software error");
optimize = false;
continue;
}
for(unsigned int i=0;i<train.size(0);i++){
whiteice::math::vertex< whiteice::math::blas_real<double> > t = train.access(0, i);
whiteice::math::vertex< whiteice::math::blas_real<double> > a;
whiteice::math::vertex< whiteice::math::blas_real<double> > b;
a.resize(t.size()-1);
b.resize(1);
b[0] = t[t.size()-1];
for(unsigned int j=0;j<(t.size()-1);j++)
a[j] = t[j];
if(tmp.add(0, a) == false || tmp.add(1, b) == false){
setError("Internal software error");
optimize = false;
continue;
}
}
if(optimize == false)
continue; // abort computations
train = tmp;
setStatus("Preprocessing data..");
if(train.preprocess(0) == false /*|| train.preprocess(1) == false*/){
setError("Bad/singular data please add more variance to data");
optimize = false;
continue;
}
//////////////////////////////////////////////////////////////////////////
// optimize neural network using LBFGS (ML starting point for HMC sampler)
std::vector<unsigned int> arch; // use double layer wide nnetwork
arch.push_back(train.dimension(0));
// arch.push_back(100*train.dimension(0));
arch.push_back(train.dimension(0) < 10 ? 10 : train.dimension(0));
arch.push_back(train.dimension(0) < 10 ? 10 : train.dimension(0));
arch.push_back(train.dimension(1));
whiteice::nnetwork< whiteice::math::blas_real<double> > nn(arch);
whiteice::LBFGS_nnetwork< whiteice::math::blas_real<double> > bfgs(nn, train, false, false);
whiteice::math::vertex< whiteice::math::blas_real<double> > w;
nn.randomize();
#if 0
// deep pretraining is disabled as a default
setStatus("Preoptimizing solution (deep learning)..");
if(deep_pretrain_nnetwork(&nn, train, true) == false){
setError("ERROR: deep pretraining of nnetwork failed.\n");
optimize = false;
continue;
}
#endif
nn.exportdata(w);
bfgs.minimize(w);
time_t t0 = time(0);
unsigned int iterations = 0;
whiteice::math::blas_real<double> error;
while(optimize && bfgs.solutionConverged() == false && bfgs.isRunning() == true){
if(optimize == false){ // we lost license to do this anymore..
setStatus("Aborting optimization");
break;
}
time_t t1 = time(0);
unsigned int counter = (unsigned int)(t1 - t0); // time-elapsed
if(bfgs.getSolution(w, error, iterations) == false){ // we lost license to continue..
setStatus("Aborting optimization");
setError("LBFGS::getSolution() failed");
optimize = false;
break;
}
char buffer[128];
snprintf(buffer, 128, "Preoptimizing solution (%d iterations, %.2f minutes): %f",
iterations, counter/60.0f, error.c[0]);
setStatus(buffer);
// update results only every 5 seconds
sleep(5);
}
if(optimize == false){
bfgs.stopComputation();
continue; // abort computation
}
// after convergence, get the best solution
if(bfgs.getSolution(w, error, iterations) == false){ // we lost license to continue..
setStatus("Aborting optimization");
setError("LBFGS::getSolution() failed");
optimize = false;
continue;
}
nn.importdata(w);
//////////////////////////////////////////////////////////////////////////
// use HMC to sample from max likelihood in order to get MAP
setStatus("Analyzing uncertainty..");
// whiteice::UHMC< whiteice::math::blas_real<double> > hmc(nn, train, true);
whiteice::HMC< whiteice::math::blas_real<double> > hmc(nn, train, true);
// whiteice::linear_ETA<float> eta;
// for high quality..
// we use just 50 samples
// const unsigned int NUMSAMPLES = 1000;
// eta.start(0.0f, NUMSAMPLES);
if(hmc.startSampler() == false){
setStatus("Starting sampler failed (internal error)");
setError("Cannot start sampler");
optimize = false;
continue;
}
// unsigned int samples = 0;
t0 = time(0);
// always analyzes results for given time length
double timeElapsed = (time(0) - executionStartedTime);
double totalTime = 0;
if(timeElapsed < optimizationTime)
totalTime = optimizationTime - timeElapsed;
while(optimize){
unsigned int samples = hmc.getNumberOfSamples();
// if(samples >= NUMSAMPLES) break;
// eta.update((float)hmc.getNumberOfSamples());
time_t t1 = time(0);
double counter = (double)(t1 - t0); // time-elapsed
double timeLeft = (totalTime - counter)/60.0;
if(timeLeft <= 0.0){
timeLeft = 0.0;
if(hmc.getNumberOfSamples() > 0)
break; // always gets a single sample from HMC
}
char buffer[128];
snprintf(buffer, 128,
"Analyzing uncertainty (%d iterations. %.2f%% error. ETA %.2f minutes)",
// 100.0*((double)samples/((double)NUMSAMPLES)),
samples,
100.0*hmc.getMeanError(1).c[0]/error.c[0],
timeLeft);
// eta.estimate()/60.0);
setStatus(buffer);
if(optimize == false){ // we lost license to continue..
setStatus("Uncertainty analysis aborted");
break;
}
// updates only every 5 seconds so that we do not take too much resources
sleep(5);
}
if(optimize == false)
continue; // abort computation
hmc.stopSampler();
//////////////////////////////////////////////////////////////////////////
// estimate mean and variance of output given inputs in 'scoring'
setStatus("Calculating scoring..");
whiteice::bayesian_nnetwork< whiteice::math::blas_real<double> > bnn;
if(hmc.getNetwork(bnn) == false){
setStatus("Exporting prediction model failed");
setError("Internal software error");
optimize = false;
continue;
}
if(results.createCluster("results", 1) == false){
setError("Internal software error");
optimize = false;
continue;
}
unsigned int NUM = scoring.size(0);
// demo version only scores 10 first examples given in a file.
if(demoVersion){
if(NUM > 10) NUM = 10;
}
for(unsigned int i=0;i<NUM;i++){
char buffer[128];
snprintf(buffer, 128, "Scoring data (%.1f%%)..",
100.0*((double)i)/((double)scoring.size(0)));
setStatus(buffer);
whiteice::math::vertex< whiteice::math::blas_real<double> > mean;
whiteice::math::matrix< whiteice::math::blas_real<double> > cov;
whiteice::math::vertex< whiteice::math::blas_real<double> > score;
auto tmp = scoring[i];
if(train.preprocess(0, tmp) == false){
setStatus("Calculating prediction failed (preprocess)");
setError("Internal software error");
optimize = false;
break;
}
if(bnn.calculate(tmp, mean, cov) == false){
setStatus("Calculating prediction failed");
setError("Internal software error");
optimize = false;
break;
}
score.resize(1);
score[0] = mean[0] + risk*cov(0,0);
if(results.add(0, score) == false){
setStatus("Calculating prediction failed (storage)");
setError("Internal software error");
optimize = false;
break;
}
if(optimize == false)
break; // lost our license to continue
}
if(optimize == false)
continue; // lost our license to continue
// finally save the results
setStatus("Saving prediction results to file..");
if(results.exportAscii(resultsFile) == false){
setStatus("Saving prediction results failed");
setError("Internal software error");
optimize = false;
break;
}
setStatus("Computations complete");
optimize = false;
}
}
int main_1dsfs(int argc,char **argv){
if(argc<2){
fprintf(stderr,"Must supply afile.saf and number of chromosomes\n");
return 0;
}
fname1 = *(argv++);
chr1 = atoi(*(argv++));
argc-=2;
getArgs(argc,argv);
dim=chr1+1;
//hook for new EJ banded version
if(isNewFormat(fname1))
return main_1dsfs_v2(fname1,chr1,nSites,nThreads,sfsfname,tole,maxIter);
if(nSites==0){//if no -nSites is specified
if(fsize(fname1)>getTotalSystemMemory())
fprintf(stderr,"Looks like you will allocate too much memory, consider starting the program with a lower -nSites argument\n");
//this doesnt make sense if ppl supply a filelist containing safs
nSites=calcNsites(fname1,chr1);
}
fprintf(stderr,"fname1:%s nChr:%d startsfs:%s nThreads:%d tole=%f maxIter=%d nSites=%lu\n",fname1,chr1,sfsfname,nThreads,tole,maxIter,nSites);
float bytes_req_megs = nSites*(sizeof(double)*(chr1+1)+sizeof(double*))/1024/1024;
float mem_avail_megs = getTotalSystemMemory()/1024/1024;//in percentile
// fprintf(stderr,"en:%zu to:%f\n",bytes_req_megs,mem_avail_megs);
fprintf(stderr,"The choice of -nSites will require atleast: %f megabyte memory, that is approx: %.2f%% of total memory\n",bytes_req_megs,bytes_req_megs*100/mem_avail_megs);
Matrix<double> GL1=alloc(nSites,dim);
gzFile gz1=getGz(fname1);
double *sfs=new double[dim];
while(1) {
if(isList==0)
readGL(gz1,nSites,chr1,GL1);
else
readGL2(gz1,nSites,chr1,GL1);
if(GL1.x==0)
break;
fprintf(stderr,"dim(GL1)=%zu,%zu\n",GL1.x,GL1.y);
if(sfsfname!=NULL){
readSFS(sfsfname,dim,sfs);
}else{
for(int i=0;i<dim;i++)
sfs[i] = (i+1)/((double)dim);
if(doBFGS){
double ts=1;
for(int i=0;i<dim-1;i++)
ts += 0.01/(1.0+i);
sfs[0]=1.0/ts;
for(int i=0;i<dim-1;i++)
sfs[i+1] = (0.01/(1.0+i))/ts;
}
normalize(sfs,dim);
}
// em2_smart(sfs2,pops,1e-6,1e3);
setThreadPars(&GL1,NULL,sfs,nThreads);
if(calcLike==0){
if(doBFGS==0)
em1(sfs,&GL1,tole,maxIter);
else
bfgs(sfs,&GL1);
}
double lik;
if(nThreads>1)
lik = lik1_master();
else
lik = lik1(sfs,&GL1,0,GL1.x);
fprintf(stderr,"likelihood: %f\n",lik);
#if 1
for(int x=0;x<dim;x++)
fprintf(stdout,"%f ",log(sfs[x]));
fprintf(stdout,"\n");
fflush(stdout);
#endif
if(isList==1)
break;
}
dalloc(GL1,nSites);
gzclose(gz1);
delete [] sfs;
return 0;
}
