/** Start training/learning a model w.r.t. the loss object (and the data supplied to it). * */ void CBMRM::Train() { CTimer totalTime; // total runtime of the training CTimer innerSolverTime; // time for inner optimization (e.g., QP or LP) CTimer lossAndGradientTime; // time for loss and gradient computation unsigned int iter = 0; // iteration count double loss = 0.0; // loss function value double exactObjVal = 0.0; // (exact) objective function value double approxObjVal = -1e99; // convex lower-bound (approximate) of objective function value double minExactObjVal = 1e99; // minimum of all previously evaluated (exact) objective function value double regVal = 0.0; // value of the regularizer term e.g., 0.5*w'*w double epsilon = 0.0; // := minExactObjVal - approxObjVal double gamma = 0.0; // := exactObjVal - approxObjVal double prevEpsilon = 0.0; double innerSolverTol = 1.0; // optimization tolerance for inner solver int exitFlag = 0; unsigned int row = 0; unsigned int col = 0; TheMatrix &w = _model->GetW(); w.Shape(row, col); TheMatrix a(row, col, SML::DENSE); // gradient vector TheMatrix w_best(row,col,SML::DENSE); // w_t at which pobj is the smallest #ifdef PARALLEL_BMRM double someloss = 0.0; // temporary loss incurred on each sub-dataset double *tmpaw = new double[row*col]; // temporary array for gradient/w double *tmpfinalaw = new double[row*col]; // temporary final array for reducing/broadcasting gradient/w #endif // start training totalTime.Start(); // Initialize piecewise linear lower bound of empirical risk { iter = 1; lossAndGradientTime.Start(); _loss->ComputeLossAndGradient(loss, a); lossAndGradientTime.Stop(); #ifdef PARALLEL_BMRM MASTER(procID) #endif { if(verbosity) { printf("Initial iteration: computing first linearization at w_0... loss(w_0)=%.6e\n",loss); fflush(stdout); } } #ifdef PARALLEL_BMRM // Aggregate computed loss value and gradient for(unsigned int rowidx=0; rowidx<row; rowidx++) { unsigned int rowlen = 0; a.GetRow(rowidx, rowlen, &tmpaw[rowidx*col]); assert(rowlen == col); } memset(tmpfinalaw,0,sizeof(double)*row*col); someloss = loss; loss = 0.0; MPI_Barrier(MPI_COMM_WORLD); MPI_Reduce(&someloss, &loss, 1, MPI_DOUBLE, MPI_SUM, ROOT_PROC_ID, MPI_COMM_WORLD); MPI_Reduce(tmpaw, tmpfinalaw, row*col, MPI_DOUBLE, MPI_SUM, ROOT_PROC_ID, MPI_COMM_WORLD); for(unsigned int rowidx=0; rowidx<row; rowidx++) a.SetRow(rowidx, col, &tmpfinalaw[rowidx*col]); #endif } do { iter++; #ifdef PARALLEL_BMRM MASTER(procID) #endif { // Minimize piecewise linear lower bound R_t innerSolverTime.Start(); innerSolver->Solve(w, a, loss, approxObjVal); innerSolverTime.Stop(); } #ifdef PARALLEL_BMRM // Broadcast updated w for(unsigned int rowidx=0; rowidx<row; rowidx++) { unsigned int rowlen = 0; w.GetRow(rowidx, rowlen, &tmpaw[rowidx*col]); assert(rowlen == col); } memset(tmpfinalaw, 0, sizeof(double)*row*col); MPI_Bcast(tmpaw, row*col, MPI_DOUBLE, ROOT_PROC_ID, MPI_COMM_WORLD); for (unsigned int rowidx=0; rowidx<row; rowidx++) w.SetRow(rowidx, col, &tmpaw[rowidx*col]); #endif // Compute new linearization with updated w lossAndGradientTime.Start(); _loss->ComputeLossAndGradient(loss, a); #ifdef LINESEARCH_BMRM loss = _loss->GetLossOfWbest(); _loss->GetWbest(w_best); #endif lossAndGradientTime.Stop(); #ifdef PARALLEL_BMRM // Aggregate computed loss value and gradient for(unsigned int rowidx=0; rowidx<row; rowidx++) { unsigned int rowlen = 0; a.GetRow(rowidx, rowlen, &tmpaw[rowidx*col]); assert(rowlen == col); } memset(tmpfinalaw,0,sizeof(double)*row*col); someloss = loss; loss = 0.0; MPI_Barrier(MPI_COMM_WORLD); MPI_Reduce(&someloss, &loss, 1, MPI_DOUBLE, MPI_SUM, ROOT_PROC_ID, MPI_COMM_WORLD); MPI_Reduce(tmpaw, tmpfinalaw, row*col, MPI_DOUBLE, MPI_SUM, ROOT_PROC_ID, MPI_COMM_WORLD); for(unsigned int rowidx=0; rowidx<row; rowidx++) a.SetRow(rowidx, col, &tmpfinalaw[rowidx*col]); #endif #ifdef PARALLEL_BMRM MASTER(procID) #endif { // Update iteration details and keep best minimizer #ifdef LINESEARCH_BMRM regVal = innerSolver->ComputeRegularizerValue(w_best); exactObjVal = loss + regVal; minExactObjVal = min(minExactObjVal,exactObjVal); #else regVal = innerSolver->ComputeRegularizerValue(w); exactObjVal = loss + regVal; if(exactObjVal < minExactObjVal) { minExactObjVal = exactObjVal; w_best.Assign(w); } #endif prevEpsilon = epsilon; gamma = exactObjVal - approxObjVal; epsilon = minExactObjVal - approxObjVal; // Optional: Adjust inner solver optimization tolerance // This reduces the number of iteration most of the time. // This in some sense mimics the proximity control in proximal BM AdjustInnerSolverOptTol(innerSolverTol, prevEpsilon, epsilon); // Display details of each iteration DisplayIterationInfo(iter,exactObjVal,approxObjVal,epsilon,gamma, loss,regVal,totalTime.CurrentCPUTotal()); // Save model obtained in previous iteration SaveCheckpointModel(iter); // Check if termination criteria satisfied exitFlag = CheckTermination(iter,epsilon,gamma,minExactObjVal,exactObjVal); } #ifdef PARALLEL_BMRM // Broadcast loop termination flag MPI_Bcast(&exitFlag, 1, MPI_INT, ROOT_PROC_ID, MPI_COMM_WORLD); #endif } while(!exitFlag); totalTime.Stop(); #ifdef PARALLEL_BMRM MASTER(procID) #endif { // Display after-training details DisplayAfterTrainingInfo(iter,minExactObjVal,approxObjVal,loss, w_best,lossAndGradientTime,innerSolverTime,totalTime); } }