void autoencoder_GPU::train(){ for(int epoch = 0; epoch < nEpochNum; epoch++){ dataprovider->reset(); printf("Epoch %d\n", epoch + 1); gpu_reset(gpu_env, reset, d_error, nLayerSize0 * nVectorPerBatch, NULL); for(int batch = 0; batch < nBatchNum; batch++){ dataprovider->getNextDeviceBatch(d_layer0act); fprop(); /* if(batch == 1){ gpu_env.status = clEnqueueReadBuffer(gpu_env.queue, d_bias0, CL_TRUE, 0, nLayerSize1 * sizeof(floatType), (void*)bias0, 0, NULL, NULL); ofstream tempStream; tempStream.open("../log/bias.log", ios_base::trunc); for(unsigned i = 0; i < nLayerSize1; i++){ tempStream << bias0[i] << ','; if((i + 1) % nLayerSize1 == 0){ tempStream << endl; } } tempStream.close(); } if(batch == 1){ gpu_env.status = clEnqueueReadBuffer(gpu_env.queue, d_weight0, CL_TRUE, 0, nLayerSize0 * nLayerSize1 * sizeof(floatType), (void*)weight0, 0, NULL, NULL); ofstream tempStream; tempStream.open("../log/weight.log", ios_base::trunc); for(unsigned i = 0; i < nLayerSize0 * nLayerSize1; i++){ tempStream << weight0[i] << ','; if((i + 1) % nLayerSize0 == 0){ tempStream << endl; } } tempStream.close(); } if(batch == 1){ gpu_env.status = clEnqueueReadBuffer(gpu_env.queue, d_layer1act, CL_TRUE, 0, nVectorPerBatch * nLayerSize1 * sizeof(floatType), (void*)layer1act, 0, NULL, NULL); ofstream tempStream; tempStream.open("../log/activation.log", ios_base::trunc); for(unsigned i = 0; i < nVectorPerBatch * nLayerSize1; i++){ tempStream << layer1act[i] << ','; if((i + 1) % nVectorPerBatch == 0){ tempStream << endl; } } tempStream.close(); // exit(0); } */ gpu_squareError(gpu_env, squareError, d_layer8act, d_layer0act, d_error, nLayerSize0 * nVectorPerBatch); bprop(); update(); /* if(!epoch){ double errsum = 0.0; gpu_env.status = clEnqueueReadBuffer(gpu_env.queue, d_error, CL_TRUE, 0, nLayerSize0 * nVectorPerBatch * sizeof(floatType), (void*)error, 0, NULL, NULL); for(int i = 0; i < nLayerSize8 * nVectorPerBatch; i++){ errsum += error[i]; } printf("Epoch %d Batch %d Error %f\n", epoch + 1, batch + 1, errsum); } */ } double errsum = 0.0; gpu_env.status = clEnqueueReadBuffer(gpu_env.queue, d_error, CL_TRUE, 0, nLayerSize0 * nVectorPerBatch * sizeof(floatType), (void*)error, 0, NULL, NULL); for(int i = 0; i < nLayerSize8 * nVectorPerBatch; i++){ errsum += error[i]; } printf("Epoch %d Error %f\n", epoch + 1, errsum); ofstream fout; fout.open("../log/errorLog.txt", ios_base::app); struct timeval now; gettimeofday(&now, NULL); fout << now.tv_sec << ',' << errsum << endl; fout.close(); } ofstream fout; gpu_env.status = clEnqueueReadBuffer(gpu_env.queue, d_weight0, CL_TRUE, 0, nLayerSize0 * nLayerSize1 * sizeof(floatType), (void*)weight0, 0, NULL, NULL); gpu_env.status = clEnqueueReadBuffer(gpu_env.queue, d_weight1, CL_TRUE, 0, nLayerSize1 * nLayerSize2 * sizeof(floatType), (void*)weight1, 0, NULL, NULL); gpu_env.status = clEnqueueReadBuffer(gpu_env.queue, d_weight2, CL_TRUE, 0, nLayerSize2 * nLayerSize3 * sizeof(floatType), (void*)weight2, 0, NULL, NULL); gpu_env.status = clEnqueueReadBuffer(gpu_env.queue, d_weight3, CL_TRUE, 0, nLayerSize3 * nLayerSize4 * sizeof(floatType), (void*)weight3, 0, NULL, NULL); gpu_env.status = clEnqueueReadBuffer(gpu_env.queue, d_weight4, CL_TRUE, 0, nLayerSize4 * nLayerSize5 * sizeof(floatType), (void*)weight4, 0, NULL, NULL); gpu_env.status = clEnqueueReadBuffer(gpu_env.queue, d_weight5, CL_TRUE, 0, nLayerSize5 * nLayerSize6 * sizeof(floatType), (void*)weight5, 0, NULL, NULL); gpu_env.status = clEnqueueReadBuffer(gpu_env.queue, d_weight6, CL_TRUE, 0, nLayerSize6 * nLayerSize7 * sizeof(floatType), (void*)weight6, 0, NULL, NULL); gpu_env.status = clEnqueueReadBuffer(gpu_env.queue, d_weight7, CL_TRUE, 0, nLayerSize7 * nLayerSize8 * sizeof(floatType), (void*)weight7, 0, NULL, NULL); gpu_env.status = clEnqueueReadBuffer(gpu_env.queue, d_bias0, CL_TRUE, 0, nLayerSize1 * sizeof(floatType), (void*)bias0, 0, NULL, NULL); gpu_env.status = clEnqueueReadBuffer(gpu_env.queue, d_bias1, CL_TRUE, 0, nLayerSize2 * sizeof(floatType), (void*)bias1, 0, NULL, NULL); gpu_env.status = clEnqueueReadBuffer(gpu_env.queue, d_bias2, CL_TRUE, 0, nLayerSize3 * sizeof(floatType), (void*)bias2, 0, NULL, NULL); gpu_env.status = clEnqueueReadBuffer(gpu_env.queue, d_bias3, CL_TRUE, 0, nLayerSize4 * sizeof(floatType), (void*)bias3, 0, NULL, NULL); gpu_env.status = clEnqueueReadBuffer(gpu_env.queue, d_bias4, CL_TRUE, 0, nLayerSize5 * sizeof(floatType), (void*)bias4, 0, NULL, NULL); gpu_env.status = clEnqueueReadBuffer(gpu_env.queue, d_bias5, CL_TRUE, 0, nLayerSize6 * sizeof(floatType), (void*)bias5, 0, NULL, NULL); gpu_env.status = clEnqueueReadBuffer(gpu_env.queue, d_bias6, CL_TRUE, 0, nLayerSize7 * sizeof(floatType), (void*)bias6, 0, NULL, NULL); gpu_env.status = clEnqueueReadBuffer(gpu_env.queue, d_bias7, CL_TRUE, 0, nLayerSize8 * sizeof(floatType), (void*)bias7, 0, NULL, NULL); fout.open("../data/autoencoderWeight.dat", ios_base::binary | ios_base::trunc); fout.write((char*)weight0, nLayerSize0 * nLayerSize1 * sizeof(floatType)); fout.write((char*)weight1, nLayerSize1 * nLayerSize2 * sizeof(floatType)); fout.write((char*)weight2, nLayerSize2 * nLayerSize3 * sizeof(floatType)); fout.write((char*)weight3, nLayerSize3 * nLayerSize4 * sizeof(floatType)); fout.write((char*)weight4, nLayerSize4 * nLayerSize5 * sizeof(floatType)); fout.write((char*)weight5, nLayerSize5 * nLayerSize6 * sizeof(floatType)); fout.write((char*)weight6, nLayerSize6 * nLayerSize7 * sizeof(floatType)); fout.write((char*)weight7, nLayerSize7 * nLayerSize8 * sizeof(floatType)); fout.close(); fout.open("../data/autoencoderBias.dat", ios_base::binary | ios_base::trunc); fout.write((char*)bias0, nLayerSize1 * sizeof(floatType)); fout.write((char*)bias1, nLayerSize2 * sizeof(floatType)); fout.write((char*)bias2, nLayerSize3 * sizeof(floatType)); fout.write((char*)bias3, nLayerSize4 * sizeof(floatType)); fout.write((char*)bias4, nLayerSize5 * sizeof(floatType)); fout.write((char*)bias5, nLayerSize6 * sizeof(floatType)); fout.write((char*)bias6, nLayerSize7 * sizeof(floatType)); fout.write((char*)bias7, nLayerSize8 * sizeof(floatType)); fout.close(); }
int main(int argc, char *argv[]){ Params params; std::map<std::string, std::string> args; readArgs(argc, argv, args); if(args.find("algo")!=args.end()){ params.algo = args["algo"]; }else{ params.algo = "qdMCNat"; } if(args.find("inst_file")!=args.end()) setParamsFromFile(args["inst_file"], args, params); else setParams(params.algo, args, params); createLogDir(params.dir_path); gen.seed(params.seed); // Load the dataset MyMatrix X_train, X_valid; VectorXd Y_train, Y_valid; loadMnist(params.ratio_train, X_train, X_valid, Y_train, Y_valid); //loadCIFAR10(params.ratio_train, X_train, X_valid, Y_train, Y_valid); //loadLightCIFAR10(params.ratio_train, X_train, X_valid, Y_train, Y_valid); // ConvNet parameters std::vector<ConvLayerParams> conv_params; ConvLayerParams conv_params1; conv_params1.Hf = 5; conv_params1.stride = 1; conv_params1.n_filter = 20; conv_params1.padding = 0; conv_params.push_back(conv_params1); ConvLayerParams conv_params2; conv_params2.Hf = 5; conv_params2.stride = 1; conv_params2.n_filter = 50; conv_params2.padding = 0; conv_params.push_back(conv_params2); std::vector<PoolLayerParams> pool_params; PoolLayerParams pool_params1; pool_params1.Hf = 2; pool_params1.stride = 2; pool_params.push_back(pool_params1); PoolLayerParams pool_params2; pool_params2.Hf = 2; pool_params2.stride = 2; pool_params.push_back(pool_params2); const unsigned n_conv_layer = conv_params.size(); for(unsigned l = 0; l < conv_params.size(); l++){ if(l==0){ conv_params[l].filter_size = conv_params[l].Hf * conv_params[l].Hf * params.img_depth; conv_params[l].N = (params.img_width - conv_params[l].Hf + 2*conv_params[l].padding)/conv_params[l].stride + 1; } else{ conv_params[l].filter_size = conv_params[l].Hf * conv_params[l].Hf * conv_params[l-1].n_filter; conv_params[l].N = (pool_params[l-1].N - conv_params[l].Hf + 2*conv_params[l].padding)/conv_params[l].stride + 1; } pool_params[l].N = (conv_params[l].N - pool_params[l].Hf)/pool_params[l].stride + 1; } // Neural Network parameters const unsigned n_training = X_train.rows(); const unsigned n_valid = X_valid.rows(); const unsigned n_feature = X_train.cols(); const unsigned n_label = Y_train.maxCoeff() + 1; params.nn_arch.insert(params.nn_arch.begin(),conv_params[n_conv_layer-1].n_filter * pool_params[n_conv_layer-1].N * pool_params[n_conv_layer-1].N); params.nn_arch.push_back(n_label); const unsigned n_layers = params.nn_arch.size(); // Optimization parameter const int n_train_batch = ceil(n_training/(float)params.train_minibatch_size); const int n_valid_batch = ceil(n_valid/(float)params.valid_minibatch_size); double prev_loss = std::numeric_limits<double>::max(); double eta = params.eta; // Create the convolutional layer std::vector<MyMatrix> conv_W(n_conv_layer); std::vector<MyMatrix> conv_W_T(n_conv_layer); std::vector<MyVector> conv_B(n_conv_layer); // Create the neural network MyMatrix W_out(params.nn_arch[n_layers-2],n_label); std::vector<MySpMatrix> W(n_layers-2); std::vector<MySpMatrix> Wt(n_layers-2); std::vector<MyVector> B(n_layers-1); double init_sigma = 0.; ActivationFunction act_func; ActivationFunction eval_act_func; if(params.act_func_name=="sigmoid"){ init_sigma = 4.0; act_func = std::bind(logistic,true,_1,_2,_3); eval_act_func = std::bind(logistic,false,_1,_2,_3); }else if(params.act_func_name=="tanh"){ init_sigma = 1.0; act_func = std::bind(my_tanh,true,_1,_2,_3); eval_act_func = std::bind(my_tanh,false,_1,_2,_3); }else if(params.act_func_name=="relu"){ init_sigma = 1.0; // TODO: Find the good value act_func = std::bind(relu,true,_1,_2,_3); eval_act_func = std::bind(relu,false,_1,_2,_3); }else{ std::cout << "Not implemented yet!" << std::endl; assert(false); } std::cout << "Initializing the network... "; params.n_params = initNetwork(params.nn_arch, params.act_func_name, params.sparsity, conv_params, pool_params, W_out, W, Wt, B, conv_W, conv_W_T, conv_B); // TODO: Init the conv bias // Deep copy of parameters for the adaptive rule std::vector<MyMatrix> mu_dW(n_layers-1); std::vector<MyVector> mu_dB(n_layers-1); MyMatrix pW_out = W_out; std::vector<MySpMatrix> pW = W; std::vector<MySpMatrix> pWt = Wt; std::vector<MyVector> pB = B; MyMatrix ppMii_out, ppM0i_out; MyVector ppM00_out; std::vector<MySpMatrix> ppMii,ppM0i; std::vector<MyVector> ppM00; MyMatrix pMii_out,pM0i_out; MyVector pM00_out; std::vector<MySpMatrix> pMii,pM0i; std::vector<MyVector> pM00; std::vector<MyMatrix> conv_ppMii, conv_ppM0i; std::vector<MyVector> conv_ppM00; std::vector<MyMatrix> conv_pMii, conv_pM0i; std::vector<MyVector> conv_pM00; // Convert the labels to one-hot vector MyMatrix one_hot = MyMatrix::Zero(n_training, n_label); labels2oneHot(Y_train,one_hot); // Configure the logger std::ostream* logger; if(args.find("verbose")!=args.end()){ getOutput("",logger); }else{ getOutput(params.file_path,logger); } double cumul_time = 0.; printDesc(params, logger); printConvDesc(params, conv_params, pool_params, logger); std::cout << "Starting the learning phase... " << std::endl; *logger << "Epoch Time(s) train_loss train_accuracy valid_loss valid_accuracy eta" << std::endl; for(unsigned i = 0; i < params.n_epoch; i++){ for(unsigned j = 0; j < n_train_batch; j++){ // Mini-batch creation unsigned curr_batch_size = 0; MyMatrix X_batch, one_hot_batch; getMiniBatch(j, params.train_minibatch_size, X_train, one_hot, params, conv_params[0], curr_batch_size, X_batch, one_hot_batch); double prev_time = gettime(); // Forward propagation for conv layer std::vector<std::vector<unsigned>> poolIdxX1(n_conv_layer); std::vector<std::vector<unsigned>> poolIdxY1(n_conv_layer); MyMatrix z0; std::vector<MyMatrix> conv_A(conv_W.size()); std::vector<MyMatrix> conv_Ap(conv_W.size()); convFprop(curr_batch_size, conv_params, pool_params, act_func, conv_W, conv_B, X_batch, conv_A, conv_Ap, z0, poolIdxX1, poolIdxY1); // Forward propagation std::vector<MyMatrix> Z(n_layers-1); std::vector<MyMatrix> A(n_layers-2); std::vector<MyMatrix> Ap(n_layers-2); fprop(params.dropout_flag, act_func, W, W_out, B, z0, Z, A, Ap); // Compute the output and the error MyMatrix out; softmax(Z[n_layers-2], out); std::vector<MyMatrix> gradB(n_layers-1); gradB[n_layers-2] = out - one_hot_batch; // Backpropagation bprop(Wt, W_out, Ap, gradB); // Backpropagation for conv layer std::vector<MyMatrix> conv_gradB(conv_W.size()); MyMatrix layer_gradB = (gradB[0] * W[0].transpose()); MyMatrix pool_gradB; layer2pool(curr_batch_size, pool_params[conv_W.size()-1].N, conv_params[conv_W.size()-1].n_filter, layer_gradB, pool_gradB); convBprop(curr_batch_size, conv_params, pool_params, conv_W_T, conv_Ap, pool_gradB, conv_gradB, poolIdxX1, poolIdxY1); if(params.algo == "bprop"){ update(eta, gradB, A, z0, params.regularizer, params.lambda, W_out, W, Wt, B); convUpdate(curr_batch_size, eta, conv_params, conv_gradB, conv_A, X_batch, "", 0., conv_W, conv_W_T, conv_B); }else{ // Compute the metric std::vector<MyMatrix> metric_gradB(n_layers-1); std::vector<MyMatrix> metric_conv_gradB(conv_params.size()); if(params.algo=="qdMCNat"){ // Monte-Carlo Approximation of the metric std::vector<MyMatrix> mc_gradB(n_layers-1); computeMcError(out, mc_gradB[n_layers-2]); // Backpropagation bprop(Wt, W_out, Ap, mc_gradB); for(unsigned k = 0; k < gradB.size(); k++){ metric_gradB[k] = mc_gradB[k].array().square(); } // Backpropagation for conv layer std::vector<MyMatrix> mc_conv_gradB(conv_W.size()); MyMatrix mc_layer_gradB = (mc_gradB[0] * W[0].transpose()); MyMatrix mc_pool_gradB; layer2pool(curr_batch_size, pool_params[conv_W.size()-1].N, conv_params[conv_W.size()-1].n_filter, mc_layer_gradB, mc_pool_gradB); convBprop(curr_batch_size, conv_params, pool_params, conv_W_T, conv_Ap, mc_pool_gradB, mc_conv_gradB, poolIdxX1, poolIdxY1); for(unsigned k = 0; k < conv_params.size(); k++){ metric_conv_gradB[k] = mc_conv_gradB[k].array().square(); } } else if(params.algo=="qdop"){ for(unsigned k = 0; k < conv_params.size(); k++){ metric_conv_gradB[k] = conv_gradB[k].array().square(); } for(unsigned k = 0; k < gradB.size(); k++){ metric_gradB[k] = gradB[k].array().square(); } } else if(params.algo=="qdNat"){ for(unsigned k = 0; k < conv_params.size(); k++){ metric_conv_gradB[k] = conv_gradB[k].array().square(); } for(unsigned k = 0; k < metric_gradB.size(); k++){ metric_gradB[k] = MyMatrix::Zero(gradB[k].rows(),gradB[k].cols()); } for(unsigned l = 0; l < n_label; l++){ MyMatrix fisher_ohbatch = MyMatrix::Zero(curr_batch_size, n_label); fisher_ohbatch.col(l).setOnes(); std::vector<MyMatrix> fgradB(n_layers-1); fgradB[n_layers-2] = out - fisher_ohbatch; bprop(Wt, W_out, Ap, fgradB); // Backpropagation for conv layer std::vector<MyMatrix> fisher_conv_gradB(conv_W.size()); MyMatrix fisher_layer_gradB = (fgradB[0] * W[0].transpose()); MyMatrix fisher_pool_gradB; layer2pool(curr_batch_size, pool_params[conv_W.size()-1].N, conv_params[conv_W.size()-1].n_filter, fisher_layer_gradB, fisher_pool_gradB); convBprop(curr_batch_size, conv_params, pool_params, conv_W_T, conv_Ap, fisher_pool_gradB, fisher_conv_gradB, poolIdxX1, poolIdxY1); for(unsigned k = 0; k < conv_params.size(); k++){ MyMatrix fisher_conv_gradB_sq = fisher_conv_gradB[k].array().square(); for(unsigned m = 0; m < out.rows(); m++){ for(unsigned f = 0; f < conv_params[k].n_filter; f++){ for(unsigned n = 0; n < conv_params[k].N * conv_params[k].N; n++){ fisher_conv_gradB_sq(f,m*conv_params[k].N*conv_params[k].N+n) *= out(m,l); } } } metric_conv_gradB[k] += fisher_conv_gradB_sq; } for(unsigned k = 0; k < W.size(); k++){ const unsigned rev_k = n_layers - k - 2; metric_gradB[rev_k] += (fgradB[rev_k].array().square().array().colwise() * out.array().col(l)).matrix(); } } } bool init_flag = false; if(i == 0 && j == 0 && !params.init_metric_id){ init_flag = true; } std::vector<MyMatrix> conv_Mii(conv_params.size()); std::vector<MyMatrix> conv_M0i(conv_params.size()); std::vector<MyVector> conv_M00(conv_params.size()); buildConvQDMetric(curr_batch_size, metric_conv_gradB, conv_A, X_batch, conv_W, params.matrix_reg, conv_Mii, conv_M0i, conv_M00); updateConvMetric(init_flag, params.metric_gamma, conv_pMii, conv_pM0i, conv_pM00, conv_Mii, conv_M0i, conv_M00); MyMatrix Mii_out, M0i_out; MyVector M00_out; std::vector<MySpMatrix> Mii(W.size()); std::vector<MySpMatrix> M0i(W.size()); std::vector<MyVector> M00(W.size()); buildQDMetric(metric_gradB, A, z0, W_out, W, params.matrix_reg, Mii_out, M0i_out, M00_out, Mii, M0i, M00); updateMetric(init_flag, params.metric_gamma, Mii_out, M0i_out, M00_out, Mii, M0i, M00, pMii_out, pM0i_out, pM00_out, pMii, pM0i, pM00); update(eta, gradB, A, z0, params.regularizer, params.lambda, W_out, W, Wt, B, Mii_out, M0i_out, M00_out, Mii, M0i, M00); } double curr_time = gettime(); cumul_time += curr_time - prev_time; if(params.minilog_flag){ double train_loss = 0.; double train_accuracy = 0.; double valid_loss = 0.; double valid_accuracy = 0.; evalModel(eval_act_func, params, n_train_batch, n_training, X_train, Y_train, conv_params, pool_params, conv_W, conv_B, W_out, W, B, train_loss, train_accuracy); evalModel(eval_act_func, params, n_valid_batch, n_valid, X_valid, Y_valid, conv_params, pool_params, conv_W, conv_B, W_out, W, B, valid_loss, valid_accuracy); // Logging *logger << i + float(j)/n_train_batch << " " << cumul_time << " " << train_loss << " " << train_accuracy << " " << valid_loss << " " << valid_accuracy << " " << eta << std::endl; } } if(!params.minilog_flag || params.adaptive_flag){ double train_loss = 0.; double train_accuracy = 0.; double valid_loss = 0.; double valid_accuracy = 0.; evalModel(eval_act_func, params, n_train_batch, n_training, X_train, Y_train, conv_params, pool_params, conv_W, conv_B, W_out, W, B, train_loss, train_accuracy); evalModel(eval_act_func, params, n_valid_batch, n_valid, X_valid, Y_valid, conv_params, pool_params, conv_W, conv_B, W_out, W, B, valid_loss, valid_accuracy); // if(params.adaptive_flag) // adaptiveRule(train_loss, prev_loss, eta, W, B, pMii, pM0i, pM00, pW, pB, ppMii, ppM0i, ppM00); // Logging if(!params.minilog_flag){ *logger << i << " " << cumul_time << " " << train_loss << " " << train_accuracy << " " << valid_loss << " " << valid_accuracy << " " << eta << std::endl; } } } }
/* -- mlp_forback -------------------------------------------------------- */ int mlp_forback( float *stims, int *stimstarts, int ndim, int negs, int *stimoffs, int nin, int *nunits, int nlevels, int *tranfns, float *activs, float *biases, int ntunits, float *weights, int nweights, float *bschange, float *wtchange, float *etas, float *etbs, float alpha, float decay, float *targs, int *targstarts, int *targoffs, int nout, int niter, int nbatch, int ransel, float *err, float *errvar) /* Carries out niter learning cycles on the machine, selecting stimuli at random from the stims and targs arrays if RANSEL is non-zero, otherwise taking them in sequence. Other parameters are as in bprop, fprop and mlp_forward, with targs etc. instead of outs etc. If nbatch is 1, then does continuous learning with momentum governed by alpha. If nbatch is greater than 1, then does batch learning, averaging errors over nbatch examples before updating. In this case, alpha is ignored. - one special case - if niter is 0, just do a single backward pass, assuming that the forward pass has already been carried out. On return activs is set to the latest error signals, and an explicit call of fprop is needed to get activations. Err returns the mean error, errvar its variance. Returns fail code. */ { float anegs = negs, cerr, errsum = 0.0, errsumsqu = 0.0; int iter, eg = -1, dofwd = niter > 0, batching = nbatch > 1, si, so; /* check inputs to avoid having to do so on each iteration */ int ifail = checkns(nunits,nlevels,nin,nout,ntunits,nweights); if (ifail) return ifail; if (nbatch <= 0) return 10; /* When batching, niter is given as number of batches - change to no of egs and ensure weight change arrays are zeroed */ if (batching) { niter *= nbatch; mlp_fillvec(0.0, wtchange, nweights); mlp_fillvec(0.0, bschange, ntunits); } if (niter == 0) niter = 1; /* Always do a backward pass */ /* Iterate */ for (iter = 1; iter <= niter; iter++) { if (ndim) { /* stimstarts is n-D array giving limits */ si = mlp_getsample(ndim, stimstarts, ransel); so = mlp_getoutsample(ndim, stimstarts, targstarts); } else { /* stimstarts 1-D array of starting points */ if (ransel) eg = (int)(erand48(seed) * anegs); else eg = (eg+1) % negs; si = *(stimstarts+eg); so = *(targstarts+eg); } if (dofwd) fprop(stims+si, stimoffs,nin,nunits,nlevels,tranfns,activs, biases,ntunits,weights,nweights); if (batching) { cerr = bprop_batch(stims+si, stimoffs, nin, nunits, nlevels, tranfns, activs, ntunits, weights, nweights, wtchange, bschange, targs+so, targoffs, nout); if (iter % nbatch == 0) bwtupd_batch(biases, ntunits, weights, nweights, wtchange, bschange, etas, etbs, decay, nbatch); } else cerr = bprop(stims+si, stimoffs, nin, nunits, nlevels, tranfns, activs, biases, ntunits, weights, nweights, wtchange, bschange, etas, etbs, alpha, decay, targs+so, targoffs, nout); errsum += cerr; errsumsqu += cerr * cerr; } /* Calculate the error and its variance over this set of trials. It's divided by 2 because bprop returns a sum of squares, but actually uses the derivative with respect to half the sum of squares. */ *err = errsum/(2*niter); *errvar = errsumsqu/(4*niter) - *err * *err; return 0; }