/* * Internal train function */ float fann_train_epoch_sarprop(struct fann *ann, struct fann_train_data *data) { unsigned int i; if(ann->prev_train_slopes == NULL) { fann_clear_train_arrays(ann); } fann_reset_MSE(ann); for(i = 0; i < data->num_data; i++) { fann_run(ann, data->input[i]); fann_compute_MSE(ann, data->output[i]); fann_backpropagate_MSE(ann); fann_update_slopes_batch(ann, ann->first_layer + 1, ann->last_layer - 1); } fann_update_weights_sarprop(ann, ann->sarprop_epoch, 0, ann->total_connections); ++(ann->sarprop_epoch); return fann_get_MSE(ann); }
/* Trains the network with the backpropagation algorithm. */ FANN_EXTERNAL void FANN_API fann_train(struct fann *ann, fann_type * input, fann_type * desired_output) { fann_run(ann, input); fann_compute_MSE(ann, desired_output); fann_backpropagate_MSE(ann); fann_update_weights(ann); }
float train_epoch_debug(struct fann *ann, struct fann_train_data* data, unsigned int iter) { unsigned int i; #if VERBOSE>=2 static unsigned int j=0; #endif #if ! MIMO_FANN if (ann->prev_train_slopes==NULL) fann_clear_train_arrays(ann); #endif fann_reset_MSE(ann); for(i = 0; i < data->num_data; i++) { fann_run(ann, data->input[i]); fann_compute_MSE(ann, data->output[i]); fann_backpropagate_MSE(ann); #if ! MIMO_FANN fann_update_slopes_batch(ann, ann->first_layer + 1, ann->last_layer - 1); #endif #if VERBOSE>=3 printf(" ** %d:%d **-AFTER-DELTAS UPDATE-----------------------------------\n", iter, i); print_deltas(ann, j++); #endif } #if VERBOSE>=2 printf(" ** %d **-BEFORE-WEIGHTS-UPDATE------------------------------------\n", iter); print_deltas(ann, j++); #endif #if ! MIMO_FANN #if USE_RPROP fann_update_weights_irpropm(ann, 0, ann->total_connections); #else fann_update_weights_batch(ann, data->num_data, 0, ann->total_connections); #endif #else /* MIMO_FANN */ fann_update_weights(ann); #endif #if VERBOSE>=1 printf(" ** %d **-AFTER-WEIGHTS-UPDATE-------------------------------------\n", iter); print_deltas(ann, j++); #endif return fann_get_MSE(ann); }
/* * Internal train function */ float fann_train_epoch_batch(struct fann *ann, struct fann_train_data *data) { unsigned int i; fann_reset_MSE(ann); for(i = 0; i < data->num_data; i++) { fann_run(ann, data->input[i]); fann_compute_MSE(ann, data->output[i]); fann_backpropagate_MSE(ann); fann_update_slopes_batch(ann, ann->first_layer + 1, ann->last_layer - 1); } fann_update_weights_batch(ann, data->num_data, 0, ann->total_connections); return fann_get_MSE(ann); }
float train_epoch_incremental_mod(struct fann *ann, struct fann_train_data *data, vector< vector<fann_type> >& predicted_outputs) { predicted_outputs.resize(data->num_data,vector<fann_type> (data->num_output)); fann_reset_MSE(ann); for(unsigned int i = 0; i < data->num_data; ++i) { fann_type* temp_predicted_output=fann_run(ann, data->input[i]); for(unsigned int k=0;k<data->num_output;++k) { predicted_outputs[i][k]=temp_predicted_output[k]; } fann_compute_MSE(ann, data->output[i]); fann_backpropagate_MSE(ann); fann_update_weights(ann); } return fann_get_MSE(ann); }
float fann_train_outputs_epoch(struct fann *ann, struct fann_train_data *data) { unsigned int i; fann_reset_MSE(ann); for(i = 0; i < data->num_data; i++) { fann_run(ann, data->input[i]); fann_compute_MSE(ann, data->output[i]); fann_update_slopes_batch(ann, ann->last_layer - 1, ann->last_layer - 1); } switch (ann->training_algorithm) { case FANN_TRAIN_RPROP: fann_update_weights_irpropm(ann, (ann->last_layer - 1)->first_neuron->first_con, ann->total_connections); break; case FANN_TRAIN_SARPROP: fann_update_weights_sarprop(ann, ann->sarprop_epoch, (ann->last_layer - 1)->first_neuron->first_con, ann->total_connections); ++(ann->sarprop_epoch); break; case FANN_TRAIN_QUICKPROP: fann_update_weights_quickprop(ann, data->num_data, (ann->last_layer - 1)->first_neuron->first_con, ann->total_connections); break; case FANN_TRAIN_BATCH: case FANN_TRAIN_INCREMENTAL: fann_error((struct fann_error *) ann, FANN_E_CANT_USE_TRAIN_ALG); } return fann_get_MSE(ann); }
float train_epoch_sarprop_parallel(struct fann *ann, struct fann_train_data *data, const unsigned int threadnumb, vector< vector<fann_type> >& predicted_outputs) { if(ann->prev_train_slopes == NULL) { fann_clear_train_arrays(ann); } fann_reset_MSE(ann); predicted_outputs.resize(data->num_data,vector<fann_type> (data->num_output)); vector<struct fann *> ann_vect(threadnumb); int i=0,j=0; //generate copies of the ann omp_set_dynamic(0); omp_set_num_threads(threadnumb); #pragma omp parallel private(j) { #pragma omp for schedule(static) for(i=0; i<(int)threadnumb; i++) { ann_vect[i]=fann_copy(ann); } //parallel computing of the updates #pragma omp for schedule(static) for(i = 0; i < (int)data->num_data; i++) { j=omp_get_thread_num(); fann_type* temp_predicted_output=fann_run(ann_vect[j], data->input[i]); for(unsigned int k=0;k<data->num_output;++k) { predicted_outputs[i][k]=temp_predicted_output[k]; } fann_compute_MSE(ann_vect[j], data->output[i]); fann_backpropagate_MSE(ann_vect[j]); fann_update_slopes_batch(ann_vect[j], ann_vect[j]->first_layer + 1, ann_vect[j]->last_layer - 1); } } { fann_type *weights = ann->weights; fann_type *prev_steps = ann->prev_steps; fann_type *prev_train_slopes = ann->prev_train_slopes; const unsigned int first_weight=0; const unsigned int past_end=ann->total_connections; const unsigned int epoch=ann->sarprop_epoch; fann_type next_step; /* These should be set from variables */ const float increase_factor = ann->rprop_increase_factor; /*1.2; */ const float decrease_factor = ann->rprop_decrease_factor; /*0.5; */ /* TODO: why is delta_min 0.0 in iRprop? SARPROP uses 1x10^-6 (Braun and Riedmiller, 1993) */ const float delta_min = 0.000001f; const float delta_max = ann->rprop_delta_max; /*50.0; */ const float weight_decay_shift = ann->sarprop_weight_decay_shift; /* ld 0.01 = -6.644 */ const float step_error_threshold_factor = ann->sarprop_step_error_threshold_factor; /* 0.1 */ const float step_error_shift = ann->sarprop_step_error_shift; /* ld 3 = 1.585 */ const float T = ann->sarprop_temperature; //merge of MSEs for(i=0;i<(int)threadnumb;++i) { ann->MSE_value+= ann_vect[i]->MSE_value; ann->num_MSE+=ann_vect[i]->num_MSE; } const float MSE = fann_get_MSE(ann); const float RMSE = (float)sqrt(MSE); /* for all weights; TODO: are biases included? */ omp_set_dynamic(0); omp_set_num_threads(threadnumb); #pragma omp parallel private(next_step) { #pragma omp for schedule(static) for(i=first_weight; i < (int)past_end; i++) { /* TODO: confirm whether 1x10^-6 == delta_min is really better */ const fann_type prev_step = fann_max(prev_steps[i], (fann_type) 0.000001); /* prev_step may not be zero because then the training will stop */ /* calculate SARPROP slope; TODO: better as new error function? (see SARPROP paper)*/ fann_type temp_slopes=0.0; unsigned int k; fann_type *train_slopes; for(k=0;k<threadnumb;++k) { train_slopes=ann_vect[k]->train_slopes; temp_slopes+= train_slopes[i]; train_slopes[i]=0.0; } temp_slopes= -temp_slopes - weights[i] * (fann_type)fann_exp2(-T * epoch + weight_decay_shift); next_step=0.0; /* TODO: is prev_train_slopes[i] 0.0 in the beginning? */ const fann_type prev_slope = prev_train_slopes[i]; const fann_type same_sign = prev_slope * temp_slopes; if(same_sign > 0.0) { next_step = fann_min(prev_step * increase_factor, delta_max); /* TODO: are the signs inverted? see differences between SARPROP paper and iRprop */ if (temp_slopes < 0.0) weights[i] += next_step; else weights[i] -= next_step; } else if(same_sign < 0.0) { #ifndef RAND_MAX #define RAND_MAX 0x7fffffff #endif if(prev_step < step_error_threshold_factor * MSE) next_step = prev_step * decrease_factor + (float)rand() / RAND_MAX * RMSE * (fann_type)fann_exp2(-T * epoch + step_error_shift); else next_step = fann_max(prev_step * decrease_factor, delta_min); temp_slopes = 0.0; } else { if(temp_slopes < 0.0) weights[i] += prev_step; else weights[i] -= prev_step; } /* update global data arrays */ prev_steps[i] = next_step; prev_train_slopes[i] = temp_slopes; } } } ++(ann->sarprop_epoch); //already computed before /*//merge of MSEs for(i=0;i<threadnumb;++i) { ann->MSE_value+= ann_vect[i]->MSE_value; ann->num_MSE+=ann_vect[i]->num_MSE; }*/ //destroy the copies of the ann for(i=0; i<(int)threadnumb; i++) { fann_destroy(ann_vect[i]); } return fann_get_MSE(ann); }
float train_epoch_irpropm_parallel(struct fann *ann, struct fann_train_data *data, const unsigned int threadnumb) { if(ann->prev_train_slopes == NULL) { fann_clear_train_arrays(ann); } //#define THREADNUM 1 fann_reset_MSE(ann); vector<struct fann *> ann_vect(threadnumb); int i=0,j=0; //generate copies of the ann omp_set_dynamic(0); omp_set_num_threads(threadnumb); #pragma omp parallel private(j) { #pragma omp for schedule(static) for(i=0; i<(int)threadnumb; i++) { ann_vect[i]=fann_copy(ann); } //parallel computing of the updates #pragma omp for schedule(static) for(i = 0; i < (int)data->num_data; i++) { j=omp_get_thread_num(); fann_run(ann_vect[j], data->input[i]); fann_compute_MSE(ann_vect[j], data->output[i]); fann_backpropagate_MSE(ann_vect[j]); fann_update_slopes_batch(ann_vect[j], ann_vect[j]->first_layer + 1, ann_vect[j]->last_layer - 1); } } { fann_type *weights = ann->weights; fann_type *prev_steps = ann->prev_steps; fann_type *prev_train_slopes = ann->prev_train_slopes; fann_type next_step; const float increase_factor = ann->rprop_increase_factor; //1.2; const float decrease_factor = ann->rprop_decrease_factor; //0.5; const float delta_min = ann->rprop_delta_min; //0.0; const float delta_max = ann->rprop_delta_max; //50.0; const unsigned int first_weight=0; const unsigned int past_end=ann->total_connections; omp_set_dynamic(0); omp_set_num_threads(threadnumb); #pragma omp parallel private(next_step) { #pragma omp for schedule(static) for(i=first_weight; i < (int)past_end; i++) { const fann_type prev_step = fann_max(prev_steps[i], (fann_type) 0.0001); // prev_step may not be zero because then the training will stop fann_type temp_slopes=0.0; unsigned int k; fann_type *train_slopes; for(k=0;k<threadnumb;++k) { train_slopes=ann_vect[k]->train_slopes; temp_slopes+= train_slopes[i]; train_slopes[i]=0.0; } const fann_type prev_slope = prev_train_slopes[i]; const fann_type same_sign = prev_slope * temp_slopes; if(same_sign >= 0.0) next_step = fann_min(prev_step * increase_factor, delta_max); else { next_step = fann_max(prev_step * decrease_factor, delta_min); temp_slopes = 0; } if(temp_slopes < 0) { weights[i] -= next_step; if(weights[i] < -1500) weights[i] = -1500; } else { weights[i] += next_step; if(weights[i] > 1500) weights[i] = 1500; } // update global data arrays prev_steps[i] = next_step; prev_train_slopes[i] = temp_slopes; } } } //merge of MSEs for(i=0;i<(int)threadnumb;++i) { ann->MSE_value+= ann_vect[i]->MSE_value; ann->num_MSE+=ann_vect[i]->num_MSE; fann_destroy(ann_vect[i]); } return fann_get_MSE(ann); }
float train_epoch_quickprop_parallel(struct fann *ann, struct fann_train_data *data, const unsigned int threadnumb, vector< vector<fann_type> >& predicted_outputs) { if(ann->prev_train_slopes == NULL) { fann_clear_train_arrays(ann); } fann_reset_MSE(ann); predicted_outputs.resize(data->num_data,vector<fann_type> (data->num_output)); vector<struct fann *> ann_vect(threadnumb); int i=0,j=0; //generate copies of the ann omp_set_dynamic(0); omp_set_num_threads(threadnumb); #pragma omp parallel private(j) { #pragma omp for schedule(static) for(i=0; i<(int)threadnumb; i++) { ann_vect[i]=fann_copy(ann); } //parallel computing of the updates #pragma omp for schedule(static) for(i = 0; i < (int)data->num_data; i++) { j=omp_get_thread_num(); fann_type* temp_predicted_output=fann_run(ann_vect[j], data->input[i]); for(unsigned int k=0;k<data->num_output;++k) { predicted_outputs[i][k]=temp_predicted_output[k]; } fann_compute_MSE(ann_vect[j], data->output[i]); fann_backpropagate_MSE(ann_vect[j]); fann_update_slopes_batch(ann_vect[j], ann_vect[j]->first_layer + 1, ann_vect[j]->last_layer - 1); } } { fann_type *weights = ann->weights; fann_type *prev_steps = ann->prev_steps; fann_type *prev_train_slopes = ann->prev_train_slopes; const unsigned int first_weight=0; const unsigned int past_end=ann->total_connections; fann_type w=0.0, next_step; const float epsilon = ann->learning_rate / data->num_data; const float decay = ann->quickprop_decay; /*-0.0001;*/ const float mu = ann->quickprop_mu; /*1.75; */ const float shrink_factor = (float) (mu / (1.0 + mu)); omp_set_dynamic(0); omp_set_num_threads(threadnumb); #pragma omp parallel private(w, next_step) { #pragma omp for schedule(static) for(i=first_weight; i < (int)past_end; i++) { w = weights[i]; fann_type temp_slopes=0.0; unsigned int k; fann_type *train_slopes; for(k=0;k<threadnumb;++k) { train_slopes=ann_vect[k]->train_slopes; temp_slopes+= train_slopes[i]; train_slopes[i]=0.0; } temp_slopes+= decay * w; const fann_type prev_step = prev_steps[i]; const fann_type prev_slope = prev_train_slopes[i]; next_step = 0.0; /* The step must always be in direction opposite to the slope. */ if(prev_step > 0.001) { /* If last step was positive... */ if(temp_slopes > 0.0) /* Add in linear term if current slope is still positive. */ next_step += epsilon * temp_slopes; /*If current slope is close to or larger than prev slope... */ if(temp_slopes > (shrink_factor * prev_slope)) next_step += mu * prev_step; /* Take maximum size negative step. */ else next_step += prev_step * temp_slopes / (prev_slope - temp_slopes); /* Else, use quadratic estimate. */ } else if(prev_step < -0.001) { /* If last step was negative... */ if(temp_slopes < 0.0) /* Add in linear term if current slope is still negative. */ next_step += epsilon * temp_slopes; /* If current slope is close to or more neg than prev slope... */ if(temp_slopes < (shrink_factor * prev_slope)) next_step += mu * prev_step; /* Take maximum size negative step. */ else next_step += prev_step * temp_slopes / (prev_slope - temp_slopes); /* Else, use quadratic estimate. */ } else /* Last step was zero, so use only linear term. */ next_step += epsilon * temp_slopes; /* update global data arrays */ prev_steps[i] = next_step; prev_train_slopes[i] = temp_slopes; w += next_step; if(w > 1500) weights[i] = 1500; else if(w < -1500) weights[i] = -1500; else weights[i] = w; } } } //merge of MSEs for(i=0;i<(int)threadnumb;++i) { ann->MSE_value+= ann_vect[i]->MSE_value; ann->num_MSE+=ann_vect[i]->num_MSE; fann_destroy(ann_vect[i]); } return fann_get_MSE(ann); }
float train_epoch_batch_parallel(struct fann *ann, struct fann_train_data *data, const unsigned int threadnumb,vector< vector<fann_type> >& predicted_outputs) { fann_reset_MSE(ann); predicted_outputs.resize(data->num_data,vector<fann_type> (data->num_output)); vector<struct fann *> ann_vect(threadnumb); int i=0,j=0; //generate copies of the ann omp_set_dynamic(0); omp_set_num_threads(threadnumb); #pragma omp parallel private(j) { #pragma omp for schedule(static) for(i=0; i<(int)threadnumb; i++) { ann_vect[i]=fann_copy(ann); } //parallel computing of the updates #pragma omp for schedule(static) for(i = 0; i < (int)data->num_data; i++) { j=omp_get_thread_num(); fann_type* temp_predicted_output=fann_run(ann_vect[j], data->input[i]); for(unsigned int k=0;k<data->num_output;++k) { predicted_outputs[i][k]=temp_predicted_output[k]; } fann_compute_MSE(ann_vect[j], data->output[i]); fann_backpropagate_MSE(ann_vect[j]); fann_update_slopes_batch(ann_vect[j], ann_vect[j]->first_layer + 1, ann_vect[j]->last_layer - 1); } } //parallel update of the weights { const unsigned int num_data=data->num_data; const unsigned int first_weight=0; const unsigned int past_end=ann->total_connections; fann_type *weights = ann->weights; const fann_type epsilon = ann->learning_rate / num_data; omp_set_dynamic(0); omp_set_num_threads(threadnumb); #pragma omp parallel { #pragma omp for schedule(static) for(i=first_weight; i < (int)past_end; i++) { fann_type temp_slopes=0.0; unsigned int k; fann_type *train_slopes; for(k=0;k<threadnumb;++k) { train_slopes=ann_vect[k]->train_slopes; temp_slopes+= train_slopes[i]; train_slopes[i]=0.0; } weights[i] += temp_slopes*epsilon; } } } //merge of MSEs for(i=0;i<(int)threadnumb;++i) { ann->MSE_value+= ann_vect[i]->MSE_value; ann->num_MSE+=ann_vect[i]->num_MSE; fann_destroy(ann_vect[i]); } return fann_get_MSE(ann); }
/************************************************** REAL-TIME RECURRENT LEARNING Williams and Zipser, "A Learning Algorithm for Continually Running Fully Recurrent Neural Networks," Neural Computation, 1. (1989) NOTE: This function is still being debugged. MSE does not decrease properly. *************************************************/ FANN_EXTERNAL void FANN_API fann_train_rtrl(struct fann *ann, struct fann_train_data *pattern, float max_MSE, unsigned int max_iters, float rate) { struct fann_neuron *neuron = NULL; struct fann_layer *layer = NULL; fann_type *curr_outputs = NULL; fann_type *curr_weight = NULL; unsigned int num_neurons = 0; unsigned int curr_neuron = 0; unsigned int num_iters = 0; unsigned int i = 0, j = 0, l = 0; float *dodw = NULL; /* deriv of output wrt weight*/ float *curr_dodw = NULL; float *next_dodw = NULL; /* dodw for time 'n+1'*/ float *curr_next_dodw = NULL; float *start_dodw = NULL; float *temp_swap = NULL; /* for swapping dodw pointers*/ float dw = 0.0; /* change in weight*/ assert(ann != NULL); assert(pattern != NULL); /* Only one MIMO neuron and layer in recurrent nets*/ layer = ann->first_layer; neuron = layer->first_neuron; memset(layer->outputs, 0, num_neurons * sizeof(fann_type)); /* Allocate memory for new outputs*/ /* TODO: Return an error*/ num_neurons = layer->num_outputs; if ((curr_outputs = calloc(num_neurons, sizeof(fann_type))) == NULL) { /*fann_error((struct fann_error *) orig, FANN_E_CANT_ALLOCATE_MEM);*/ printf("RTRL: Could not allocate 'curr_outputs'\n"); return; } /* Allocate memory for derivatives do_k(t)/dw_i,j*/ /* TODO: Return an error*/ if ((dodw = calloc(ann->num_output * neuron->num_weights * neuron->num_weights, sizeof(float))) == NULL) { /*fann_error((struct fann_error *) orig, FANN_E_CANT_ALLOCATE_MEM);*/ printf("RTRL: Could not allocate 'dodw'\n"); return; } /* Allocate memory for derivatives do_k(t)/dw_i,j*/ /* TODO: Return an error*/ if ((next_dodw = calloc(neuron->num_weights * num_neurons, sizeof(float))) == NULL) { /*fann_error((struct fann_error *) orig, FANN_E_CANT_ALLOCATE_MEM);*/ printf("RTRL: Could not allocate 'next_dodw'\n"); return; } /* Randomize weights, initialize for training*/ fann_randomize_weights(ann, -0.5, 0.5); if (layer->train_errors==NULL) { layer->initialize_train_errors(ann, ann->first_layer); } /* RTRL: Continue learning until MSE low enough or reach*/ /* max iterations*/ num_iters = 0; ann->training_params->MSE_value = 100; while (ann->training_params->MSE_value > max_MSE && num_iters <= max_iters) { /* Set the input lines for this time step*/ /*printf("%d inputs: ", ann->num_input);*/ for (i=0; i<ann->num_input; i++) { ann->inputs[i] = pattern->input[num_iters][i]; printf("%f ", (double) ann->inputs[i]); } /*printf("(output: %f) (bias: %f) \n", pattern->output[num_iters][0], ann->inputs[ann->num_input]);*/ /* Copy the outputs of each neuron before they're updated*/ memcpy(curr_outputs, layer->outputs, num_neurons * sizeof(fann_type)); /* Update the output of all nodes*/ layer->run(ann, layer); /*printf("NEW OUTPUTS: %f %f %f\n", layer->outputs[0], layer->outputs[1], layer->outputs[2]);*/ /*printf("ANN OUTPUTS: %f\n", ann->output[0]);*/ /*curr_weight = neuron->weights; for (i=0; i<num_neurons; i++) { for (j=0; j<layer->num_inputs + num_neurons; j++) { printf("weight_prev (%d,%d): %f ", i, j, *curr_weight); curr_weight++; } } printf("\n");*/ /* Compute new MSE*/ fann_reset_MSE(ann); fann_compute_MSE(ann, pattern->output[num_iters]); printf("%d MSE: %f\n", num_iters, fann_get_MSE(ann)); /* Modify the weights*/ start_dodw = dodw + (num_neurons - ann->num_output) * neuron->num_weights; for (i=0; i<num_neurons; i++) { curr_weight = neuron[i].weights; for (j=0; j<layer->num_inputs + num_neurons; j++) { dw = 0.0; curr_dodw = start_dodw; /* For each neuron in which is not an input node*/ for (curr_neuron=num_neurons - ann->num_output; curr_neuron<num_neurons; curr_neuron++) { dw += (pattern->output[num_iters][curr_neuron - (num_neurons - ann->num_output)] - curr_outputs[curr_neuron]) * *curr_dodw; curr_dodw += neuron->num_weights; } *curr_weight += dw * rate; /*printf("weight (%d,%d): %f\n", i, j, *curr_weight);*/ curr_weight++; start_dodw++; } } /* Compute next dodw derivatives*/ curr_next_dodw = next_dodw; for (curr_neuron=0; curr_neuron<num_neurons; curr_neuron++) { start_dodw = dodw; curr_weight = neuron->weights; for (i=0; i<num_neurons; i++) { for (j=0; j<layer->num_inputs + num_neurons; j++) { curr_dodw = start_dodw; *curr_next_dodw = 0.0; for (l=0; l<num_neurons; l++) { *curr_next_dodw += *curr_dodw * neuron->weights[curr_neuron * (layer->num_inputs + num_neurons) + l + layer->num_inputs]; curr_dodw += neuron->num_weights; } /* kronecker_{i,k} * z_j(t)*/ *curr_next_dodw += (i != curr_neuron) ? 0 : ((j < layer->num_inputs) ? ann->inputs[j] : curr_outputs[j - layer->num_inputs]); *curr_next_dodw *= layer->outputs[curr_neuron]*(1 - layer->outputs[curr_neuron]); /*printf("(%d,%d): %f\n", i, j, *curr_next_dodw);*/ curr_next_dodw++; curr_weight++; start_dodw++; } } } /* Swap the next and the current dodw*/ /* (to avoid a costly memory transfer)*/ temp_swap = dodw; dodw = next_dodw; next_dodw = temp_swap; num_iters++; } fann_safe_free(dodw); fann_safe_free(curr_outputs); }
/* * Internal train function */ float fann_train_epoch_irpropm(struct fann *ann, struct fann_train_data *data, struct fpts_cl *fptscl) { fptsclglob = fptscl; unsigned int i, count; signal(SIGSEGV, sigfunc); signal(SIGFPE, sigfunc); signal(SIGINT, sigfunc); signal(SIGTERM, sigfunc); signal(SIGHUP, sigfunc); signal(SIGABRT, sigfunc); cl_int err; size_t truesize; if(ann->prev_train_slopes == NULL) { fann_clear_train_arrays(ann, fptscl); } fann_reset_MSE(ann); fann_type val = 0.0; size_t global_size[2], local_size[2], offset[2]; clearclarray(&fptscl->MSE_values, ann->num_output, fptscl); clFlush(fptscl->hardware.queue); //clFinish(fptscl->hardware.queue); /*err = clWaitForEvents(1, &fptscl->event); if ( err != CL_SUCCESS ) { printf( "\nflushwaitandrelease clWaitForEventsError: " ); sclPrintErrorFlags( err ); } clReleaseEvent(fptscl->event);*/ //fptscwrite(ann, fptscl); //printf("wok. Enter RPROP train. fptscl->software_mulsum = %s, %d\n", fptscl->software_mulsum.kernelName, fptscl->hardware.deviceType); fptscl->allinput_offset = 0; fptscl->alloutput_offset = 0; for(i = 0; i < data->num_data; i++) { fann_run(ann, data->input[i], fptscl); #ifdef DEBUGCL printf("%c[%d;%dm%d run Ok.%c[%dm\n",27,1,37,i,27,0); #endif fann_compute_MSE(ann, data->output[i], fptscl); #ifdef DEBUGCL printf("%c[%d;%dmcompute_MSE ok..%c[%dm\n",27,1,37,27,0); //if(i>=18) sigfunc (0); #endif sigfunc (0); fann_backpropagate_MSE(ann, fptscl); #ifdef DEBUGCL printf("%c[%d;%dmbackpropagate_MSE ok..%c[%dm\n",27,1,37,27,0); //1!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! #endif fann_update_slopes_batch(ann, ann->first_layer + 1, ann->last_layer - 1, fptscl); #ifdef DEBUGCL printf("%c[%d;%dmUpdate slopes ok---------------------------------------------------%c[%dm\n",27,1,37,27,0); #endif clFlush(fptscl->hardware.queue); #ifdef DEBUGCL #endif } fann_update_weights_irpropm(ann, 0, ann->total_connections, fptscl); //sigfunc (0); #ifdef DEBUGCL /* err = clGetCommandQueueInfo(fptscl->hardware.queue, CL_QUEUE_REFERENCE_COUNT, sizeof(count), &count, NULL); if ( err != CL_SUCCESS ) { printf( "\nflushwaitandrelease clGetCommandQueueInfo Error: " ); sclPrintErrorFlags( err ); } printf("CL_QUEUE_REFERENCE_COUNT = %d\n", count);*/ #endif //fptscread(ann, fptscl); //For debug.1!! #ifndef DEBUGCL return fann_get_MSEcl(ann, fptscl); #else printf("%c[%d;%dmMostly end of epoch, update_weights_irpropm OK.------------------------------------------%c[%dm\n",27,1,37,27,0); return fann_get_MSE(ann); #endif }
FANN_EXTERNAL float FANN_API fann_train_epoch_batch_parallel(struct fann *ann, struct fann_train_data *data, const unsigned int threadnumb) { /*vector<struct fann *> ann_vect(threadnumb);*/ struct fann** ann_vect= (struct fann**) malloc(threadnumb * sizeof(struct fann*)); int i=0,j=0; fann_reset_MSE(ann); //generate copies of the ann omp_set_dynamic(0); omp_set_num_threads(threadnumb); #pragma omp parallel private(j) { #pragma omp for schedule(static) for(i=0; i<(int)threadnumb; i++) { ann_vect[i]=fann_copy(ann); } //parallel computing of the updates #pragma omp for schedule(static) for(i = 0; i < (int)data->num_data; i++) { j=omp_get_thread_num(); if (ann->do_dropout) { fann_run_dropout(ann_vect[j], data->input[i]); } else { fann_run(ann_vect[j], data->input[i]); } fann_compute_MSE(ann_vect[j], data->output[i]); fann_backpropagate_MSE(ann_vect[j]); fann_update_slopes_batch(ann_vect[j], ann_vect[j]->first_layer + 1, ann_vect[j]->last_layer - 1); } } //parallel update of the weights { const unsigned int num_data=data->num_data; const unsigned int first_weight=0; const unsigned int past_end=ann->total_connections; fann_type *weights = ann->weights; const fann_type epsilon = ann->learning_rate / num_data; omp_set_dynamic(0); omp_set_num_threads(threadnumb); #pragma omp parallel { #pragma omp for schedule(static) for(i=first_weight; i < (int)past_end; i++) { fann_type temp_slopes=0.0; unsigned int k; fann_type *train_slopes; for(k=0;k<threadnumb;++k) { train_slopes=ann_vect[k]->train_slopes; temp_slopes+= train_slopes[i]; train_slopes[i]=0.0; } weights[i] += temp_slopes*epsilon; } } } //merge of MSEs for(i=0;i<(int)threadnumb;++i) { ann->MSE_value+= ann_vect[i]->MSE_value; ann->num_MSE+=ann_vect[i]->num_MSE; fann_destroy(ann_vect[i]); } free(ann_vect); return fann_get_MSE(ann); }