// fann_randomize_weights Give each connection a random weight between min_weight and max_weight
int sci_fann_randomize_weights(char * fname)
{
int * pi_minweight_addr = NULL;
int * pi_maxweight_addr = NULL;
int res;
double minweight = 0.0, maxweight = 0.0;
struct fann * result_ann = NULL;
SciErr _sciErr;
if ((Rhs!=3)&&(Lhs!=1))
{
Scierror(999,"%s usage: ann_out = %s(ann_in, weight_min, weight_max)", fname, fname);
return 0;
}
// Get the ann
res = detect_fannlist(1);
if (res==-1) return 0;
result_ann = createCFannStructFromScilabFannStruct(1,&res);
if (res==-1) return 0;
if (result_ann==NULL)
{
Scierror(999,"%s: Problem while creating the fann scilab structure\n",fname);
return 0;
}
_sciErr = getVarAddressFromPosition(pvApiCtx, 2, &pi_minweight_addr);
if (_sciErr.iErr)
{
printError(&_sciErr, 0);
return 0;
}
getScalarDouble(pvApiCtx, pi_minweight_addr, &minweight);
_sciErr = getVarAddressFromPosition(pvApiCtx, 3, &pi_maxweight_addr);
if (_sciErr.iErr)
{
printError(&_sciErr, 0);
return 0;
}
getScalarDouble(pvApiCtx, pi_maxweight_addr, &maxweight);
fann_randomize_weights(result_ann,(fann_type)minweight,(fann_type)maxweight);
res = createScilabFannStructFromCFannStruct(result_ann, Rhs + 1);
if (res==-1) return 0;
LhsVar(1) = Rhs + 1;
return 0;
}
struct fann *random_network(){
// create and randomize activation function
struct fann *ann = fann_create_standard(num_layers,num_input,num_hidden_nodes,num_output);
fann_set_activation_function_hidden(ann,FANN_SIGMOID);
fann_set_activation_function_output(ann,FANN_SIGMOID);
// randomize weights
fann_randomize_weights(ann,-1,1);
// return
return ann;
}
bool Trainer::Train(const AnnData& data, const float* random_weight_limit,
std::size_t max_epochs, std::size_t epochs_between_reports,
float desired_error,
float* mse, std::size_t* bit_fail) {
if (random_weight_limit == NULL)
fann_init_weights(ann_, data.data());
else
fann_randomize_weights(ann_, -*random_weight_limit, *random_weight_limit);
fann_shuffle_train_data(data.data());
fann_train_on_data(ann_, data.data(), max_epochs, epochs_between_reports,
desired_error);
return GetMseAndBitFail(ann_, &mse, &bit_fail);
}
/***
* @function rspamd_fann.create(nlayers, [layer1, ... layern])
* Creates new neural network with `nlayers` that contains `layer1`...`layern`
* neurons in each layer
* @param {number} nlayers number of layers
* @param {number} layerI number of neurons in each layer
* @return {fann} fann object
*/
static gint
lua_fann_create (lua_State *L)
{
#ifndef WITH_FANN
return 0;
#else
struct fann *f, **pfann;
guint nlayers, *layers, i;
nlayers = luaL_checknumber (L, 1);
if (nlayers > 0) {
layers = g_malloc (nlayers * sizeof (layers[0]));
if (lua_type (L, 2) == LUA_TNUMBER) {
for (i = 0; i < nlayers; i ++) {
layers[i] = luaL_checknumber (L, i + 2);
}
}
else if (lua_type (L, 2) == LUA_TTABLE) {
for (i = 0; i < nlayers; i ++) {
lua_rawgeti (L, 2, i + 1);
layers[i] = luaL_checknumber (L, -1);
lua_pop (L, 1);
}
}
f = fann_create_standard_array (nlayers, layers);
fann_set_activation_function_hidden (f, FANN_SIGMOID_SYMMETRIC);
fann_set_activation_function_output (f, FANN_SIGMOID_SYMMETRIC);
fann_set_training_algorithm (f, FANN_TRAIN_INCREMENTAL);
fann_randomize_weights (f, 0, 1);
if (f != NULL) {
pfann = lua_newuserdata (L, sizeof (gpointer));
*pfann = f;
rspamd_lua_setclass (L, "rspamd{fann}", -1);
}
else {
lua_pushnil (L);
}
g_free (layers);
}
else {
lua_pushnil (L);
}
return 1;
#endif
}
bool ViFann::setWeights(const Weights &initialization, const qreal &minimum, const qreal &maximum)
{
if(mNetwork == NULL) return false;
mWeights = initialization;
if(initialization == Random)
{
fann_randomize_weights(mNetwork, minimum, maximum);
mWeightsMinimum = minimum;
mWeightsMaximum = maximum;
}
else if(initialization == WidrowNguyen)
{
// Create fake training set so that FANN can determine the min and max values
fann_train_data *data = fann_create_train(1, 2, 1);
data->input[0][0] = 1;
data->input[0][1] = -1;
fann_init_weights(mNetwork, data);
fann_destroy_train(data);
}
return true;
}
int main()
{
struct fann *ann;
struct fann_train_data *train_data, *test_data;
const float desired_error = (const float)0.0;
unsigned int max_neurons = 30;
unsigned int neurons_between_reports = 1;
unsigned int bit_fail_train, bit_fail_test;
float mse_train, mse_test;
unsigned int i = 0;
fann_type *output;
fann_type steepness;
int multi = 0;
enum fann_activationfunc_enum activation;
enum fann_train_enum training_algorithm = FANN_TRAIN_RPROP;
printf("Reading data.\n");
train_data = fann_read_train_from_file("../benchmarks/datasets/parity8.train");
test_data = fann_read_train_from_file("../benchmarks/datasets/parity8.test");
fann_scale_train_data(train_data, -1, 1);
fann_scale_train_data(test_data, -1, 1);
printf("Creating network.\n");
ann = fann_create_shortcut(2, fann_num_input_train_data(train_data), fann_num_output_train_data(train_data));
fann_set_training_algorithm(ann, training_algorithm);
fann_set_activation_function_hidden(ann, FANN_SIGMOID_SYMMETRIC);
fann_set_activation_function_output(ann, FANN_LINEAR);
fann_set_train_error_function(ann, FANN_ERRORFUNC_LINEAR);
if(!multi)
{
/*steepness = 0.5;*/
steepness = 1;
fann_set_cascade_activation_steepnesses(ann, &steepness, 1);
/*activation = FANN_SIN_SYMMETRIC;*/
activation = FANN_SIGMOID_SYMMETRIC;
fann_set_cascade_activation_functions(ann, &activation, 1);
fann_set_cascade_num_candidate_groups(ann, 8);
}
if(training_algorithm == FANN_TRAIN_QUICKPROP)
{
fann_set_learning_rate(ann, 0.35);
fann_randomize_weights(ann, -2.0,2.0);
}
fann_set_bit_fail_limit(ann, 0.9);
fann_set_train_stop_function(ann, FANN_STOPFUNC_BIT);
fann_print_parameters(ann);
fann_save(ann, "cascade_train2.net");
printf("Training network.\n");
fann_cascadetrain_on_data(ann, train_data, max_neurons, neurons_between_reports, desired_error);
fann_print_connections(ann);
mse_train = fann_test_data(ann, train_data);
bit_fail_train = fann_get_bit_fail(ann);
mse_test = fann_test_data(ann, test_data);
bit_fail_test = fann_get_bit_fail(ann);
printf("\nTrain error: %f, Train bit-fail: %d, Test error: %f, Test bit-fail: %d\n\n",
mse_train, bit_fail_train, mse_test, bit_fail_test);
for(i = 0; i < train_data->num_data; i++)
{
output = fann_run(ann, train_data->input[i]);
if((train_data->output[i][0] >= 0 && output[0] <= 0) ||
(train_data->output[i][0] <= 0 && output[0] >= 0))
{
printf("ERROR: %f does not match %f\n", train_data->output[i][0], output[0]);
}
}
printf("Saving network.\n");
fann_save(ann, "cascade_train.net");
printf("Cleaning up.\n");
fann_destroy_train(train_data);
fann_destroy_train(test_data);
fann_destroy(ann);
return 0;
}
/**************************************************
REAL-TIME RECURRENT LEARNING
Williams and Zipser, "A Learning Algorithm for
Continually Running Fully Recurrent Neural
Networks," Neural Computation, 1. (1989)
NOTE: This function is still being debugged.
MSE does not decrease properly.
*************************************************/
FANN_EXTERNAL void FANN_API fann_train_rtrl(struct fann *ann, struct fann_train_data *pattern,
float max_MSE, unsigned int max_iters, float rate)
{
struct fann_neuron *neuron = NULL;
struct fann_layer *layer = NULL;
fann_type *curr_outputs = NULL;
fann_type *curr_weight = NULL;
unsigned int num_neurons = 0;
unsigned int curr_neuron = 0;
unsigned int num_iters = 0;
unsigned int i = 0, j = 0, l = 0;
float *dodw = NULL; /* deriv of output wrt weight*/
float *curr_dodw = NULL;
float *next_dodw = NULL; /* dodw for time 'n+1'*/
float *curr_next_dodw = NULL;
float *start_dodw = NULL;
float *temp_swap = NULL; /* for swapping dodw pointers*/
float dw = 0.0; /* change in weight*/
assert(ann != NULL);
assert(pattern != NULL);
/* Only one MIMO neuron and layer in recurrent nets*/
layer = ann->first_layer;
neuron = layer->first_neuron;
memset(layer->outputs, 0, num_neurons * sizeof(fann_type));
/* Allocate memory for new outputs*/
/* TODO: Return an error*/
num_neurons = layer->num_outputs;
if ((curr_outputs = calloc(num_neurons, sizeof(fann_type))) == NULL)
{
/*fann_error((struct fann_error *) orig, FANN_E_CANT_ALLOCATE_MEM);*/
printf("RTRL: Could not allocate 'curr_outputs'\n");
return;
}
/* Allocate memory for derivatives do_k(t)/dw_i,j*/
/* TODO: Return an error*/
if ((dodw = calloc(ann->num_output * neuron->num_weights * neuron->num_weights, sizeof(float))) == NULL)
{
/*fann_error((struct fann_error *) orig, FANN_E_CANT_ALLOCATE_MEM);*/
printf("RTRL: Could not allocate 'dodw'\n");
return;
}
/* Allocate memory for derivatives do_k(t)/dw_i,j*/
/* TODO: Return an error*/
if ((next_dodw = calloc(neuron->num_weights * num_neurons, sizeof(float))) == NULL)
{
/*fann_error((struct fann_error *) orig, FANN_E_CANT_ALLOCATE_MEM);*/
printf("RTRL: Could not allocate 'next_dodw'\n");
return;
}
/* Randomize weights, initialize for training*/
fann_randomize_weights(ann, -0.5, 0.5);
if (layer->train_errors==NULL)
{
layer->initialize_train_errors(ann, ann->first_layer);
}
/* RTRL: Continue learning until MSE low enough or reach*/
/* max iterations*/
num_iters = 0;
ann->training_params->MSE_value = 100;
while (ann->training_params->MSE_value > max_MSE && num_iters <= max_iters)
{
/* Set the input lines for this time step*/
/*printf("%d inputs: ", ann->num_input);*/
for (i=0; i<ann->num_input; i++)
{
ann->inputs[i] = pattern->input[num_iters][i];
printf("%f ", (double) ann->inputs[i]);
}
/*printf("(output: %f) (bias: %f) \n", pattern->output[num_iters][0], ann->inputs[ann->num_input]);*/
/* Copy the outputs of each neuron before they're updated*/
memcpy(curr_outputs, layer->outputs, num_neurons * sizeof(fann_type));
/* Update the output of all nodes*/
layer->run(ann, layer);
/*printf("NEW OUTPUTS: %f %f %f\n", layer->outputs[0], layer->outputs[1], layer->outputs[2]);*/
/*printf("ANN OUTPUTS: %f\n", ann->output[0]);*/
/*curr_weight = neuron->weights;
for (i=0; i<num_neurons; i++)
{
for (j=0; j<layer->num_inputs + num_neurons; j++)
{
printf("weight_prev (%d,%d): %f ", i, j, *curr_weight);
curr_weight++;
}
}
printf("\n");*/
/* Compute new MSE*/
fann_reset_MSE(ann);
fann_compute_MSE(ann, pattern->output[num_iters]);
printf("%d MSE: %f\n", num_iters, fann_get_MSE(ann));
/* Modify the weights*/
start_dodw = dodw + (num_neurons - ann->num_output) * neuron->num_weights;
for (i=0; i<num_neurons; i++)
{
curr_weight = neuron[i].weights;
for (j=0; j<layer->num_inputs + num_neurons; j++)
{
dw = 0.0;
curr_dodw = start_dodw;
/* For each neuron in which is not an input node*/
for (curr_neuron=num_neurons - ann->num_output; curr_neuron<num_neurons; curr_neuron++)
{
dw += (pattern->output[num_iters][curr_neuron - (num_neurons - ann->num_output)] -
curr_outputs[curr_neuron]) * *curr_dodw;
curr_dodw += neuron->num_weights;
}
*curr_weight += dw * rate;
/*printf("weight (%d,%d): %f\n", i, j, *curr_weight);*/
curr_weight++;
start_dodw++;
}
}
/* Compute next dodw derivatives*/
curr_next_dodw = next_dodw;
for (curr_neuron=0; curr_neuron<num_neurons; curr_neuron++)
{
start_dodw = dodw;
curr_weight = neuron->weights;
for (i=0; i<num_neurons; i++)
{
for (j=0; j<layer->num_inputs + num_neurons; j++)
{
curr_dodw = start_dodw;
*curr_next_dodw = 0.0;
for (l=0; l<num_neurons; l++)
{
*curr_next_dodw += *curr_dodw *
neuron->weights[curr_neuron * (layer->num_inputs + num_neurons) + l + layer->num_inputs];
curr_dodw += neuron->num_weights;
}
/* kronecker_{i,k} * z_j(t)*/
*curr_next_dodw += (i != curr_neuron) ? 0 :
((j < layer->num_inputs) ? ann->inputs[j] : curr_outputs[j - layer->num_inputs]);
*curr_next_dodw *= layer->outputs[curr_neuron]*(1 - layer->outputs[curr_neuron]);
/*printf("(%d,%d): %f\n", i, j, *curr_next_dodw);*/
curr_next_dodw++;
curr_weight++;
start_dodw++;
}
}
}
/* Swap the next and the current dodw*/
/* (to avoid a costly memory transfer)*/
temp_swap = dodw;
dodw = next_dodw;
next_dodw = temp_swap;
num_iters++;
}
fann_safe_free(dodw);
fann_safe_free(curr_outputs);
}
void OcrNet::Train(cv::Mat &image, std::map< int, std::set<ImageRegion> > &image_lines, std::string &char_set)
{
std::map<int, std::set<ImageRegion> >::iterator image_lines_it;
std::set<ImageRegion> regions;
std::set<ImageRegion>::iterator regions_it;
unsigned int char_count = 0, char_length = 0, repeat = 2, index = 0;
struct fann_train_data **data;
fann_type **inputs, **outputs;
char_length = char_set.length();
data = new struct fann_train_data* [char_length];
for(uint j=0; j < char_length; j++)
{
data[j] = new struct fann_train_data;
data[j]->num_data = repeat;
data[j]->num_input = Ann[j]->num_input;
data[j]->num_output = Ann[j]->num_output;
inputs = new fann_type*[data[j]->num_data];
outputs = new fann_type*[data[j]->num_data];
data[j]->input = inputs;
data[j]->output = outputs;
for(uint i = 0; i < repeat; i++, char_count = 0)
{
for(image_lines_it = image_lines.begin();
char_count < char_length && image_lines_it != image_lines.end();
image_lines_it++, char_count++)
{
regions = (std::set<ImageRegion>) image_lines_it->second;
for(regions_it = regions.begin();
char_count < char_length && regions_it != regions.end();
regions_it++, char_count++)
{
index = i * char_length + char_count;
inputs[index] = new fann_type[data->num_input];
outputs[index] = new fann_type[data->num_output];
CreateInput(inputs[index], data->num_input, image, *regions_it);
CreateOutput(outputs[index], data->num_output, char_count);
std::cout << "Repeat Index: " << i << ", index: " << index << std::endl;
}
}
}
fann_randomize_weights(Ann[j], -0.001, 0.0);
// fann_init_weights(Ann, data);
fann_train_on_data(Ann[j], &data[j], MaxEpochs, 10, Error);
for(unsigned int i=0; i < data[j]->num_data; i++)
{
delete [] inputs[i];
delete [] outputs[i];
}
}
delete data;
delete [] inputs;
delete [] outputs;
}
void BasicBrain::randomize() {
fann_randomize_weights(nn_, -1, 1);
}
/***
* @function rspamd_fann.create_full(params)
* Creates new neural network with parameters:
* - `layers` {table/numbers}: table of layers in form: {N1, N2, N3 ... Nn} where N is number of neurons in a layer
* - `activation_hidden` {string}: activation function type for hidden layers (`tanh` by default)
* - `activation_output` {string}: activation function type for output layer (`tanh` by default)
* - `sparsed` {float}: create sparsed ANN, where number is a coefficient for sparsing
* - `learn` {string}: learning algorithm (quickprop, rprop or incremental)
* - `randomize` {boolean}: randomize weights (true by default)
* @return {fann} fann object
*/
static gint
lua_fann_create_full (lua_State *L)
{
#ifndef WITH_FANN
return 0;
#else
struct fann *f, **pfann;
guint nlayers, *layers, i;
const gchar *activation_hidden = NULL, *activation_output, *learn_alg = NULL;
gdouble sparsed = 0.0;
gboolean randomize_ann = TRUE;
GError *err = NULL;
if (lua_type (L, 1) == LUA_TTABLE) {
lua_pushstring (L, "layers");
lua_gettable (L, 1);
if (lua_type (L, -1) != LUA_TTABLE) {
return luaL_error (L, "bad layers attribute");
}
nlayers = rspamd_lua_table_size (L, -1);
if (nlayers < 2) {
return luaL_error (L, "bad layers attribute");
}
layers = g_new0 (guint, nlayers);
for (i = 0; i < nlayers; i ++) {
lua_rawgeti (L, -1, i + 1);
layers[i] = luaL_checknumber (L, -1);
lua_pop (L, 1);
}
lua_pop (L, 1); /* Table */
if (!rspamd_lua_parse_table_arguments (L, 1, &err,
"sparsed=N;randomize=B;learn=S;activation_hidden=S;activation_output=S",
&sparsed, &randomize_ann, &learn_alg, &activation_hidden, &activation_output)) {
g_free (layers);
if (err) {
gint r;
r = luaL_error (L, "invalid arguments: %s", err->message);
g_error_free (err);
return r;
}
else {
return luaL_error (L, "invalid arguments");
}
}
if (sparsed != 0.0) {
f = fann_create_standard_array (nlayers, layers);
}
else {
f = fann_create_sparse_array (sparsed, nlayers, layers);
}
if (f != NULL) {
pfann = lua_newuserdata (L, sizeof (gpointer));
*pfann = f;
rspamd_lua_setclass (L, "rspamd{fann}", -1);
}
else {
g_free (layers);
return luaL_error (L, "cannot create fann");
}
fann_set_activation_function_hidden (f,
string_to_activation_func (activation_hidden));
fann_set_activation_function_output (f,
string_to_activation_func (activation_output));
fann_set_training_algorithm (f, string_to_learn_alg (learn_alg));
if (randomize_ann) {
fann_randomize_weights (f, 0, 1);
}
g_free (layers);
}
else {
return luaL_error (L, "bad arguments");
}
return 1;
#endif
}
struct fann * setup_net(struct fann_train_data * data)
{
struct fann *ann;
#if MIMO_FANN
#if OPTIMIZE == 0
ann = fann_create_standard( 3, data->num_input, H_DIM, data->num_output);
fann_set_activation_function_hidden(ann, FANN_SIGMOID_SYMMETRIC);
fann_set_activation_function_output(ann, FANN_SIGMOID_SYMMETRIC);
#endif
#if OPTIMIZE == 1
unsigned int i, j;
struct fann_descr *descr=(struct fann_descr*) calloc(1, sizeof(struct fann_descr));
fann_setup_descr(descr, 2, data->num_input);
i=0;
fann_setup_layer_descr(
&(descr->layers_descr[i]),
"connected_any_any",
1,
NULL
);
for (j=0; j< descr->layers_descr[i].num_neurons; j++)
{
fann_setup_neuron_descr(
descr->layers_descr[i].neurons_descr+j,
H_DIM,
"scalar_rprop_sigmoid_symmetric",
NULL
);
}
i=1;
fann_setup_layer_descr(
&(descr->layers_descr[i]),
"connected_any_any",
1,
NULL
);
for (j=0; j< descr->layers_descr[i].num_neurons; j++)
{
fann_setup_neuron_descr(
descr->layers_descr[i].neurons_descr+j,
data->num_output,
"scalar_rprop_sigmoid_symmetric",
NULL
);
}
ann = fann_create_from_descr( descr );
#endif
#if OPTIMIZE >= 2
{
unsigned int layers[] = { data->num_input, H_DIM, data->num_output };
/*char *type;
asprintf(&type, "%s_%s_%s", vals(implementation), vals(algorithm), vals(activation));*/
ann = fann_create_standard_array_typed(layer_type, neuron_type, 3, layers);
}
#endif
#else /*MIMO_FANN*/
#ifdef SPARSE
ann = fann_create_sparse( SPARSE, 3, data->num_input, H_DIM, data->num_output);
#else
ann = fann_create_standard( 3, data->num_input, H_DIM, data->num_output);
#endif
fann_set_activation_function_hidden(ann, FANN_SIGMOID_SYMMETRIC);
fann_set_activation_function_output(ann, FANN_SIGMOID_SYMMETRIC);
#endif /*MIMO_FANN*/
fann_set_train_stop_function(ann, FANN_STOPFUNC_BIT);
fann_set_bit_fail_limit(ann, 0.01f);
fann_set_activation_steepness_hidden(ann, 1);
fann_set_activation_steepness_output(ann, 1);
#if INIT_WEIGHTS == 1
fann_randomize_weights(ann,0,1);
#endif
#if INIT_WEIGHTS == 2
fann_init_weights(ann, data);
#endif
#ifdef USE_RPROP
fann_set_training_algorithm(ann, FANN_TRAIN_RPROP);
#else
fann_set_training_algorithm(ann, FANN_TRAIN_BATCH);
#endif
return ann;
}
