/**
* @method rspamd_fann:train(inputs, outputs)
* Trains neural network with samples. Inputs and outputs should be tables of
* equal size, each row in table should be N inputs and M outputs, e.g.
* {0, 1, 1} -> {0}
* @param {table} inputs input samples
* @param {table} outputs output samples
* @return {number} number of samples learned
*/
static gint
lua_fann_train (lua_State *L)
{
#ifndef WITH_FANN
return 0;
#else
struct fann *f = rspamd_lua_check_fann (L, 1);
guint ninputs, noutputs, j;
fann_type *cur_input, *cur_output;
gboolean ret = FALSE;
if (f != NULL) {
/* First check sanity, call for table.getn for that */
ninputs = rspamd_lua_table_size (L, 2);
noutputs = rspamd_lua_table_size (L, 3);
if (ninputs != fann_get_num_input (f) ||
noutputs != fann_get_num_output (f)) {
msg_err ("bad number of inputs(%d, expected %d) and "
"output(%d, expected %d) args for train",
ninputs, fann_get_num_input (f),
noutputs, fann_get_num_output (f));
}
else {
cur_input = g_malloc (ninputs * sizeof (fann_type));
for (j = 0; j < ninputs; j ++) {
lua_rawgeti (L, 2, j + 1);
cur_input[j] = lua_tonumber (L, -1);
lua_pop (L, 1);
}
cur_output = g_malloc (noutputs * sizeof (fann_type));
for (j = 0; j < noutputs; j++) {
lua_rawgeti (L, 3, j + 1);
cur_output[j] = lua_tonumber (L, -1);
lua_pop (L, 1);
}
fann_train (f, cur_input, cur_output);
g_free (cur_input);
g_free (cur_output);
ret = TRUE;
}
}
lua_pushboolean (L, ret);
return 1;
#endif
}
/**
* @method rspamd_fann:test(inputs)
* Tests neural network with samples. Inputs is a single sample of input data.
* The function returns table of results, e.g.:
* {0, 1, 1} -> {0}
* @param {table} inputs input sample
* @return {table/number} outputs values
*/
static gint
lua_fann_test (lua_State *L)
{
#ifndef WITH_FANN
return 0;
#else
struct fann *f = rspamd_lua_check_fann (L, 1);
guint ninputs, noutputs, i, tbl_idx = 2;
fann_type *cur_input, *cur_output;
if (f != NULL) {
/* First check sanity, call for table.getn for that */
if (lua_isnumber (L, 2)) {
ninputs = lua_tonumber (L, 2);
tbl_idx = 3;
}
else {
ninputs = rspamd_lua_table_size (L, 2);
if (ninputs == 0) {
msg_err ("empty inputs number");
lua_pushnil (L);
return 1;
}
}
cur_input = g_malloc0 (ninputs * sizeof (fann_type));
for (i = 0; i < ninputs; i++) {
lua_rawgeti (L, tbl_idx, i + 1);
cur_input[i] = lua_tonumber (L, -1);
lua_pop (L, 1);
}
cur_output = fann_run (f, cur_input);
noutputs = fann_get_num_output (f);
lua_createtable (L, noutputs, 0);
for (i = 0; i < noutputs; i ++) {
lua_pushnumber (L, cur_output[i]);
lua_rawseti (L, -2, i + 1);
}
g_free (cur_input);
}
else {
lua_pushnil (L);
}
return 1;
#endif
}
static void
lua_url_table_inserter (struct rspamd_url *url, gsize start_offset,
gsize end_offset, gpointer ud)
{
lua_State *L = ud;
struct rspamd_lua_url *lua_url;
gint n;
n = rspamd_lua_table_size (L, -1);
lua_url = lua_newuserdata (L, sizeof (struct rspamd_lua_url));
rspamd_lua_setclass (L, "rspamd{url}", -1);
lua_url->url = url;
lua_pushinteger (L, n + 1);
lua_pushlstring (L, url->string, url->urllen);
lua_settable (L, -3);
}
/**
* @method rspamd_fann:train_threaded(inputs, outputs, callback, event_base, {params})
* Trains neural network with batch of samples. Inputs and outputs should be tables of
* equal size, each row in table should be N inputs and M outputs, e.g.
* {{0, 1, 1}, ...} -> {{0}, {1} ...}
* @param {table} inputs input samples
* @param {table} outputs output samples
* @param {callback} function that is called when train is completed
*/
static gint
lua_fann_train_threaded (lua_State *L)
{
#ifndef WITH_FANN
return 0;
#else
struct fann *f = rspamd_lua_check_fann (L, 1);
guint ninputs, noutputs, ndata, i, j;
struct lua_fann_train_cbdata *cbdata;
struct event_base *ev_base = lua_check_ev_base (L, 5);
GError *err = NULL;
const guint max_epochs_default = 1000;
const gdouble desired_mse_default = 0.0001;
if (f != NULL && lua_type (L, 2) == LUA_TTABLE &&
lua_type (L, 3) == LUA_TTABLE && lua_type (L, 4) == LUA_TFUNCTION &&
ev_base != NULL) {
/* First check sanity, call for table.getn for that */
ndata = rspamd_lua_table_size (L, 2);
ninputs = fann_get_num_input (f);
noutputs = fann_get_num_output (f);
cbdata = g_malloc0 (sizeof (*cbdata));
cbdata->L = L;
cbdata->f = f;
cbdata->train = rspamd_fann_create_train (ndata, ninputs, noutputs);
lua_pushvalue (L, 4);
cbdata->cbref = luaL_ref (L, LUA_REGISTRYINDEX);
if (rspamd_socketpair (cbdata->pair, 0) == -1) {
msg_err ("cannot open socketpair: %s", strerror (errno));
cbdata->pair[0] = -1;
cbdata->pair[1] = -1;
goto err;
}
for (i = 0; i < ndata; i ++) {
lua_rawgeti (L, 2, i + 1);
if (rspamd_lua_table_size (L, -1) != ninputs) {
msg_err ("invalid number of inputs: %d, %d expected",
rspamd_lua_table_size (L, -1), ninputs);
goto err;
}
for (j = 0; j < ninputs; j ++) {
lua_rawgeti (L, -1, j + 1);
cbdata->train->input[i][j] = lua_tonumber (L, -1);
lua_pop (L, 1);
}
lua_pop (L, 1);
lua_rawgeti (L, 3, i + 1);
if (rspamd_lua_table_size (L, -1) != noutputs) {
msg_err ("invalid number of outputs: %d, %d expected",
rspamd_lua_table_size (L, -1), noutputs);
goto err;
}
for (j = 0; j < noutputs; j++) {
lua_rawgeti (L, -1, j + 1);
cbdata->train->output[i][j] = lua_tonumber (L, -1);
lua_pop (L, 1);
}
}
cbdata->max_epochs = max_epochs_default;
cbdata->desired_mse = desired_mse_default;
if (lua_type (L, 5) == LUA_TTABLE) {
rspamd_lua_parse_table_arguments (L, 5, NULL,
"max_epochs=I;desired_mse=N",
&cbdata->max_epochs, &cbdata->desired_mse);
}
/* Now we can call training in a separate thread */
rspamd_socket_nonblocking (cbdata->pair[0]);
event_set (&cbdata->io, cbdata->pair[0], EV_READ, lua_fann_thread_notify,
cbdata);
event_base_set (ev_base, &cbdata->io);
/* TODO: add timeout */
event_add (&cbdata->io, NULL);
cbdata->t = rspamd_create_thread ("fann train", lua_fann_train_thread,
cbdata, &err);
if (cbdata->t == NULL) {
msg_err ("cannot create training thread: %e", err);
if (err) {
g_error_free (err);
}
goto err;
}
}
else {
return luaL_error (L, "invalid arguments");
}
return 0;
err:
if (cbdata->pair[0] != -1) {
close (cbdata->pair[0]);
}
if (cbdata->pair[1] != -1) {
close (cbdata->pair[1]);
}
fann_destroy_train (cbdata->train);
luaL_unref (L, LUA_REGISTRYINDEX, cbdata->cbref);
g_free (cbdata);
return luaL_error (L, "invalid arguments");
#endif
}
/***
* @function rspamd_fann.create_full(params)
* Creates new neural network with parameters:
* - `layers` {table/numbers}: table of layers in form: {N1, N2, N3 ... Nn} where N is number of neurons in a layer
* - `activation_hidden` {string}: activation function type for hidden layers (`tanh` by default)
* - `activation_output` {string}: activation function type for output layer (`tanh` by default)
* - `sparsed` {float}: create sparsed ANN, where number is a coefficient for sparsing
* - `learn` {string}: learning algorithm (quickprop, rprop or incremental)
* - `randomize` {boolean}: randomize weights (true by default)
* @return {fann} fann object
*/
static gint
lua_fann_create_full (lua_State *L)
{
#ifndef WITH_FANN
return 0;
#else
struct fann *f, **pfann;
guint nlayers, *layers, i;
const gchar *activation_hidden = NULL, *activation_output, *learn_alg = NULL;
gdouble sparsed = 0.0;
gboolean randomize_ann = TRUE;
GError *err = NULL;
if (lua_type (L, 1) == LUA_TTABLE) {
lua_pushstring (L, "layers");
lua_gettable (L, 1);
if (lua_type (L, -1) != LUA_TTABLE) {
return luaL_error (L, "bad layers attribute");
}
nlayers = rspamd_lua_table_size (L, -1);
if (nlayers < 2) {
return luaL_error (L, "bad layers attribute");
}
layers = g_new0 (guint, nlayers);
for (i = 0; i < nlayers; i ++) {
lua_rawgeti (L, -1, i + 1);
layers[i] = luaL_checknumber (L, -1);
lua_pop (L, 1);
}
lua_pop (L, 1); /* Table */
if (!rspamd_lua_parse_table_arguments (L, 1, &err,
"sparsed=N;randomize=B;learn=S;activation_hidden=S;activation_output=S",
&sparsed, &randomize_ann, &learn_alg, &activation_hidden, &activation_output)) {
g_free (layers);
if (err) {
gint r;
r = luaL_error (L, "invalid arguments: %s", err->message);
g_error_free (err);
return r;
}
else {
return luaL_error (L, "invalid arguments");
}
}
if (sparsed != 0.0) {
f = fann_create_standard_array (nlayers, layers);
}
else {
f = fann_create_sparse_array (sparsed, nlayers, layers);
}
if (f != NULL) {
pfann = lua_newuserdata (L, sizeof (gpointer));
*pfann = f;
rspamd_lua_setclass (L, "rspamd{fann}", -1);
}
else {
g_free (layers);
return luaL_error (L, "cannot create fann");
}
fann_set_activation_function_hidden (f,
string_to_activation_func (activation_hidden));
fann_set_activation_function_output (f,
string_to_activation_func (activation_output));
fann_set_training_algorithm (f, string_to_learn_alg (learn_alg));
if (randomize_ann) {
fann_randomize_weights (f, 0, 1);
}
g_free (layers);
}
else {
return luaL_error (L, "bad arguments");
}
return 1;
#endif
}
