struct rspamd_worker *
rspamd_fork_worker (struct rspamd_main *rspamd_main,
struct rspamd_worker_conf *cf,
guint index,
struct event_base *ev_base)
{
struct rspamd_worker *wrk;
gint rc;
struct rlimit rlim;
/* Starting worker process */
wrk = (struct rspamd_worker *) g_malloc0 (sizeof (struct rspamd_worker));
if (!rspamd_socketpair (wrk->control_pipe)) {
msg_err ("socketpair failure: %s", strerror (errno));
exit (-errno);
}
if (!rspamd_socketpair (wrk->srv_pipe)) {
msg_err ("socketpair failure: %s", strerror (errno));
exit (-errno);
}
wrk->srv = rspamd_main;
wrk->type = cf->type;
wrk->cf = cf;
REF_RETAIN (cf);
wrk->index = index;
wrk->ctx = cf->ctx;
wrk->finish_actions = g_ptr_array_new ();
wrk->pid = fork ();
switch (wrk->pid) {
case 0:
/* Update pid for logging */
rspamd_log_update_pid (cf->type, rspamd_main->logger);
/* Init PRNG after fork */
rc = ottery_init (rspamd_main->cfg->libs_ctx->ottery_cfg);
if (rc != OTTERY_ERR_NONE) {
msg_err_main ("cannot initialize PRNG: %d", rc);
abort ();
}
rspamd_random_seed_fast ();
#ifdef HAVE_EVUTIL_RNG_INIT
evutil_secure_rng_init ();
#endif
/* Remove the inherited event base */
event_reinit (rspamd_main->ev_base);
event_base_free (rspamd_main->ev_base);
/* Drop privilleges */
rspamd_worker_drop_priv (rspamd_main);
/* Set limits */
rspamd_worker_set_limits (rspamd_main, cf);
/* Re-set stack limit */
getrlimit (RLIMIT_STACK, &rlim);
rlim.rlim_cur = 100 * 1024 * 1024;
rlim.rlim_max = rlim.rlim_cur;
setrlimit (RLIMIT_STACK, &rlim);
setproctitle ("%s process", cf->worker->name);
rspamd_pidfile_close (rspamd_main->pfh);
/* Do silent log reopen to avoid collisions */
rspamd_log_close (rspamd_main->logger);
rspamd_log_open (rspamd_main->logger);
wrk->start_time = rspamd_get_calendar_ticks ();
#if ((GLIB_MAJOR_VERSION == 2) && (GLIB_MINOR_VERSION <= 30))
# if (GLIB_MINOR_VERSION > 20)
/* Ugly hack for old glib */
if (!g_thread_get_initialized ()) {
g_thread_init (NULL);
}
# else
g_thread_init (NULL);
# endif
#endif
msg_info_main ("starting %s process %P (%d)", cf->worker->name,
getpid (), index);
/* Close parent part of socketpair */
close (wrk->control_pipe[0]);
close (wrk->srv_pipe[0]);
rspamd_socket_nonblocking (wrk->control_pipe[1]);
rspamd_socket_nonblocking (wrk->srv_pipe[1]);
/* Execute worker */
cf->worker->worker_start_func (wrk);
exit (EXIT_FAILURE);
break;
case -1:
msg_err_main ("cannot fork main process. %s", strerror (errno));
rspamd_pidfile_remove (rspamd_main->pfh);
exit (-errno);
break;
default:
/* Close worker part of socketpair */
close (wrk->control_pipe[1]);
close (wrk->srv_pipe[1]);
rspamd_socket_nonblocking (wrk->control_pipe[0]);
rspamd_socket_nonblocking (wrk->srv_pipe[0]);
rspamd_srv_start_watching (wrk, ev_base);
/* Insert worker into worker's table, pid is index */
g_hash_table_insert (rspamd_main->workers, GSIZE_TO_POINTER (
wrk->pid), wrk);
break;
}
return wrk;
}
static gint
lua_worker_spawn_process (lua_State *L)
{
struct rspamd_worker *w = lua_check_worker (L, 1);
struct rspamd_lua_process_cbdata *cbdata;
struct rspamd_abstract_worker_ctx *actx;
struct rspamd_srv_command srv_cmd;
const gchar *cmdline = NULL, *input = NULL;
gsize inputlen = 0;
pid_t pid;
GError *err = NULL;
gint func_cbref, cb_cbref;
if (!rspamd_lua_parse_table_arguments (L, 2, &err,
"func=F;exec=S;stdin=V;*on_complete=F", &func_cbref,
&cmdline, &inputlen, &input, &cb_cbref)) {
msg_err ("cannot get parameters list: %e", err);
if (err) {
g_error_free (err);
}
return 0;
}
cbdata = g_malloc0 (sizeof (*cbdata));
cbdata->cb_cbref = cb_cbref;
cbdata->func_cbref = func_cbref;
if (input) {
cbdata->out_buf = g_string_new_len (input, inputlen);
cbdata->out_pos = 0;
}
if (rspamd_socketpair (cbdata->sp, TRUE) == -1) {
msg_err ("cannot spawn socketpair: %s", strerror (errno));
luaL_unref (L, LUA_REGISTRYINDEX, cbdata->func_cbref);
luaL_unref (L, LUA_REGISTRYINDEX, cbdata->cb_cbref);
g_free (cbdata);
return 0;
}
actx = w->ctx;
cbdata->wrk = w;
cbdata->L = L;
cbdata->ev_base = actx->ev_base;
cbdata->sz = (guint64)-1;
pid = fork ();
if (pid == -1) {
msg_err ("cannot spawn process: %s", strerror (errno));
close (cbdata->sp[0]);
close (cbdata->sp[1]);
luaL_unref (L, LUA_REGISTRYINDEX, cbdata->func_cbref);
luaL_unref (L, LUA_REGISTRYINDEX, cbdata->cb_cbref);
g_free (cbdata);
return 0;
}
else if (pid == 0) {
/* Child */
gint rc;
gchar inbuf[4];
rspamd_log_update_pid (w->cf->type, w->srv->logger);
rc = ottery_init (w->srv->cfg->libs_ctx->ottery_cfg);
if (rc != OTTERY_ERR_NONE) {
msg_err ("cannot initialize PRNG: %d", rc);
abort ();
}
rspamd_random_seed_fast ();
#ifdef HAVE_EVUTIL_RNG_INIT
evutil_secure_rng_init ();
#endif
close (cbdata->sp[0]);
/* Here we assume that we can block on writing results */
rspamd_socket_blocking (cbdata->sp[1]);
event_reinit (cbdata->ev_base);
g_hash_table_remove_all (w->signal_events);
rspamd_worker_unblock_signals ();
rspamd_lua_execute_lua_subprocess (L, cbdata);
/* Wait for parent to reply and exit */
rc = read (cbdata->sp[1], inbuf, sizeof (inbuf));
if (memcmp (inbuf, "\0\0\0\0", 4) == 0) {
exit (EXIT_SUCCESS);
}
else {
msg_err ("got invalid reply from parent");
exit (EXIT_FAILURE);
}
}
cbdata->cpid = pid;
cbdata->io_buf = g_string_sized_new (8);
/* Notify main */
memset (&srv_cmd, 0, sizeof (srv_cmd));
srv_cmd.type = RSPAMD_SRV_ON_FORK;
srv_cmd.cmd.on_fork.state = child_create;
srv_cmd.cmd.on_fork.cpid = pid;
srv_cmd.cmd.on_fork.ppid = getpid ();
rspamd_srv_send_command (w, cbdata->ev_base, &srv_cmd, -1, NULL, NULL);
close (cbdata->sp[1]);
rspamd_socket_nonblocking (cbdata->sp[0]);
/* Parent */
rspamd_worker_set_signal_handler (SIGCHLD, w, cbdata->ev_base,
rspamd_lua_cld_handler,
cbdata);
/* Add result pipe waiting */
event_set (&cbdata->ev, cbdata->sp[0], EV_READ | EV_PERSIST,
rspamd_lua_subprocess_io, cbdata);
event_base_set (cbdata->ev_base, &cbdata->ev);
/* TODO: maybe add timeout? */
event_add (&cbdata->ev, NULL);
return 0;
}
/**
* @method rspamd_fann:train_threaded(inputs, outputs, callback, event_base, {params})
* Trains neural network with batch of samples. Inputs and outputs should be tables of
* equal size, each row in table should be N inputs and M outputs, e.g.
* {{0, 1, 1}, ...} -> {{0}, {1} ...}
* @param {table} inputs input samples
* @param {table} outputs output samples
* @param {callback} function that is called when train is completed
*/
static gint
lua_fann_train_threaded (lua_State *L)
{
#ifndef WITH_FANN
return 0;
#else
struct fann *f = rspamd_lua_check_fann (L, 1);
guint ninputs, noutputs, ndata, i, j;
struct lua_fann_train_cbdata *cbdata;
struct event_base *ev_base = lua_check_ev_base (L, 5);
GError *err = NULL;
const guint max_epochs_default = 1000;
const gdouble desired_mse_default = 0.0001;
if (f != NULL && lua_type (L, 2) == LUA_TTABLE &&
lua_type (L, 3) == LUA_TTABLE && lua_type (L, 4) == LUA_TFUNCTION &&
ev_base != NULL) {
/* First check sanity, call for table.getn for that */
ndata = rspamd_lua_table_size (L, 2);
ninputs = fann_get_num_input (f);
noutputs = fann_get_num_output (f);
cbdata = g_malloc0 (sizeof (*cbdata));
cbdata->L = L;
cbdata->f = f;
cbdata->train = rspamd_fann_create_train (ndata, ninputs, noutputs);
lua_pushvalue (L, 4);
cbdata->cbref = luaL_ref (L, LUA_REGISTRYINDEX);
if (rspamd_socketpair (cbdata->pair, 0) == -1) {
msg_err ("cannot open socketpair: %s", strerror (errno));
cbdata->pair[0] = -1;
cbdata->pair[1] = -1;
goto err;
}
for (i = 0; i < ndata; i ++) {
lua_rawgeti (L, 2, i + 1);
if (rspamd_lua_table_size (L, -1) != ninputs) {
msg_err ("invalid number of inputs: %d, %d expected",
rspamd_lua_table_size (L, -1), ninputs);
goto err;
}
for (j = 0; j < ninputs; j ++) {
lua_rawgeti (L, -1, j + 1);
cbdata->train->input[i][j] = lua_tonumber (L, -1);
lua_pop (L, 1);
}
lua_pop (L, 1);
lua_rawgeti (L, 3, i + 1);
if (rspamd_lua_table_size (L, -1) != noutputs) {
msg_err ("invalid number of outputs: %d, %d expected",
rspamd_lua_table_size (L, -1), noutputs);
goto err;
}
for (j = 0; j < noutputs; j++) {
lua_rawgeti (L, -1, j + 1);
cbdata->train->output[i][j] = lua_tonumber (L, -1);
lua_pop (L, 1);
}
}
cbdata->max_epochs = max_epochs_default;
cbdata->desired_mse = desired_mse_default;
if (lua_type (L, 5) == LUA_TTABLE) {
rspamd_lua_parse_table_arguments (L, 5, NULL,
"max_epochs=I;desired_mse=N",
&cbdata->max_epochs, &cbdata->desired_mse);
}
/* Now we can call training in a separate thread */
rspamd_socket_nonblocking (cbdata->pair[0]);
event_set (&cbdata->io, cbdata->pair[0], EV_READ, lua_fann_thread_notify,
cbdata);
event_base_set (ev_base, &cbdata->io);
/* TODO: add timeout */
event_add (&cbdata->io, NULL);
cbdata->t = rspamd_create_thread ("fann train", lua_fann_train_thread,
cbdata, &err);
if (cbdata->t == NULL) {
msg_err ("cannot create training thread: %e", err);
if (err) {
g_error_free (err);
}
goto err;
}
}
else {
return luaL_error (L, "invalid arguments");
}
return 0;
err:
if (cbdata->pair[0] != -1) {
close (cbdata->pair[0]);
}
if (cbdata->pair[1] != -1) {
close (cbdata->pair[1]);
}
fann_destroy_train (cbdata->train);
luaL_unref (L, LUA_REGISTRYINDEX, cbdata->cbref);
g_free (cbdata);
return luaL_error (L, "invalid arguments");
#endif
}
struct rspamd_worker *
rspamd_fork_worker (struct rspamd_main *rspamd_main,
struct rspamd_worker_conf *cf,
guint index)
{
struct rspamd_worker *cur;
/* Starting worker process */
cur = (struct rspamd_worker *) g_malloc0 (sizeof (struct rspamd_worker));
if (!rspamd_socketpair (cur->control_pipe)) {
msg_err ("socketpair failure: %s", strerror (errno));
exit (-errno);
}
cur->srv = rspamd_main;
cur->type = cf->type;
cur->cf = g_malloc (sizeof (struct rspamd_worker_conf));
memcpy (cur->cf, cf, sizeof (struct rspamd_worker_conf));
cur->index = index;
cur->ctx = cf->ctx;
cur->pid = fork ();
switch (cur->pid) {
case 0:
/* Update pid for logging */
rspamd_log_update_pid (cf->type, rspamd_main->logger);
/* Lock statfile pool if possible XXX */
/* Init PRNG after fork */
ottery_init (NULL);
g_random_set_seed (ottery_rand_uint32 ());
/* Drop privilleges */
rspamd_worker_drop_priv (rspamd_main);
/* Set limits */
rspamd_worker_set_limits (rspamd_main, cf);
setproctitle ("%s process", cf->worker->name);
rspamd_pidfile_close (rspamd_main->pfh);
/* Do silent log reopen to avoid collisions */
rspamd_log_close (rspamd_main->logger);
rspamd_log_open (rspamd_main->logger);
cur->start_time = rspamd_get_calendar_ticks ();
#if ((GLIB_MAJOR_VERSION == 2) && (GLIB_MINOR_VERSION <= 30))
# if (GLIB_MINOR_VERSION > 20)
/* Ugly hack for old glib */
if (!g_thread_get_initialized ()) {
g_thread_init (NULL);
}
# else
g_thread_init (NULL);
# endif
#endif
msg_info_main ("starting %s process %P", cf->worker->name, getpid ());
/* Close parent part of socketpair */
close (cur->control_pipe[0]);
rspamd_socket_nonblocking (cur->control_pipe[1]);
/* Execute worker */
cf->worker->worker_start_func (cur);
break;
case -1:
msg_err_main ("cannot fork main process. %s", strerror (errno));
rspamd_pidfile_remove (rspamd_main->pfh);
exit (-errno);
break;
default:
/* Close worker part of socketpair */
close (cur->control_pipe[1]);
rspamd_socket_nonblocking (cur->control_pipe[0]);
/* Insert worker into worker's table, pid is index */
g_hash_table_insert (rspamd_main->workers, GSIZE_TO_POINTER (
cur->pid), cur);
break;
}
return cur;
}