rspamd_regexp_t*
rspamd_regexp_cache_create (struct rspamd_regexp_cache *cache,
const gchar *pattern,
const gchar *flags, GError **err)
{
rspamd_regexp_t *res;
if (cache == NULL) {
rspamd_regexp_library_init ();
cache = global_re_cache;
}
g_assert (cache != NULL);
res = rspamd_regexp_cache_query (cache, pattern, flags);
if (res != NULL) {
return res;
}
res = rspamd_regexp_new (pattern, flags, err);
if (res) {
REF_RETAIN (res);
g_hash_table_insert (cache->tbl, res->id, res);
}
return res;
}
static struct cache_savepoint *
rspamd_symbols_cache_make_checkpoint (struct rspamd_task *task,
struct symbols_cache *cache)
{
struct cache_savepoint *checkpoint;
if (cache->items_by_id->len != cache->items_by_order->d->len) {
/*
* Cache has been modified, need to resort it
*/
msg_info_cache ("symbols cache has been modified since last check:"
" old items: %ud, new items: %ud",
cache->items_by_order->d->len, cache->items_by_id->len);
rspamd_symbols_cache_resort (cache);
}
checkpoint = rspamd_mempool_alloc0 (task->task_pool, sizeof (*checkpoint));
/* Bit 0: check started, Bit 1: check finished */
checkpoint->processed_bits = rspamd_mempool_alloc0 (task->task_pool,
NBYTES (cache->used_items) * 2);
checkpoint->waitq = g_ptr_array_new ();
g_assert (cache->items_by_order != NULL);
checkpoint->version = cache->items_by_order->d->len;
checkpoint->order = cache->items_by_order;
REF_RETAIN (checkpoint->order);
rspamd_mempool_add_destructor (task->task_pool,
rspamd_symbols_cache_order_unref, checkpoint->order);
rspamd_mempool_add_destructor (task->task_pool,
rspamd_ptr_array_free_hard, checkpoint->waitq);
task->checkpoint = checkpoint;
rspamd_create_metric_result (task, DEFAULT_METRIC);
return checkpoint;
}
struct rspamd_email_address *
rspamd_email_address_ref (struct rspamd_email_address *addr)
{
REF_RETAIN (addr);
return addr;
}
struct rspamd_re_cache *
rspamd_re_cache_ref (struct rspamd_re_cache *cache)
{
if (cache) {
REF_RETAIN (cache);
}
return cache;
}
rspamd_regexp_t*
rspamd_regexp_ref (rspamd_regexp_t *re)
{
g_assert (re != NULL);
REF_RETAIN (re);
return re;
}
struct rspamd_re_runtime *
rspamd_re_cache_runtime_new (struct rspamd_re_cache *cache)
{
struct rspamd_re_runtime *rt;
g_assert (cache != NULL);
rt = g_slice_alloc0 (sizeof (*rt));
rt->cache = cache;
REF_RETAIN (cache);
rt->checked = g_slice_alloc0 (NBYTES (cache->nre));
rt->results = g_slice_alloc0 (cache->nre);
return rt;
}
/**
* Write HTTP request
*/
static void
write_http_request (struct http_callback_data *cbd)
{
gchar datebuf[128];
struct rspamd_http_message *msg;
rspamd_mempool_t *pool;
pool = cbd->map->pool;
if (cbd->fd != -1) {
close (cbd->fd);
}
cbd->fd = rspamd_inet_address_connect (cbd->addr, SOCK_STREAM, TRUE);
if (cbd->fd != -1) {
msg = rspamd_http_new_message (HTTP_REQUEST);
if (cbd->stage == map_load_file) {
msg->url = rspamd_fstring_new_init (cbd->data->path, strlen (cbd->data->path));
if (cbd->data->last_checked != 0 && cbd->stage == map_load_file) {
rspamd_http_date_format (datebuf, sizeof (datebuf),
cbd->data->last_checked);
rspamd_http_message_add_header (msg, "If-Modified-Since", datebuf);
}
}
else if (cbd->stage == map_load_pubkey) {
msg->url = rspamd_fstring_new_init (cbd->data->path, strlen (cbd->data->path));
msg->url = rspamd_fstring_append (msg->url, ".pub", 4);
}
else if (cbd->stage == map_load_signature) {
msg->url = rspamd_fstring_new_init (cbd->data->path, strlen (cbd->data->path));
msg->url = rspamd_fstring_append (msg->url, ".sig", 4);
}
else {
g_assert_not_reached ();
}
rspamd_http_connection_write_message (cbd->conn, msg, cbd->data->host,
NULL, cbd, cbd->fd, &cbd->tv, cbd->ev_base);
REF_RETAIN (cbd);
}
else {
msg_err_pool ("cannot connect to %s: %s", cbd->data->host,
strerror (errno));
}
}
void rspamd_regexp_cache_insert (struct rspamd_regexp_cache* cache,
const gchar *pattern,
const gchar *flags, rspamd_regexp_t *re)
{
g_assert (re != NULL);
g_assert (pattern != NULL);
if (cache == NULL) {
rspamd_regexp_library_init ();
cache = global_re_cache;
}
g_assert (cache != NULL);
/* Generate custom id */
rspamd_regexp_generate_id (pattern, flags, re->id);
REF_RETAIN (re);
g_hash_table_insert (cache->tbl, re->id, re);
}
/**
* Async HTTP callback
*/
static void
http_callback (gint fd, short what, void *ud)
{
struct rspamd_map *map = ud;
struct http_map_data *data;
struct http_callback_data *cbd;
rspamd_mempool_t *pool;
gchar tmpbuf[PATH_MAX];
data = map->map_data;
pool = map->pool;
if (!g_atomic_int_compare_and_exchange (map->locked, 0, 1)) {
msg_debug_pool (
"don't try to reread map as it is locked by other process, will reread it later");
jitter_timeout_event (map, TRUE, FALSE, FALSE);
return;
}
/* Plan event */
cbd = g_slice_alloc0 (sizeof (struct http_callback_data));
rspamd_snprintf (tmpbuf, sizeof (tmpbuf),
"%s" G_DIR_SEPARATOR_S "rspamd_map%d-XXXXXX",
map->cfg->temp_dir, map->id);
cbd->out_fd = mkstemp (tmpbuf);
if (cbd->out_fd == -1) {
g_slice_free1 (sizeof (*cbd), cbd);
msg_err_pool ("cannot create tempfile: %s", strerror (errno));
jitter_timeout_event (map, FALSE, FALSE, TRUE);
g_atomic_int_set (map->locked, 0);
return;
}
cbd->tmpfile = g_strdup (tmpbuf);
cbd->ev_base = map->ev_base;
cbd->map = map;
cbd->data = data;
cbd->fd = -1;
cbd->cbdata.state = 0;
cbd->cbdata.prev_data = *cbd->map->user_data;
cbd->cbdata.cur_data = NULL;
cbd->cbdata.map = cbd->map;
cbd->stage = map_resolve_host2;
double_to_tv (map->cfg->map_timeout, &cbd->tv);
REF_INIT_RETAIN (cbd, free_http_cbdata);
msg_debug_pool ("reading map data from %s", data->host);
/* Send both A and AAAA requests */
if (map->r->r) {
if (rdns_make_request_full (map->r->r, rspamd_map_dns_callback, cbd,
map->cfg->dns_timeout, map->cfg->dns_retransmits, 1,
data->host, RDNS_REQUEST_A)) {
REF_RETAIN (cbd);
}
if (rdns_make_request_full (map->r->r, rspamd_map_dns_callback, cbd,
map->cfg->dns_timeout, map->cfg->dns_retransmits, 1,
data->host, RDNS_REQUEST_AAAA)) {
REF_RETAIN (cbd);
}
jitter_timeout_event (map, FALSE, FALSE, FALSE);
map->dtor = free_http_cbdata_dtor;
map->dtor_data = cbd;
}
else {
msg_warn_pool ("cannot load map: DNS resolver is not initialized");
jitter_timeout_event (map, FALSE, FALSE, TRUE);
}
/* We don't need own ref as it is now refcounted by DNS requests */
REF_RELEASE (cbd);
}
void
rspamd_stat_init (struct rspamd_config *cfg, struct event_base *ev_base)
{
GList *cur, *curst;
struct rspamd_classifier_config *clf;
struct rspamd_statfile_config *stf;
struct rspamd_stat_backend *bk;
struct rspamd_statfile *st;
struct rspamd_classifier *cl;
const ucl_object_t *cache_obj = NULL, *cache_name_obj;
const gchar *cache_name = NULL;
if (stat_ctx == NULL) {
stat_ctx = g_slice_alloc0 (sizeof (*stat_ctx));
}
stat_ctx->backends_subrs = stat_backends;
stat_ctx->backends_count = G_N_ELEMENTS (stat_backends);
stat_ctx->classifiers_subrs = stat_classifiers;
stat_ctx->classifiers_count = G_N_ELEMENTS (stat_classifiers);
stat_ctx->tokenizers_subrs = stat_tokenizers;
stat_ctx->tokenizers_count = G_N_ELEMENTS (stat_tokenizers);
stat_ctx->caches_subrs = stat_caches;
stat_ctx->caches_count = G_N_ELEMENTS (stat_caches);
stat_ctx->cfg = cfg;
stat_ctx->statfiles = g_ptr_array_new ();
stat_ctx->classifiers = g_ptr_array_new ();
stat_ctx->async_elts = g_queue_new ();
stat_ctx->ev_base = ev_base;
REF_RETAIN (stat_ctx->cfg);
/* Create statfiles from the classifiers */
cur = cfg->classifiers;
while (cur) {
clf = cur->data;
bk = rspamd_stat_get_backend (clf->backend);
if (bk == NULL) {
msg_err_config ("cannot get backend of type %s, so disable classifier"
" %s completely", clf->backend, clf->name);
cur = g_list_next (cur);
continue;
}
/* XXX:
* Here we get the first classifier tokenizer config as the only one
* We NO LONGER support multiple tokenizers per rspamd instance
*/
if (stat_ctx->tkcf == NULL) {
stat_ctx->tokenizer = rspamd_stat_get_tokenizer (clf->tokenizer->name);
g_assert (stat_ctx->tokenizer != NULL);
stat_ctx->tkcf = stat_ctx->tokenizer->get_config (cfg->cfg_pool,
clf->tokenizer, NULL);
}
cl = g_slice_alloc0 (sizeof (*cl));
cl->cfg = clf;
cl->ctx = stat_ctx;
cl->statfiles_ids = g_array_new (FALSE, FALSE, sizeof (gint));
cl->subrs = rspamd_stat_get_classifier (clf->classifier);
g_assert (cl->subrs != NULL);
cl->subrs->init_func (cfg->cfg_pool, cl);
/* Init classifier cache */
cache_name = NULL;
if (clf->opts) {
cache_obj = ucl_object_find_key (clf->opts, "cache");
cache_name_obj = NULL;
if (cache_obj) {
cache_name_obj = ucl_object_find_any_key (cache_obj,
"name", "type", NULL);
}
if (cache_name_obj) {
cache_name = ucl_object_tostring (cache_name_obj);
}
}
if (cache_name == NULL) {
/* We assume that learn cache is the same as backend */
cache_name = clf->backend;
}
curst = clf->statfiles;
while (curst) {
stf = curst->data;
st = g_slice_alloc0 (sizeof (*st));
st->classifier = cl;
st->stcf = stf;
st->backend = bk;
st->bkcf = bk->init (stat_ctx, cfg, st);
msg_debug_config ("added backend %s for symbol %s",
bk->name, stf->symbol);
/* XXX: bad hack to pass statfiles configuration to cache */
if (cl->cache == NULL) {
cl->cache = rspamd_stat_get_cache (cache_name);
g_assert (cl->cache != NULL);
cl->cachecf = cl->cache->init (stat_ctx, cfg, st, cache_obj);
if (cl->cachecf == NULL) {
msg_err_config ("error adding cache %s for symbol %s",
cl->cache->name, stf->symbol);
cl->cache = NULL;
}
else {
msg_debug_config ("added cache %s for symbol %s",
cl->cache->name, stf->symbol);
}
}
if (st->bkcf == NULL) {
msg_err_config ("cannot init backend %s for statfile %s",
clf->backend, stf->symbol);
g_slice_free1 (sizeof (*st), st);
}
else {
st->id = stat_ctx->statfiles->len;
g_ptr_array_add (stat_ctx->statfiles, st);
g_array_append_val (cl->statfiles_ids, st->id);
}
curst = curst->next;
}
g_ptr_array_add (stat_ctx->classifiers, cl);
cur = cur->next;
}
}
void
rspamd_fuzzy_backend_count_redis (struct rspamd_fuzzy_backend *bk,
rspamd_fuzzy_count_cb cb, void *ud,
void *subr_ud)
{
struct rspamd_fuzzy_backend_redis *backend = subr_ud;
struct rspamd_fuzzy_redis_session *session;
struct upstream *up;
struct timeval tv;
rspamd_inet_addr_t *addr;
g_assert (backend != NULL);
session = g_slice_alloc0 (sizeof (*session));
session->backend = backend;
REF_RETAIN (session->backend);
session->callback.cb_count = cb;
session->cbdata = ud;
session->command = RSPAMD_FUZZY_REDIS_COMMAND_COUNT;
session->nargs = 2;
session->argv = g_malloc (sizeof (gchar *) * 2);
session->argv[0] = g_strdup ("HLEN");
session->argv[1] = g_strdup (backend->redis_object);
up = rspamd_upstream_get (backend->read_servers,
RSPAMD_UPSTREAM_ROUND_ROBIN,
NULL,
0);
session->up = up;
addr = rspamd_upstream_addr (up);
g_assert (addr != NULL);
session->ctx = rspamd_redis_pool_connect (backend->pool,
backend->dbname, backend->password,
rspamd_inet_address_to_string (addr),
rspamd_inet_address_get_port (addr));
if (session->ctx == NULL) {
rspamd_fuzzy_redis_session_dtor (session);
if (cb) {
cb (0, subr_ud);
}
}
else {
if (redisAsyncCommandArgv (session->ctx, rspamd_fuzzy_redis_count_callback,
session, session->nargs,
(const gchar **)session->argv, NULL) != REDIS_OK) {
rspamd_fuzzy_redis_session_dtor (session);
if (cb) {
cb (0, subr_ud);
}
}
else {
/* Add timeout */
event_set (&session->timeout, -1, EV_TIMEOUT, rspamd_fuzzy_redis_timeout,
session);
event_base_set (rspamd_fuzzy_backend_event_base (bk),
&session->timeout);
double_to_tv (backend->timeout, &tv);
event_add (&session->timeout, &tv);
}
}
}
struct rspamd_worker *
rspamd_fork_worker (struct rspamd_main *rspamd_main,
struct rspamd_worker_conf *cf,
guint index,
struct event_base *ev_base)
{
struct rspamd_worker *wrk;
gint rc;
struct rlimit rlim;
/* Starting worker process */
wrk = (struct rspamd_worker *) g_malloc0 (sizeof (struct rspamd_worker));
if (!rspamd_socketpair (wrk->control_pipe)) {
msg_err ("socketpair failure: %s", strerror (errno));
exit (-errno);
}
if (!rspamd_socketpair (wrk->srv_pipe)) {
msg_err ("socketpair failure: %s", strerror (errno));
exit (-errno);
}
wrk->srv = rspamd_main;
wrk->type = cf->type;
wrk->cf = cf;
REF_RETAIN (cf);
wrk->index = index;
wrk->ctx = cf->ctx;
wrk->finish_actions = g_ptr_array_new ();
wrk->pid = fork ();
switch (wrk->pid) {
case 0:
/* Update pid for logging */
rspamd_log_update_pid (cf->type, rspamd_main->logger);
/* Init PRNG after fork */
rc = ottery_init (rspamd_main->cfg->libs_ctx->ottery_cfg);
if (rc != OTTERY_ERR_NONE) {
msg_err_main ("cannot initialize PRNG: %d", rc);
abort ();
}
rspamd_random_seed_fast ();
#ifdef HAVE_EVUTIL_RNG_INIT
evutil_secure_rng_init ();
#endif
/* Remove the inherited event base */
event_reinit (rspamd_main->ev_base);
event_base_free (rspamd_main->ev_base);
/* Drop privilleges */
rspamd_worker_drop_priv (rspamd_main);
/* Set limits */
rspamd_worker_set_limits (rspamd_main, cf);
/* Re-set stack limit */
getrlimit (RLIMIT_STACK, &rlim);
rlim.rlim_cur = 100 * 1024 * 1024;
rlim.rlim_max = rlim.rlim_cur;
setrlimit (RLIMIT_STACK, &rlim);
setproctitle ("%s process", cf->worker->name);
rspamd_pidfile_close (rspamd_main->pfh);
/* Do silent log reopen to avoid collisions */
rspamd_log_close (rspamd_main->logger);
rspamd_log_open (rspamd_main->logger);
wrk->start_time = rspamd_get_calendar_ticks ();
#if ((GLIB_MAJOR_VERSION == 2) && (GLIB_MINOR_VERSION <= 30))
# if (GLIB_MINOR_VERSION > 20)
/* Ugly hack for old glib */
if (!g_thread_get_initialized ()) {
g_thread_init (NULL);
}
# else
g_thread_init (NULL);
# endif
#endif
msg_info_main ("starting %s process %P (%d)", cf->worker->name,
getpid (), index);
/* Close parent part of socketpair */
close (wrk->control_pipe[0]);
close (wrk->srv_pipe[0]);
rspamd_socket_nonblocking (wrk->control_pipe[1]);
rspamd_socket_nonblocking (wrk->srv_pipe[1]);
/* Execute worker */
cf->worker->worker_start_func (wrk);
exit (EXIT_FAILURE);
break;
case -1:
msg_err_main ("cannot fork main process. %s", strerror (errno));
rspamd_pidfile_remove (rspamd_main->pfh);
exit (-errno);
break;
default:
/* Close worker part of socketpair */
close (wrk->control_pipe[1]);
close (wrk->srv_pipe[1]);
rspamd_socket_nonblocking (wrk->control_pipe[0]);
rspamd_socket_nonblocking (wrk->srv_pipe[0]);
rspamd_srv_start_watching (wrk, ev_base);
/* Insert worker into worker's table, pid is index */
g_hash_table_insert (rspamd_main->workers, GSIZE_TO_POINTER (
wrk->pid), wrk);
break;
}
return wrk;
}
