static void
parse_flags (struct fuzzy_rule *rule,
struct rspamd_config *cfg,
const ucl_object_t *val,
gint cb_id)
{
const ucl_object_t *elt;
struct fuzzy_mapping *map;
const gchar *sym = NULL;
if (val->type == UCL_STRING) {
msg_err_config (
"string mappings are deprecated and no longer supported, use new style configuration");
}
else if (val->type == UCL_OBJECT) {
elt = ucl_object_find_key (val, "symbol");
if (elt == NULL || !ucl_object_tostring_safe (elt, &sym)) {
sym = ucl_object_key (val);
}
if (sym != NULL) {
map =
rspamd_mempool_alloc (fuzzy_module_ctx->fuzzy_pool,
sizeof (struct fuzzy_mapping));
map->symbol = sym;
elt = ucl_object_find_key (val, "flag");
if (elt != NULL && ucl_obj_toint_safe (elt, &map->fuzzy_flag)) {
elt = ucl_object_find_key (val, "max_score");
if (elt != NULL) {
map->weight = ucl_obj_todouble (elt);
}
else {
map->weight = rule->max_score;
}
/* Add flag to hash table */
g_hash_table_insert (rule->mappings,
GINT_TO_POINTER (map->fuzzy_flag), map);
rspamd_symbols_cache_add_symbol (cfg->cache,
map->symbol, 0,
NULL, NULL,
SYMBOL_TYPE_VIRTUAL|SYMBOL_TYPE_FINE,
cb_id);
}
else {
msg_err_config ("fuzzy_map parameter has no flag definition");
}
}
else {
msg_err_config ("fuzzy_map parameter has no symbol definition");
}
}
else {
msg_err_config ("fuzzy_map parameter is of an unsupported type");
}
}
static void
rspamd_ucl_fin_cb (rspamd_mempool_t * pool, struct map_cb_data *data)
{
struct rspamd_ucl_map_cbdata *cbdata = data->cur_data, *prev =
data->prev_data;
ucl_object_t *obj;
struct ucl_parser *parser;
guint32 checksum;
ucl_object_iter_t it = NULL;
const ucl_object_t *cur;
struct rspamd_config *cfg = data->map->cfg;
if (prev != NULL) {
if (prev->buf != NULL) {
g_string_free (prev->buf, TRUE);
}
g_free (prev);
}
if (cbdata == NULL) {
msg_err_config ("map fin error: new data is NULL");
return;
}
checksum = XXH64 (cbdata->buf->str, cbdata->buf->len, 0);
if (data->map->checksum != checksum) {
/* New data available */
parser = ucl_parser_new (0);
if (!ucl_parser_add_chunk (parser, cbdata->buf->str,
cbdata->buf->len)) {
msg_err_config ("cannot parse map %s: %s",
data->map->uri,
ucl_parser_get_error (parser));
ucl_parser_free (parser);
}
else {
obj = ucl_parser_get_object (parser);
ucl_parser_free (parser);
it = NULL;
while ((cur = ucl_iterate_object (obj, &it, true))) {
ucl_object_replace_key (cbdata->cfg->rcl_obj, (ucl_object_t *)cur,
cur->key, cur->keylen, false);
}
ucl_object_unref (obj);
data->map->checksum = checksum;
}
}
else {
msg_info_config ("do not reload map %s, checksum is the same: %d",
data->map->uri,
checksum);
}
}
gpointer
rspamd_redis_init (struct rspamd_stat_ctx *ctx,
struct rspamd_config *cfg, struct rspamd_statfile *st)
{
struct redis_stat_ctx *backend;
struct rspamd_statfile_config *stf = st->stcf;
struct rspamd_redis_stat_elt *st_elt;
const ucl_object_t *obj;
gboolean ret = FALSE;
backend = g_slice_alloc0 (sizeof (*backend));
/* First search in backend configuration */
obj = ucl_object_lookup (st->classifier->cfg->opts, "backend");
if (obj != NULL && ucl_object_type (obj) == UCL_OBJECT) {
ret = rspamd_redis_try_ucl (backend, obj, cfg, stf->symbol);
}
/* Now try statfiles config */
if (!ret) {
ret = rspamd_redis_try_ucl (backend, stf->opts, cfg, stf->symbol);
}
/* Now try classifier config */
if (!ret) {
ret = rspamd_redis_try_ucl (backend, st->classifier->cfg->opts, cfg,
stf->symbol);
}
if (!ret) {
msg_err_config ("cannot init redis backend for %s", stf->symbol);
g_slice_free1 (sizeof (*backend), backend);
return NULL;
}
stf->clcf->flags |= RSPAMD_FLAG_CLASSIFIER_INCREMENTING_BACKEND;
backend->stcf = stf;
st_elt = g_slice_alloc0 (sizeof (*st_elt));
st_elt->ev_base = ctx->ev_base;
st_elt->ctx = backend;
backend->stat_elt = rspamd_stat_ctx_register_async (
rspamd_redis_async_stat_cb,
rspamd_redis_async_stat_fin,
st_elt,
REDIS_STAT_TIMEOUT);
st_elt->async = backend->stat_elt;
return (gpointer)backend;
}
gboolean
rspamd_config_calculate_checksum (struct rspamd_config *cfg)
{
gint fd;
void *map;
struct stat st;
/* Compute checksum for config file that should be used by xml dumper */
if ((fd = open (cfg->cfg_name, O_RDONLY)) == -1) {
msg_err_config (
"config file %s is no longer available, cannot calculate checksum");
return FALSE;
}
if (stat (cfg->cfg_name, &st) == -1) {
msg_err_config ("cannot stat %s: %s", cfg->cfg_name, strerror (errno));
close (fd);
return FALSE;
}
/* Now mmap this file to simplify reading process */
if ((map =
mmap (NULL, st.st_size, PROT_READ, MAP_SHARED, fd, 0)) == MAP_FAILED) {
msg_err_config ("cannot mmap %s: %s", cfg->cfg_name, strerror (errno));
close (fd);
return FALSE;
}
close (fd);
/* Get checksum for a file */
cfg->checksum = g_compute_checksum_for_string (G_CHECKSUM_MD5,
map,
st.st_size);
munmap (map, st.st_size);
return TRUE;
}
gboolean
rspamd_parse_bind_line (struct rspamd_config *cfg,
struct rspamd_worker_conf *cf,
const gchar *str)
{
struct rspamd_worker_bind_conf *cnf;
gchar **tokens, *err;
gboolean ret = TRUE;
if (str == NULL) {
return FALSE;
}
if (str[0] == '[') {
/* This is an ipv6 address */
gsize len, ntok;
const gchar *start, *ip_pos;
start = str + 1;
len = strcspn (start, "]");
if (start[len] != ']') {
return FALSE;
}
ip_pos = start;
start += len + 1;
ntok = 1;
if (*start == ':') {
ntok = 2;
start ++;
}
tokens = g_malloc_n (ntok + 1, sizeof (gchar *));
tokens[ntok] = NULL;
tokens[0] = g_malloc (len + 1);
rspamd_strlcpy (tokens[0], ip_pos, len + 1);
if (ntok > 1) {
tokens[1] = g_strdup (start);
}
}
else {
tokens = g_strsplit_set (str, ":", 0);
}
if (!tokens || !tokens[0]) {
return FALSE;
}
cnf =
rspamd_mempool_alloc0 (cfg->cfg_pool,
sizeof (struct rspamd_worker_bind_conf));
cnf->cnt = 1024;
if (strcmp (tokens[0], "systemd") == 0) {
/* The actual socket will be passed by systemd environment */
cnf->is_systemd = TRUE;
cnf->cnt = strtoul (tokens[1], &err, 10);
cnf->addrs = NULL;
if (err == NULL || *err == '\0') {
cnf->name = rspamd_mempool_strdup (cfg->cfg_pool, str);
LL_PREPEND (cf->bind_conf, cnf);
}
else {
msg_err_config ("cannot parse bind line: %s", str);
ret = FALSE;
}
}
else {
if (!rspamd_parse_host_port_priority_strv (tokens, &cnf->addrs,
NULL, &cnf->name, DEFAULT_BIND_PORT, cfg->cfg_pool)) {
msg_err_config ("cannot parse bind line: %s", str);
ret = FALSE;
}
else {
cnf->cnt = cnf->addrs->len;
LL_PREPEND (cf->bind_conf, cnf);
}
}
g_strfreev (tokens);
return ret;
}
/*
* Perform post load actions
*/
void
rspamd_config_post_load (struct rspamd_config *cfg)
{
#ifdef HAVE_CLOCK_GETTIME
struct timespec ts;
#endif
struct metric *def_metric;
#ifdef HAVE_CLOCK_GETTIME
#ifdef HAVE_CLOCK_PROCESS_CPUTIME_ID
clock_getres (CLOCK_PROCESS_CPUTIME_ID, &ts);
# elif defined(HAVE_CLOCK_VIRTUAL)
clock_getres (CLOCK_VIRTUAL, &ts);
# else
clock_getres (CLOCK_REALTIME, &ts);
# endif
cfg->clock_res = log10 (1000000. / ts.tv_nsec);
if (cfg->clock_res < 0) {
cfg->clock_res = 0;
}
if (cfg->clock_res > 3) {
cfg->clock_res = 3;
}
#else
/* For gettimeofday */
cfg->clock_res = 1;
#endif
rspamd_regexp_library_init ();
if ((def_metric =
g_hash_table_lookup (cfg->metrics, DEFAULT_METRIC)) == NULL) {
def_metric = rspamd_config_new_metric (cfg, NULL, DEFAULT_METRIC);
def_metric->actions[METRIC_ACTION_REJECT].score = DEFAULT_SCORE;
}
if (cfg->tld_file == NULL) {
/* Try to guess tld file */
GString *fpath = g_string_new (NULL);
rspamd_printf_gstring (fpath, "%s%c%s", RSPAMD_PLUGINSDIR,
G_DIR_SEPARATOR, "effective_tld_names.dat");
if (access (fpath->str, R_OK)) {
msg_warn_config ("url_tld option is not specified but %s is available,"
" therefore this file is assumed as TLD file for URL"
" extraction", fpath->str);
cfg->tld_file = rspamd_mempool_strdup (cfg->cfg_pool, fpath->str);
}
else {
msg_err_config ("no url_tld option has been specified, URL's detection "
"will be awfully broken");
}
g_string_free (fpath, TRUE);
}
/* Lua options */
(void)rspamd_lua_post_load_config (cfg);
init_dynamic_config (cfg);
rspamd_url_init (cfg->tld_file);
/* Insert classifiers symbols */
(void)rspamd_config_insert_classify_symbols (cfg);
}
void
rspamd_stat_init (struct rspamd_config *cfg, struct event_base *ev_base)
{
GList *cur, *curst;
struct rspamd_classifier_config *clf;
struct rspamd_statfile_config *stf;
struct rspamd_stat_backend *bk;
struct rspamd_statfile *st;
struct rspamd_classifier *cl;
const ucl_object_t *cache_obj = NULL, *cache_name_obj;
const gchar *cache_name = NULL;
if (stat_ctx == NULL) {
stat_ctx = g_slice_alloc0 (sizeof (*stat_ctx));
}
stat_ctx->backends_subrs = stat_backends;
stat_ctx->backends_count = G_N_ELEMENTS (stat_backends);
stat_ctx->classifiers_subrs = stat_classifiers;
stat_ctx->classifiers_count = G_N_ELEMENTS (stat_classifiers);
stat_ctx->tokenizers_subrs = stat_tokenizers;
stat_ctx->tokenizers_count = G_N_ELEMENTS (stat_tokenizers);
stat_ctx->caches_subrs = stat_caches;
stat_ctx->caches_count = G_N_ELEMENTS (stat_caches);
stat_ctx->cfg = cfg;
stat_ctx->statfiles = g_ptr_array_new ();
stat_ctx->classifiers = g_ptr_array_new ();
stat_ctx->async_elts = g_queue_new ();
stat_ctx->ev_base = ev_base;
REF_RETAIN (stat_ctx->cfg);
/* Create statfiles from the classifiers */
cur = cfg->classifiers;
while (cur) {
clf = cur->data;
bk = rspamd_stat_get_backend (clf->backend);
if (bk == NULL) {
msg_err_config ("cannot get backend of type %s, so disable classifier"
" %s completely", clf->backend, clf->name);
cur = g_list_next (cur);
continue;
}
/* XXX:
* Here we get the first classifier tokenizer config as the only one
* We NO LONGER support multiple tokenizers per rspamd instance
*/
if (stat_ctx->tkcf == NULL) {
stat_ctx->tokenizer = rspamd_stat_get_tokenizer (clf->tokenizer->name);
g_assert (stat_ctx->tokenizer != NULL);
stat_ctx->tkcf = stat_ctx->tokenizer->get_config (cfg->cfg_pool,
clf->tokenizer, NULL);
}
cl = g_slice_alloc0 (sizeof (*cl));
cl->cfg = clf;
cl->ctx = stat_ctx;
cl->statfiles_ids = g_array_new (FALSE, FALSE, sizeof (gint));
cl->subrs = rspamd_stat_get_classifier (clf->classifier);
g_assert (cl->subrs != NULL);
cl->subrs->init_func (cfg->cfg_pool, cl);
/* Init classifier cache */
cache_name = NULL;
if (clf->opts) {
cache_obj = ucl_object_find_key (clf->opts, "cache");
cache_name_obj = NULL;
if (cache_obj) {
cache_name_obj = ucl_object_find_any_key (cache_obj,
"name", "type", NULL);
}
if (cache_name_obj) {
cache_name = ucl_object_tostring (cache_name_obj);
}
}
if (cache_name == NULL) {
/* We assume that learn cache is the same as backend */
cache_name = clf->backend;
}
curst = clf->statfiles;
while (curst) {
stf = curst->data;
st = g_slice_alloc0 (sizeof (*st));
st->classifier = cl;
st->stcf = stf;
st->backend = bk;
st->bkcf = bk->init (stat_ctx, cfg, st);
msg_debug_config ("added backend %s for symbol %s",
bk->name, stf->symbol);
/* XXX: bad hack to pass statfiles configuration to cache */
if (cl->cache == NULL) {
cl->cache = rspamd_stat_get_cache (cache_name);
g_assert (cl->cache != NULL);
cl->cachecf = cl->cache->init (stat_ctx, cfg, st, cache_obj);
if (cl->cachecf == NULL) {
msg_err_config ("error adding cache %s for symbol %s",
cl->cache->name, stf->symbol);
cl->cache = NULL;
}
else {
msg_debug_config ("added cache %s for symbol %s",
cl->cache->name, stf->symbol);
}
}
if (st->bkcf == NULL) {
msg_err_config ("cannot init backend %s for statfile %s",
clf->backend, stf->symbol);
g_slice_free1 (sizeof (*st), st);
}
else {
st->id = stat_ctx->statfiles->len;
g_ptr_array_add (stat_ctx->statfiles, st);
g_array_append_val (cl->statfiles_ids, st->id);
}
curst = curst->next;
}
g_ptr_array_add (stat_ctx->classifiers, cl);
cur = cur->next;
}
}
gint
regexp_module_config (struct rspamd_config *cfg)
{
struct regexp_module_item *cur_item;
const ucl_object_t *sec, *value, *elt;
ucl_object_iter_t it = NULL;
gint res = TRUE, id;
if (!rspamd_config_is_module_enabled (cfg, "regexp")) {
return TRUE;
}
sec = ucl_object_find_key (cfg->rcl_obj, "regexp");
if (sec == NULL) {
msg_err_config ("regexp module enabled, but no rules are defined");
return TRUE;
}
regexp_module_ctx->max_size = 0;
while ((value = ucl_iterate_object (sec, &it, true)) != NULL) {
if (g_ascii_strncasecmp (ucl_object_key (value), "max_size",
sizeof ("max_size") - 1) == 0) {
regexp_module_ctx->max_size = ucl_obj_toint (value);
rspamd_mime_expression_set_re_limit (regexp_module_ctx->max_size);
}
else if (g_ascii_strncasecmp (ucl_object_key (value), "max_threads",
sizeof ("max_threads") - 1) == 0) {
msg_warn_config ("regexp module is now single threaded, max_threads is ignored");
}
else if (value->type == UCL_STRING) {
cur_item = rspamd_mempool_alloc0 (regexp_module_ctx->regexp_pool,
sizeof (struct regexp_module_item));
cur_item->symbol = ucl_object_key (value);
if (!read_regexp_expression (regexp_module_ctx->regexp_pool,
cur_item, ucl_object_key (value),
ucl_obj_tostring (value), cfg)) {
res = FALSE;
}
else {
rspamd_symbols_cache_add_symbol (cfg->cache,
cur_item->symbol,
0,
process_regexp_item,
cur_item,
SYMBOL_TYPE_NORMAL, -1);
}
}
else if (value->type == UCL_USERDATA) {
/* Just a lua function */
cur_item = rspamd_mempool_alloc0 (regexp_module_ctx->regexp_pool,
sizeof (struct regexp_module_item));
cur_item->symbol = ucl_object_key (value);
cur_item->lua_function = ucl_object_toclosure (value);
rspamd_symbols_cache_add_symbol (cfg->cache,
cur_item->symbol,
0,
process_regexp_item,
cur_item,
SYMBOL_TYPE_NORMAL, -1);
}
else if (value->type == UCL_OBJECT) {
const gchar *description = NULL, *group = NULL,
*metric = DEFAULT_METRIC;
gdouble score = 0.0;
gboolean one_shot = FALSE, is_lua = FALSE, valid_expression = TRUE;
/* We have some lua table, extract its arguments */
elt = ucl_object_find_key (value, "callback");
if (elt == NULL || elt->type != UCL_USERDATA) {
/* Try plain regexp expression */
elt = ucl_object_find_any_key (value, "regexp", "re", NULL);
if (elt != NULL && ucl_object_type (elt) == UCL_STRING) {
cur_item = rspamd_mempool_alloc0 (regexp_module_ctx->regexp_pool,
sizeof (struct regexp_module_item));
cur_item->symbol = ucl_object_key (value);
if (!read_regexp_expression (regexp_module_ctx->regexp_pool,
cur_item, ucl_object_key (value),
ucl_obj_tostring (elt), cfg)) {
res = FALSE;
}
else {
valid_expression = TRUE;
}
}
else {
msg_err_config (
"no callback/expression defined for regexp symbol: "
"%s", ucl_object_key (value));
}
}
else {
is_lua = TRUE;
cur_item = rspamd_mempool_alloc0 (
regexp_module_ctx->regexp_pool,
sizeof (struct regexp_module_item));
cur_item->symbol = ucl_object_key (value);
cur_item->lua_function = ucl_object_toclosure (value);
}
if (cur_item && (is_lua || valid_expression)) {
id = rspamd_symbols_cache_add_symbol (cfg->cache,
cur_item->symbol,
0,
process_regexp_item,
cur_item,
SYMBOL_TYPE_NORMAL, -1);
elt = ucl_object_find_key (value, "condition");
if (elt != NULL && ucl_object_type (elt) == UCL_USERDATA) {
struct ucl_lua_funcdata *conddata;
conddata = ucl_object_toclosure (elt);
rspamd_symbols_cache_add_condition (cfg->cache, id,
conddata->L, conddata->idx);
}
elt = ucl_object_find_key (value, "metric");
if (elt) {
metric = ucl_object_tostring (elt);
}
elt = ucl_object_find_key (value, "description");
if (elt) {
description = ucl_object_tostring (elt);
}
elt = ucl_object_find_key (value, "group");
if (elt) {
group = ucl_object_tostring (elt);
}
elt = ucl_object_find_key (value, "score");
if (elt) {
score = ucl_object_todouble (elt);
}
elt = ucl_object_find_key (value, "one_shot");
if (elt) {
one_shot = ucl_object_toboolean (elt);
}
rspamd_config_add_metric_symbol (cfg, metric, cur_item->symbol,
score, description, group, one_shot, FALSE);
}
}
else {
msg_warn_config ("unknown type of attribute %s for regexp module",
ucl_object_key (value));
}
}
return res;
}
struct rspamd_map *
rspamd_map_add (struct rspamd_config *cfg,
const gchar *map_line,
const gchar *description,
map_cb_t read_callback,
map_fin_cb_t fin_callback,
void **user_data)
{
struct rspamd_map *new_map;
const gchar *def;
struct file_map_data *fdata;
struct http_map_data *hdata;
gchar *cksum_encoded, cksum[rspamd_cryptobox_HASHBYTES];
rspamd_mempool_t *pool;
struct http_parser_url up;
rspamd_ftok_t tok;
if (cfg->map_pool == NULL) {
cfg->map_pool = rspamd_mempool_new (rspamd_mempool_suggest_size (),
"map");
memcpy (cfg->map_pool->tag.uid, cfg->cfg_pool->tag.uid,
sizeof (cfg->map_pool->tag.uid));
}
new_map = rspamd_mempool_alloc0 (cfg->map_pool, sizeof (struct rspamd_map));
/* First of all detect protocol line */
if (rspamd_map_check_proto (cfg, map_line, new_map) == NULL) {
return NULL;
}
new_map->read_callback = read_callback;
new_map->fin_callback = fin_callback;
new_map->user_data = user_data;
new_map->cfg = cfg;
new_map->id = g_random_int ();
new_map->locked =
rspamd_mempool_alloc0_shared (cfg->cfg_pool, sizeof (gint));
def = new_map->uri;
if (description != NULL) {
new_map->description =
rspamd_mempool_strdup (cfg->cfg_pool, description);
}
/* Now check for each proto separately */
if (new_map->protocol == MAP_PROTO_FILE) {
fdata =
rspamd_mempool_alloc0 (cfg->map_pool,
sizeof (struct file_map_data));
if (access (def, R_OK) == -1) {
if (errno != ENOENT) {
msg_err_config ("cannot open file '%s': %s", def, strerror
(errno));
return NULL;
}
msg_info_config (
"map '%s' is not found, but it can be loaded automatically later",
def);
/* We still can add this file */
fdata->st.st_mtime = -1;
}
else {
stat (def, &fdata->st);
}
fdata->filename = rspamd_mempool_strdup (cfg->map_pool, def);
new_map->map_data = fdata;
}
else if (new_map->protocol == MAP_PROTO_HTTP) {
hdata =
rspamd_mempool_alloc0 (cfg->map_pool,
sizeof (struct http_map_data));
memset (&up, 0, sizeof (up));
if (http_parser_parse_url (new_map->uri, strlen (new_map->uri), FALSE,
&up) != 0) {
msg_err_config ("cannot parse HTTP url: %s", new_map->uri);
return NULL;
}
else {
if (!(up.field_set & 1 << UF_HOST)) {
msg_err_config ("cannot parse HTTP url: %s: no host", new_map->uri);
return NULL;
}
tok.begin = new_map->uri + up.field_data[UF_HOST].off;
tok.len = up.field_data[UF_HOST].len;
hdata->host = rspamd_mempool_ftokdup (cfg->map_pool, &tok);
if (up.field_set & 1 << UF_PORT) {
hdata->port = up.port;
}
else {
hdata->port = 80;
}
if (up.field_set & 1 << UF_PATH) {
tok.begin = new_map->uri + up.field_data[UF_PATH].off;
tok.len = strlen (tok.begin);
hdata->path = rspamd_mempool_ftokdup (cfg->map_pool, &tok);
}
}
new_map->map_data = hdata;
}
/* Temp pool */
rspamd_cryptobox_hash (cksum, new_map->uri, strlen (new_map->uri), NULL, 0);
cksum_encoded = rspamd_encode_base32 (cksum, sizeof (cksum));
new_map->pool = rspamd_mempool_new (rspamd_mempool_suggest_size (), "map");
rspamd_strlcpy (new_map->pool->tag.uid, cksum_encoded,
sizeof (new_map->pool->tag.uid));
g_free (cksum_encoded);
pool = new_map->pool;
msg_info_pool ("added map %s", new_map->uri);
cfg->maps = g_list_prepend (cfg->maps, new_map);
return new_map;
}
gboolean
rspamd_map_add (struct rspamd_config *cfg,
const gchar *map_line,
const gchar *description,
map_cb_t read_callback,
map_fin_cb_t fin_callback,
void **user_data)
{
struct rspamd_map *new_map;
enum fetch_proto proto;
const gchar *def, *p, *hostend;
struct file_map_data *fdata;
struct http_map_data *hdata;
gchar portbuf[6], *cksum_encoded, cksum[BLAKE2B_OUTBYTES];
gint i, s, r;
struct addrinfo hints, *res;
rspamd_mempool_t *pool;
/* First of all detect protocol line */
if (!rspamd_map_check_proto (map_line, (int *)&proto, &def)) {
return FALSE;
}
/* Constant pool */
if (cfg->map_pool == NULL) {
cfg->map_pool = rspamd_mempool_new (rspamd_mempool_suggest_size (),
"map");
memcpy (cfg->map_pool->tag.uid, cfg->cfg_pool->tag.uid,
sizeof (cfg->map_pool->tag.uid));
}
new_map = rspamd_mempool_alloc0 (cfg->map_pool, sizeof (struct rspamd_map));
new_map->read_callback = read_callback;
new_map->fin_callback = fin_callback;
new_map->user_data = user_data;
new_map->protocol = proto;
new_map->cfg = cfg;
new_map->id = g_random_int ();
new_map->locked =
rspamd_mempool_alloc0_shared (cfg->cfg_pool, sizeof (gint));
if (proto == MAP_PROTO_FILE) {
new_map->uri = rspamd_mempool_strdup (cfg->cfg_pool, def);
def = new_map->uri;
}
else {
new_map->uri = rspamd_mempool_strdup (cfg->cfg_pool, map_line);
}
if (description != NULL) {
new_map->description =
rspamd_mempool_strdup (cfg->cfg_pool, description);
}
/* Now check for each proto separately */
if (proto == MAP_PROTO_FILE) {
fdata =
rspamd_mempool_alloc0 (cfg->map_pool,
sizeof (struct file_map_data));
if (access (def, R_OK) == -1) {
if (errno != ENOENT) {
msg_err_config ("cannot open file '%s': %s", def, strerror
(errno));
return FALSE;
}
msg_info_config (
"map '%s' is not found, but it can be loaded automatically later",
def);
/* We still can add this file */
fdata->st.st_mtime = -1;
}
else {
stat (def, &fdata->st);
}
fdata->filename = rspamd_mempool_strdup (cfg->map_pool, def);
new_map->map_data = fdata;
}
else if (proto == MAP_PROTO_HTTP) {
hdata =
rspamd_mempool_alloc0 (cfg->map_pool,
sizeof (struct http_map_data));
/* Try to search port */
if ((p = strchr (def, ':')) != NULL) {
hostend = p;
i = 0;
p++;
while (g_ascii_isdigit (*p) && i < (gint)sizeof (portbuf) - 1) {
portbuf[i++] = *p++;
}
if (*p != '/') {
msg_info_config ("bad http map definition: %s", def);
return FALSE;
}
portbuf[i] = '\0';
hdata->port = atoi (portbuf);
}
else {
/* Default http port */
rspamd_snprintf (portbuf, sizeof (portbuf), "80");
hdata->port = 80;
/* Now separate host from path */
if ((p = strchr (def, '/')) == NULL) {
msg_info_config ("bad http map definition: %s", def);
return FALSE;
}
hostend = p;
}
hdata->host = rspamd_mempool_alloc (cfg->map_pool, hostend - def + 1);
rspamd_strlcpy (hdata->host, def, hostend - def + 1);
hdata->path = rspamd_mempool_strdup (cfg->map_pool, p);
/* Now try to resolve */
memset (&hints, 0, sizeof (hints));
hints.ai_family = AF_UNSPEC; /* Allow IPv4 or IPv6 */
hints.ai_socktype = SOCK_STREAM; /* Stream socket */
hints.ai_flags = 0;
hints.ai_protocol = 0; /* Any protocol */
hints.ai_canonname = NULL;
hints.ai_addr = NULL;
hints.ai_next = NULL;
if ((r = getaddrinfo (hdata->host, portbuf, &hints, &res)) == 0) {
hdata->addr = res;
rspamd_mempool_add_destructor (cfg->cfg_pool,
(rspamd_mempool_destruct_t)freeaddrinfo, hdata->addr);
}
else {
msg_err_config ("address resolution for %s failed: %s",
hdata->host,
gai_strerror (r));
return FALSE;
}
/* Now try to connect */
if ((s = rspamd_socket_tcp (hdata->addr, FALSE, FALSE)) == -1) {
msg_info_config ("cannot connect to http server %s: %d, %s",
hdata->host,
errno,
strerror (errno));
return FALSE;
}
close (s);
hdata->conn = rspamd_http_connection_new (http_map_read, http_map_error,
http_map_finish,
RSPAMD_HTTP_BODY_PARTIAL | RSPAMD_HTTP_CLIENT_SIMPLE,
RSPAMD_HTTP_CLIENT, NULL);
new_map->map_data = hdata;
}
/* Temp pool */
blake2b (cksum, new_map->uri, NULL, sizeof (cksum), strlen (new_map->uri), 0);
cksum_encoded = rspamd_encode_base32 (cksum, sizeof (cksum));
new_map->pool = rspamd_mempool_new (rspamd_mempool_suggest_size (), "map");
memcpy (new_map->pool->tag.uid, cksum_encoded,
sizeof (new_map->pool->tag.uid));
g_free (cksum_encoded);
pool = new_map->pool;
msg_info_pool ("added map %s", new_map->uri);
cfg->maps = g_list_prepend (cfg->maps, new_map);
return TRUE;
}
static struct rspamd_map_backend *
rspamd_map_parse_backend (struct rspamd_config *cfg, const gchar *map_line)
{
struct rspamd_map_backend *bk;
struct file_map_data *fdata = NULL;
struct http_map_data *hdata = NULL;
struct http_parser_url up;
rspamd_ftok_t tok;
bk = g_slice_alloc0 (sizeof (*bk));
REF_INIT_RETAIN (bk, rspamd_map_backend_dtor);
if (!rspamd_map_check_proto (cfg, map_line, bk)) {
goto err;
}
/* Now check for each proto separately */
if (bk->protocol == MAP_PROTO_FILE) {
fdata = g_slice_alloc0 (sizeof (struct file_map_data));
fdata->st.st_mtime = -1;
if (access (bk->uri, R_OK) == -1) {
if (errno != ENOENT) {
msg_err_config ("cannot open file '%s': %s", bk->uri, strerror (errno));
return NULL;
}
msg_info_config (
"map '%s' is not found, but it can be loaded automatically later",
bk->uri);
}
fdata->filename = g_strdup (bk->uri);
bk->data.fd = fdata;
}
else if (bk->protocol == MAP_PROTO_HTTP || bk->protocol == MAP_PROTO_HTTPS) {
hdata = g_slice_alloc0 (sizeof (struct http_map_data));
memset (&up, 0, sizeof (up));
if (http_parser_parse_url (bk->uri, strlen (bk->uri), FALSE,
&up) != 0) {
msg_err_config ("cannot parse HTTP url: %s", bk->uri);
goto err;
}
else {
if (!(up.field_set & 1 << UF_HOST)) {
msg_err_config ("cannot parse HTTP url: %s: no host", bk->uri);
return NULL;
}
tok.begin = bk->uri + up.field_data[UF_HOST].off;
tok.len = up.field_data[UF_HOST].len;
hdata->host = rspamd_ftokdup (&tok);
if (up.field_set & 1 << UF_PORT) {
hdata->port = up.port;
}
else {
if (bk->protocol == MAP_PROTO_HTTP) {
hdata->port = 80;
}
else {
hdata->port = 443;
}
}
if (up.field_set & 1 << UF_PATH) {
tok.begin = bk->uri + up.field_data[UF_PATH].off;
tok.len = strlen (tok.begin);
hdata->path = rspamd_ftokdup (&tok);
}
}
bk->data.hd = hdata;
}
return bk;
err:
MAP_RELEASE (bk, "rspamd_map_backend");
if (hdata) {
g_slice_free1 (sizeof (*hdata), hdata);
}
if (fdata) {
g_slice_free1 (sizeof (*fdata), fdata);
}
return NULL;
}
gboolean
rspamd_lua_check_condition (struct rspamd_config *cfg, const gchar *condition)
{
lua_State *L = cfg->lua_state;
gchar *hostbuf, *condbuf;
gsize hostlen;
gboolean res;
#ifdef HAVE_SYS_UTSNAME_H
struct utsname uts;
#endif
/* Set some globals for condition */
/* XXX: think what other variables can be useful */
hostlen = sysconf (_SC_HOST_NAME_MAX) + 1;
hostbuf = alloca (hostlen);
gethostname (hostbuf, hostlen);
hostbuf[hostlen - 1] = '\0';
/* Hostname */
lua_pushstring (L, hostbuf);
lua_setglobal (L, "hostname");
/* Config file name */
lua_pushstring (L, cfg->cfg_name);
lua_setglobal (L, "cfg_name");
/* Check for uname */
#ifdef HAVE_SYS_UTSNAME_H
uname (&uts);
lua_pushstring (L, uts.sysname);
lua_setglobal (L, "osname");
lua_pushstring (L, uts.release);
lua_setglobal (L, "osrelease");
#else
lua_pushstring (L, "unknown");
lua_setglobal (L, "osname");
lua_pushstring (L, "");
lua_setglobal (L, "osrelease");
#endif
#ifdef HAVE_OPENSSL
lua_pushboolean (L, TRUE);
#else
lua_pushboolean (L, FALSE);
#endif
lua_setglobal (L, "rspamd_supports_rsa");
/* Rspamd paths */
lua_newtable (L);
rspamd_lua_table_set (L, "confdir", RSPAMD_CONFDIR);
rspamd_lua_table_set (L, "rundir", RSPAMD_RUNDIR);
rspamd_lua_table_set (L, "dbdir", RSPAMD_DBDIR);
rspamd_lua_table_set (L, "logdir", RSPAMD_LOGDIR);
rspamd_lua_table_set (L, "pluginsdir", RSPAMD_PLUGINSDIR);
rspamd_lua_table_set (L, "prefix", RSPAMD_PREFIX);
lua_setglobal (L, "rspamd_paths");
/* Make fake string */
hostlen = sizeof (FAKE_RES_VAR "=") + strlen (condition);
condbuf = g_malloc (hostlen);
rspamd_strlcpy (condbuf, FAKE_RES_VAR "=", sizeof (FAKE_RES_VAR "="));
g_strlcat (condbuf, condition, hostlen);
/* Evaluate condition */
if (luaL_dostring (L, condbuf) != 0) {
msg_err_config("eval of '%s' failed: '%s'", condition, lua_tostring (L, -1));
g_free (condbuf);
return FALSE;
}
/* Get global variable res to get result */
lua_getglobal (L, FAKE_RES_VAR);
if (!lua_isboolean (L, -1)) {
msg_err_config("bad string evaluated: %s, type: %s", condbuf,
lua_typename (L, lua_type (L, -1)));
g_free (condbuf);
return FALSE;
}
res = lua_toboolean (L, -1);
g_free (condbuf);
return res;
}
/* Do post load initialization based on lua */
void
rspamd_lua_post_load_config (struct rspamd_config *cfg)
{
lua_State *L = cfg->lua_state;
const gchar *name, *val;
gchar *sym;
struct rspamd_expression *expr, *old_expr;
ucl_object_t *obj;
gsize keylen;
GError *err = NULL;
/* First check all module options that may be overriden in 'config' global */
lua_getglobal (L, "config");
if (lua_istable (L, -1)) {
/* Iterate */
for (lua_pushnil (L); lua_next (L, -2); lua_pop (L, 1)) {
/* 'key' is at index -2 and 'value' is at index -1 */
/* Key must be a string and value must be a table */
name = luaL_checklstring (L, -2, &keylen);
if (name != NULL && lua_istable (L, -1)) {
obj = ucl_object_lua_import (L, lua_gettop (L));
if (obj != NULL) {
ucl_object_insert_key_merged (cfg->rcl_obj,
obj,
name,
keylen,
true);
}
}
}
}
/* Check metrics settings */
lua_getglobal (L, "metrics");
if (lua_istable (L, -1)) {
/* Iterate */
for (lua_pushnil (L); lua_next (L, -2); lua_pop (L, 1)) {
/* 'key' is at index -2 and 'value' is at index -1 */
/* Key must be a string and value must be a table */
name = luaL_checkstring (L, -2);
if (name != NULL && lua_istable (L, -1)) {
lua_process_metric (L, name, cfg);
}
}
}
/* Check composites */
lua_getglobal (L, "composites");
if (lua_istable (L, -1)) {
/* Iterate */
for (lua_pushnil (L); lua_next (L, -2); lua_pop (L, 1)) {
/* 'key' is at index -2 and 'value' is at index -1 */
/* Key must be a string and value must be a table */
name = luaL_checkstring (L, -2);
if (name != NULL && lua_isstring (L, -1)) {
val = lua_tostring (L, -1);
sym = rspamd_mempool_strdup (cfg->cfg_pool, name);
if (!rspamd_parse_expression (val, 0, &composite_expr_subr, NULL,
cfg->cfg_pool, &err, &expr)) {
msg_err_config("cannot parse composite expression '%s': %s", val,
err->message);
g_error_free (err);
err = NULL;
continue;
}
/* Now check hash table for this composite */
if ((old_expr =
g_hash_table_lookup (cfg->composite_symbols,
name)) != NULL) {
msg_info_config("replacing composite symbol %s", name);
g_hash_table_replace (cfg->composite_symbols, sym, expr);
}
else {
g_hash_table_insert (cfg->composite_symbols, sym, expr);
rspamd_symbols_cache_add_symbol (cfg->cache, sym,
0, NULL, NULL, SYMBOL_TYPE_COMPOSITE, -1);
}
}
}
}
}
gpointer
rspamd_stat_cache_redis_init (struct rspamd_stat_ctx *ctx,
struct rspamd_config *cfg,
struct rspamd_statfile *st,
const ucl_object_t *cf)
{
struct rspamd_redis_cache_ctx *cache_ctx;
struct rspamd_statfile_config *stf = st->stcf;
const ucl_object_t *obj;
gboolean ret = FALSE;
cache_ctx = g_slice_alloc0 (sizeof (*cache_ctx));
/* First search in backend configuration */
obj = ucl_object_lookup (st->classifier->cfg->opts, "backend");
if (obj != NULL && ucl_object_type (obj) == UCL_OBJECT) {
ret = rspamd_redis_cache_try_ucl (cache_ctx, obj, cfg, stf->symbol);
}
/* Now try statfiles config */
if (!ret) {
ret = rspamd_redis_cache_try_ucl (cache_ctx, stf->opts, cfg, stf->symbol);
}
/* Now try classifier config */
if (!ret) {
ret = rspamd_redis_cache_try_ucl (cache_ctx, st->classifier->cfg->opts, cfg,
stf->symbol);
}
/* Now try global redis settings */
if (!ret) {
obj = ucl_object_lookup (cfg->rcl_obj, "redis");
if (obj) {
const ucl_object_t *specific_obj;
specific_obj = ucl_object_lookup (obj, "statistics");
if (specific_obj) {
ret = rspamd_redis_cache_try_ucl (cache_ctx, specific_obj, cfg,
stf->symbol);
}
else {
ret = rspamd_redis_cache_try_ucl (cache_ctx, obj, cfg,
stf->symbol);
}
}
}
if (!ret) {
msg_err_config ("cannot init redis cache for %s", stf->symbol);
g_slice_free1 (sizeof (*cache_ctx), cache_ctx);
return NULL;
}
cache_ctx->stcf = stf;
return (gpointer)cache_ctx;
}
static gboolean
rspamd_redis_try_ucl (struct redis_stat_ctx *backend,
const ucl_object_t *obj,
struct rspamd_config *cfg,
const gchar *symbol)
{
const ucl_object_t *elt, *relt, *users_enabled;
const gchar *lua_script;
elt = ucl_object_lookup_any (obj, "read_servers", "servers", NULL);
if (elt == NULL) {
return FALSE;
}
backend->read_servers = rspamd_upstreams_create (cfg->ups_ctx);
if (!rspamd_upstreams_from_ucl (backend->read_servers, elt,
REDIS_DEFAULT_PORT, NULL)) {
msg_err ("statfile %s cannot get read servers configuration",
symbol);
return FALSE;
}
relt = elt;
elt = ucl_object_lookup (obj, "write_servers");
if (elt == NULL) {
/* Use read servers as write ones */
g_assert (relt != NULL);
backend->write_servers = rspamd_upstreams_create (cfg->ups_ctx);
if (!rspamd_upstreams_from_ucl (backend->write_servers, relt,
REDIS_DEFAULT_PORT, NULL)) {
msg_err ("statfile %s cannot get write servers configuration",
symbol);
return FALSE;
}
}
else {
backend->write_servers = rspamd_upstreams_create (cfg->ups_ctx);
if (!rspamd_upstreams_from_ucl (backend->write_servers, elt,
REDIS_DEFAULT_PORT, NULL)) {
msg_err ("statfile %s cannot get write servers configuration",
symbol);
rspamd_upstreams_destroy (backend->write_servers);
backend->write_servers = NULL;
}
}
elt = ucl_object_lookup (obj, "prefix");
if (elt == NULL || ucl_object_type (elt) != UCL_STRING) {
/* Default non-users statistics */
backend->redis_object = REDIS_DEFAULT_OBJECT;
/*
* Make redis backend compatible with sqlite3 backend in users settings
*/
users_enabled = ucl_object_lookup_any (obj, "per_user",
"users_enabled", NULL);
if (users_enabled != NULL) {
if (ucl_object_type (users_enabled) == UCL_BOOLEAN) {
backend->enable_users = ucl_object_toboolean (users_enabled);
backend->cbref_user = -1;
if (backend->enable_users) {
backend->redis_object = REDIS_DEFAULT_USERS_OBJECT;
}
}
else if (ucl_object_type (users_enabled) == UCL_STRING) {
lua_script = ucl_object_tostring (users_enabled);
if (luaL_dostring (cfg->lua_state, lua_script) != 0) {
msg_err_config ("cannot execute lua script for users "
"extraction: %s", lua_tostring (cfg->lua_state, -1));
}
else {
if (lua_type (cfg->lua_state, -1) == LUA_TFUNCTION) {
backend->enable_users = TRUE;
backend->cbref_user = luaL_ref (cfg->lua_state,
LUA_REGISTRYINDEX);
}
else {
msg_err_config ("lua script must return "
"function(task) and not %s",
lua_typename (cfg->lua_state, lua_type (
cfg->lua_state, -1)));
}
}
}
}
else {
backend->enable_users = FALSE;
}
}
else {
/* XXX: sanity check */
backend->redis_object = ucl_object_tostring (elt);
}
elt = ucl_object_lookup (obj, "timeout");
if (elt) {
backend->timeout = ucl_object_todouble (elt);
}
else {
backend->timeout = REDIS_DEFAULT_TIMEOUT;
}
elt = ucl_object_lookup (obj, "password");
if (elt) {
backend->password = ucl_object_tostring (elt);
}
else {
backend->password = NULL;
}
elt = ucl_object_lookup_any (obj, "db", "database", "dbname", NULL);
if (elt) {
backend->dbname = ucl_object_tostring (elt);
}
else {
backend->dbname = NULL;
}
return TRUE;
}
static gint
fuzzy_parse_rule (struct rspamd_config *cfg, const ucl_object_t *obj, gint cb_id)
{
const ucl_object_t *value, *cur;
struct fuzzy_rule *rule;
ucl_object_iter_t it = NULL;
const char *k = NULL;
if (obj->type != UCL_OBJECT) {
msg_err_config ("invalid rule definition");
return -1;
}
rule = fuzzy_rule_new (fuzzy_module_ctx->default_symbol,
fuzzy_module_ctx->fuzzy_pool);
if ((value = ucl_object_find_key (obj, "mime_types")) != NULL) {
it = NULL;
while ((cur = ucl_iterate_object (value, &it, value->type == UCL_ARRAY))
!= NULL) {
rule->mime_types = g_list_concat (rule->mime_types,
parse_mime_types (ucl_obj_tostring (cur)));
}
}
if (rule->mime_types != NULL) {
rspamd_mempool_add_destructor (fuzzy_module_ctx->fuzzy_pool,
(rspamd_mempool_destruct_t)g_list_free, rule->mime_types);
}
if ((value = ucl_object_find_key (obj, "max_score")) != NULL) {
rule->max_score = ucl_obj_todouble (value);
}
if ((value = ucl_object_find_key (obj, "symbol")) != NULL) {
rule->symbol = ucl_obj_tostring (value);
}
if ((value = ucl_object_find_key (obj, "read_only")) != NULL) {
rule->read_only = ucl_obj_toboolean (value);
}
if ((value = ucl_object_find_key (obj, "skip_unknown")) != NULL) {
rule->skip_unknown = ucl_obj_toboolean (value);
}
if ((value = ucl_object_find_key (obj, "servers")) != NULL) {
rule->servers = rspamd_upstreams_create ();
rspamd_mempool_add_destructor (fuzzy_module_ctx->fuzzy_pool,
(rspamd_mempool_destruct_t)rspamd_upstreams_destroy,
rule->servers);
rspamd_upstreams_from_ucl (rule->servers, value, DEFAULT_PORT, NULL);
}
if ((value = ucl_object_find_key (obj, "fuzzy_map")) != NULL) {
it = NULL;
while ((cur = ucl_iterate_object (value, &it, true)) != NULL) {
parse_flags (rule, cfg, cur, cb_id);
}
}
if ((value = ucl_object_find_key (obj, "encryption_key")) != NULL) {
/* Create key from user's input */
k = ucl_object_tostring (value);
if (k == NULL || (rule->peer_key =
rspamd_http_connection_make_peer_key (k)) == NULL) {
msg_err_config ("bad encryption key value: %s",
k);
return -1;
}
rule->local_key = rspamd_http_connection_gen_key ();
}
if ((value = ucl_object_find_key (obj, "fuzzy_key")) != NULL) {
/* Create key from user's input */
k = ucl_object_tostring (value);
}
/* Setup keys */
if (k == NULL) {
/* Use some default key for all ops */
k = "rspamd";
}
rule->hash_key = g_string_sized_new (BLAKE2B_KEYBYTES);
blake2 (rule->hash_key->str, k, NULL, BLAKE2B_KEYBYTES, strlen (k), 0);
rule->hash_key->len = BLAKE2B_KEYBYTES;
if ((value = ucl_object_find_key (obj, "fuzzy_shingles_key")) != NULL) {
k = ucl_object_tostring (value);
}
if (k == NULL) {
k = "rspamd";
}
rule->shingles_key = g_string_sized_new (16);
blake2 (rule->shingles_key->str, k, NULL, 16, strlen (k), 0);
rule->shingles_key->len = 16;
if (rspamd_upstreams_count (rule->servers) == 0) {
msg_err_config ("no servers defined for fuzzy rule with symbol: %s",
rule->symbol);
return -1;
}
else {
fuzzy_module_ctx->fuzzy_rules = g_list_prepend (
fuzzy_module_ctx->fuzzy_rules,
rule);
if (rule->symbol != fuzzy_module_ctx->default_symbol) {
rspamd_symbols_cache_add_symbol (cfg->cache, rule->symbol,
0,
NULL, NULL,
SYMBOL_TYPE_VIRTUAL|SYMBOL_TYPE_FINE,
cb_id);
}
}
rspamd_mempool_add_destructor (fuzzy_module_ctx->fuzzy_pool, fuzzy_free_rule,
rule);
return 0;
}
gboolean
rspamd_config_add_metric_symbol (struct rspamd_config *cfg,
const gchar *metric_name, const gchar *symbol,
gdouble score, const gchar *description, const gchar *group,
gboolean one_shot, gboolean rewrite_existing)
{
struct rspamd_symbols_group *sym_group;
struct rspamd_symbol_def *sym_def;
GList *metric_list;
struct metric *metric;
gdouble *score_ptr;
g_assert (cfg != NULL);
g_assert (symbol != NULL);
if (metric_name == NULL) {
metric_name = DEFAULT_METRIC;
}
metric = g_hash_table_lookup (cfg->metrics, metric_name);
if (metric == NULL) {
msg_err_config ("metric %s has not been found", metric_name);
return FALSE;
}
if (g_hash_table_lookup (cfg->metrics_symbols, symbol) != NULL &&
!rewrite_existing) {
msg_debug_config ("symbol %s has been already registered, do not override");
return FALSE;
}
sym_def =
rspamd_mempool_alloc0 (cfg->cfg_pool, sizeof (struct rspamd_symbol_def));
score_ptr = rspamd_mempool_alloc (cfg->cfg_pool, sizeof (gdouble));
*score_ptr = score;
sym_def->score = score;
sym_def->weight_ptr = score_ptr;
sym_def->name = rspamd_mempool_strdup (cfg->cfg_pool, symbol);
sym_def->one_shot = one_shot;
if (description) {
sym_def->description = rspamd_mempool_strdup (cfg->cfg_pool, description);
}
msg_debug_config ("registered symbol %s with weight %.2f in metric %s and group %s",
sym_def->name, score, metric->name, group);
g_hash_table_insert (metric->symbols, sym_def->name, sym_def);
if ((metric_list =
g_hash_table_lookup (cfg->metrics_symbols, sym_def->name)) == NULL) {
metric_list = g_list_prepend (NULL, metric);
rspamd_mempool_add_destructor (cfg->cfg_pool,
(rspamd_mempool_destruct_t)g_list_free,
metric_list);
g_hash_table_insert (cfg->metrics_symbols, sym_def->name, metric_list);
}
else {
/* Slow but keep start element of list in safe */
if (!g_list_find (metric_list, metric)) {
metric_list = g_list_append (metric_list, metric);
}
}
/* Search for symbol group */
if (group == NULL) {
group = "ungrouped";
}
sym_group = g_hash_table_lookup (metric->groups, group);
if (sym_group == NULL) {
/* Create new group */
sym_group = rspamd_config_new_group (cfg, metric, group);
}
sym_def->gr = sym_group;
g_hash_table_insert (sym_group->symbols, sym_def->name, sym_def);
return TRUE;
}
struct rspamd_map*
rspamd_map_add_from_ucl (struct rspamd_config *cfg,
const ucl_object_t *obj,
const gchar *description,
map_cb_t read_callback,
map_fin_cb_t fin_callback,
void **user_data)
{
ucl_object_iter_t it = NULL;
const ucl_object_t *cur, *elt;
struct rspamd_map *map;
struct rspamd_map_backend *bk;
g_assert (obj != NULL);
if (ucl_object_type (obj) == UCL_STRING) {
/* Just a plain string */
return rspamd_map_add (cfg, ucl_object_tostring (obj), NULL,
read_callback, fin_callback, user_data);
}
map = g_slice_alloc0 (sizeof (struct rspamd_map));
map->read_callback = read_callback;
map->fin_callback = fin_callback;
map->user_data = user_data;
map->cfg = cfg;
map->id = g_random_int ();
map->locked =
rspamd_mempool_alloc0_shared (cfg->cfg_pool, sizeof (gint));
map->cache =
rspamd_mempool_alloc0_shared (cfg->cfg_pool, sizeof (*map->cache));
map->backends = g_ptr_array_new ();
map->poll_timeout = cfg->map_timeout;
if (description) {
map->description = g_strdup (description);
}
if (ucl_object_type (obj) == UCL_ARRAY) {
/* Add array of maps as multiple backends */
while ((cur = ucl_object_iterate (obj, &it, true)) != NULL) {
if (ucl_object_type (cur) == UCL_STRING) {
bk = rspamd_map_parse_backend (cfg, ucl_object_tostring (cur));
if (bk != NULL) {
g_ptr_array_add (map->backends, bk);
if (!map->name) {
map->name = g_strdup (ucl_object_tostring (cur));
}
}
}
else {
msg_err_config ("bad map element type: %s",
ucl_object_type_to_string (ucl_object_type (cur)));
}
}
if (map->backends->len == 0) {
msg_err_config ("map has no urls to be loaded");
goto err;
}
}
else if (ucl_object_type (obj) == UCL_OBJECT) {
elt = ucl_object_lookup (obj, "name");
if (elt && ucl_object_type (elt) == UCL_STRING) {
map->name = g_strdup (ucl_object_tostring (elt));
}
elt = ucl_object_lookup (obj, "description");
if (elt && ucl_object_type (elt) == UCL_STRING) {
if (map->description) {
g_free (map->description);
}
map->description = g_strdup (ucl_object_tostring (elt));
}
elt = ucl_object_lookup_any (obj, "timeout", "poll", "poll_time",
"watch_interval", NULL);
if (elt) {
map->poll_timeout = ucl_object_todouble (elt);
}
elt = ucl_object_lookup_any (obj, "upstreams", "url", "urls", NULL);
if (elt == NULL) {
msg_err_config ("map has no urls to be loaded");
goto err;
}
if (ucl_object_type (obj) == UCL_ARRAY) {
/* Add array of maps as multiple backends */
while ((cur = ucl_object_iterate (elt, &it, true)) != NULL) {
if (ucl_object_type (cur) == UCL_STRING) {
bk = rspamd_map_parse_backend (cfg, ucl_object_tostring (cur));
if (bk != NULL) {
g_ptr_array_add (map->backends, bk);
if (!map->name) {
map->name = g_strdup (ucl_object_tostring (cur));
}
}
}
else {
msg_err_config ("bad map element type: %s",
ucl_object_type_to_string (ucl_object_type (cur)));
goto err;
}
}
if (map->backends->len == 0) {
msg_err_config ("map has no urls to be loaded");
goto err;
}
}
else if (ucl_object_type (elt) == UCL_STRING) {
bk = rspamd_map_parse_backend (cfg, ucl_object_tostring (elt));
if (bk != NULL) {
g_ptr_array_add (map->backends, bk);
if (!map->name) {
map->name = g_strdup (ucl_object_tostring (cur));
}
}
}
if (map->backends->len == 0) {
msg_err_config ("map has no urls to be loaded");
goto err;
}
}
rspamd_map_calculate_hash (map);
msg_info_map ("added map from ucl");
cfg->maps = g_list_prepend (cfg->maps, map);
return map;
err:
g_ptr_array_free (map->backends, TRUE);
g_free (map->name);
g_free (map->description);
g_slice_free1 (sizeof (*map), map);
return NULL;
}
static const gchar *
rspamd_map_check_proto (struct rspamd_config *cfg,
const gchar *map_line, struct rspamd_map *map)
{
const gchar *pos = map_line, *end, *end_key;
g_assert (map != NULL);
g_assert (pos != NULL);
end = pos + strlen (pos);
if (g_ascii_strncasecmp (pos, "sign+", sizeof ("sign+") - 1) == 0) {
map->is_signed = TRUE;
pos += sizeof ("sign+") - 1;
}
if (g_ascii_strncasecmp (pos, "key=", sizeof ("key=") - 1) == 0) {
pos += sizeof ("key=") - 1;
end_key = memchr (pos, '+', end - pos);
if (end_key != NULL) {
map->trusted_pubkey = rspamd_pubkey_from_base32 (pos, end_key - pos,
RSPAMD_KEYPAIR_SIGN, RSPAMD_CRYPTOBOX_MODE_25519);
if (map->trusted_pubkey == NULL) {
msg_err_config ("cannot read pubkey from map: %s",
map_line);
return NULL;
}
pos = end_key + 1;
}
else if (end - pos > 64) {
/* Try hex encoding */
map->trusted_pubkey = rspamd_pubkey_from_hex (pos, 64,
RSPAMD_KEYPAIR_SIGN, RSPAMD_CRYPTOBOX_MODE_25519);
if (map->trusted_pubkey == NULL) {
msg_err_config ("cannot read pubkey from map: %s",
map_line);
return NULL;
}
pos += 64;
}
else {
msg_err_config ("cannot read pubkey from map: %s",
map_line);
return NULL;
}
if (*pos == '+' || *pos == ':') {
pos ++;
}
}
map->protocol = MAP_PROTO_FILE;
if (g_ascii_strncasecmp (pos, "http://",
sizeof ("http://") - 1) == 0) {
map->protocol = MAP_PROTO_HTTP;
/* Include http:// */
map->uri = rspamd_mempool_strdup (cfg->cfg_pool, pos);
pos += sizeof ("http://") - 1;
}
else if (g_ascii_strncasecmp (pos, "file://", sizeof ("file://") -
1) == 0) {
pos += sizeof ("file://") - 1;
/* Exclude file:// */
map->uri = rspamd_mempool_strdup (cfg->cfg_pool, pos);
}
else if (*pos == '/') {
/* Trivial file case */
map->uri = rspamd_mempool_strdup (cfg->cfg_pool, pos);
}
else {
msg_err_config ("invalid map fetching protocol: %s", map_line);
return NULL;
}
return pos;
}
void*
rspamd_fuzzy_backend_init_redis (struct rspamd_fuzzy_backend *bk,
const ucl_object_t *obj, struct rspamd_config *cfg, GError **err)
{
struct rspamd_fuzzy_backend_redis *backend;
const ucl_object_t *elt;
gboolean ret = FALSE;
guchar id_hash[rspamd_cryptobox_HASHBYTES];
rspamd_cryptobox_hash_state_t st;
backend = g_slice_alloc0 (sizeof (*backend));
backend->timeout = REDIS_DEFAULT_TIMEOUT;
backend->redis_object = REDIS_DEFAULT_OBJECT;
ret = rspamd_fuzzy_backend_redis_try_ucl (backend, obj, cfg);
/* Now try global redis settings */
if (!ret) {
elt = ucl_object_lookup (cfg->rcl_obj, "redis");
if (elt) {
const ucl_object_t *specific_obj;
specific_obj = ucl_object_lookup_any (elt, "fuzzy", "fuzzy_storage",
NULL);
if (specific_obj) {
ret = rspamd_fuzzy_backend_redis_try_ucl (backend, specific_obj,
cfg);
}
else {
ret = rspamd_fuzzy_backend_redis_try_ucl (backend, elt, cfg);
}
}
}
if (!ret) {
msg_err_config ("cannot init redis backend for fuzzy storage");
g_slice_free1 (sizeof (*backend), backend);
return NULL;
}
REF_INIT_RETAIN (backend, rspamd_fuzzy_backend_redis_dtor);
backend->pool = cfg->redis_pool;
rspamd_cryptobox_hash_init (&st, NULL, 0);
rspamd_cryptobox_hash_update (&st, backend->redis_object,
strlen (backend->redis_object));
if (backend->dbname) {
rspamd_cryptobox_hash_update (&st, backend->dbname,
strlen (backend->dbname));
}
if (backend->password) {
rspamd_cryptobox_hash_update (&st, backend->password,
strlen (backend->password));
}
rspamd_cryptobox_hash_final (&st, id_hash);
backend->id = rspamd_encode_base32 (id_hash, sizeof (id_hash));
return backend;
}
gint
dkim_module_config (struct rspamd_config *cfg)
{
const ucl_object_t *value;
gint res = TRUE, cb_id = -1, check_id = -1;
guint cache_size;
gboolean got_trusted = FALSE;
if (!rspamd_config_is_module_enabled (cfg, "dkim")) {
return TRUE;
}
dkim_module_ctx->whitelist_ip = radix_create_compressed ();
if ((value =
rspamd_config_get_module_opt (cfg, "dkim", "symbol_reject")) != NULL) {
dkim_module_ctx->symbol_reject = ucl_obj_tostring (value);
}
else {
dkim_module_ctx->symbol_reject = DEFAULT_SYMBOL_REJECT;
}
if ((value =
rspamd_config_get_module_opt (cfg, "dkim",
"symbol_tempfail")) != NULL) {
dkim_module_ctx->symbol_tempfail = ucl_obj_tostring (value);
}
else {
dkim_module_ctx->symbol_tempfail = DEFAULT_SYMBOL_TEMPFAIL;
}
if ((value =
rspamd_config_get_module_opt (cfg, "dkim", "symbol_allow")) != NULL) {
dkim_module_ctx->symbol_allow = ucl_obj_tostring (value);
}
else {
dkim_module_ctx->symbol_allow = DEFAULT_SYMBOL_ALLOW;
}
if ((value =
rspamd_config_get_module_opt (cfg, "dkim", "symbol_na")) != NULL) {
dkim_module_ctx->symbol_na = ucl_obj_tostring (value);
}
else {
dkim_module_ctx->symbol_na = DEFAULT_SYMBOL_NA;
}
if ((value =
rspamd_config_get_module_opt (cfg, "dkim",
"dkim_cache_size")) != NULL) {
cache_size = ucl_obj_toint (value);
}
else {
cache_size = DEFAULT_CACHE_SIZE;
}
if ((value =
rspamd_config_get_module_opt (cfg, "dkim", "time_jitter")) != NULL) {
dkim_module_ctx->time_jitter = ucl_obj_todouble (value);
}
else {
dkim_module_ctx->time_jitter = DEFAULT_TIME_JITTER;
}
if ((value =
rspamd_config_get_module_opt (cfg, "dkim", "max_sigs")) != NULL) {
dkim_module_ctx->max_sigs = ucl_object_toint (value);
}
if ((value =
rspamd_config_get_module_opt (cfg, "dkim", "whitelist")) != NULL) {
rspamd_config_radix_from_ucl (cfg, value, "DKIM whitelist",
&dkim_module_ctx->whitelist_ip, NULL);
}
if ((value =
rspamd_config_get_module_opt (cfg, "dkim", "domains")) != NULL) {
if (!rspamd_map_add_from_ucl (cfg, value,
"DKIM domains", rspamd_kv_list_read, rspamd_kv_list_fin,
(void **)&dkim_module_ctx->dkim_domains)) {
msg_warn_config ("cannot load dkim domains list from %s",
ucl_obj_tostring (value));
}
else {
got_trusted = TRUE;
}
}
if (!got_trusted && (value =
rspamd_config_get_module_opt (cfg, "dkim", "trusted_domains")) != NULL) {
if (!rspamd_map_add_from_ucl (cfg, value,
"DKIM domains", rspamd_kv_list_read, rspamd_kv_list_fin,
(void **)&dkim_module_ctx->dkim_domains)) {
msg_warn_config ("cannot load dkim domains list from %s",
ucl_obj_tostring (value));
}
else {
got_trusted = TRUE;
}
}
if ((value =
rspamd_config_get_module_opt (cfg, "dkim",
"strict_multiplier")) != NULL) {
dkim_module_ctx->strict_multiplier = ucl_obj_toint (value);
}
else {
dkim_module_ctx->strict_multiplier = 1;
}
if ((value =
rspamd_config_get_module_opt (cfg, "dkim", "trusted_only")) != NULL) {
dkim_module_ctx->trusted_only = ucl_obj_toboolean (value);
}
else {
dkim_module_ctx->trusted_only = FALSE;
}
if ((value =
rspamd_config_get_module_opt (cfg, "dkim", "skip_multi")) != NULL) {
dkim_module_ctx->skip_multi = ucl_obj_toboolean (value);
}
else {
dkim_module_ctx->skip_multi = FALSE;
}
if (dkim_module_ctx->trusted_only && !got_trusted) {
msg_err_config (
"trusted_only option is set and no trusted domains are defined; disabling dkim module completely as it is useless in this case");
}
else {
cb_id = rspamd_symbols_cache_add_symbol (cfg->cache,
dkim_module_ctx->symbol_reject,
0,
dkim_symbol_callback,
NULL,
SYMBOL_TYPE_NORMAL|SYMBOL_TYPE_FINE,
-1);
rspamd_symbols_cache_add_symbol (cfg->cache,
dkim_module_ctx->symbol_na,
0,
NULL, NULL,
SYMBOL_TYPE_VIRTUAL|SYMBOL_TYPE_FINE,
cb_id);
rspamd_symbols_cache_add_symbol (cfg->cache,
dkim_module_ctx->symbol_tempfail,
0,
NULL, NULL,
SYMBOL_TYPE_VIRTUAL|SYMBOL_TYPE_FINE,
cb_id);
rspamd_symbols_cache_add_symbol (cfg->cache,
dkim_module_ctx->symbol_allow,
0,
NULL, NULL,
SYMBOL_TYPE_VIRTUAL|SYMBOL_TYPE_FINE,
cb_id);
dkim_module_ctx->dkim_hash = rspamd_lru_hash_new (
cache_size,
g_free, /* Keys are just C-strings */
dkim_module_key_dtor);
msg_info_config ("init internal dkim module");
#ifndef HAVE_OPENSSL
msg_warn_config (
"openssl is not found so dkim rsa check is disabled, only check body hash, it is NOT safe to trust these results");
#endif
}
if ((value = rspamd_config_get_module_opt (cfg, "dkim", "sign_condition"))
!= NULL) {
const gchar *lua_script;
lua_script = ucl_object_tostring (value);
if (lua_script) {
if (luaL_dostring (cfg->lua_state, lua_script) != 0) {
msg_err_config ("cannot execute lua script for fuzzy "
"learn condition: %s", lua_tostring (cfg->lua_state, -1));
}
else {
if (lua_type (cfg->lua_state, -1) == LUA_TFUNCTION) {
dkim_module_ctx->sign_condition_ref = luaL_ref (cfg->lua_state,
LUA_REGISTRYINDEX);
dkim_module_ctx->dkim_sign_hash = rspamd_lru_hash_new (
128,
g_free, /* Keys are just C-strings */
(GDestroyNotify)rspamd_dkim_sign_key_unref);
check_id = rspamd_symbols_cache_add_symbol (cfg->cache,
"DKIM_SIGN",
0,
dkim_sign_callback,
NULL,
SYMBOL_TYPE_CALLBACK|SYMBOL_TYPE_FINE,
-1);
msg_info_config ("init condition script for DKIM signing");
/*
* Allow dkim signing to be executed only after dkim check
*/
if (cb_id > 0) {
rspamd_symbols_cache_add_delayed_dependency (cfg->cache,
"DKIM_SIGN", dkim_module_ctx->symbol_reject);
}
rspamd_config_add_metric_symbol (cfg, DEFAULT_METRIC,
"DKIM_SIGN", 0.0, "DKIM signature fake symbol",
"dkim", RSPAMD_SYMBOL_FLAG_IGNORE, 1);
}
else {
msg_err_config ("lua script must return "
"function(task) and not %s",
lua_typename (cfg->lua_state,
lua_type (cfg->lua_state, -1)));
}
}
}
}
return res;
}
static gboolean
rspamd_fuzzy_backend_redis_try_ucl (struct rspamd_fuzzy_backend_redis *backend,
const ucl_object_t *obj,
struct rspamd_config *cfg)
{
const ucl_object_t *elt, *relt;
elt = ucl_object_lookup_any (obj, "read_servers", "servers", NULL);
if (elt == NULL) {
return FALSE;
}
backend->read_servers = rspamd_upstreams_create (cfg->ups_ctx);
if (!rspamd_upstreams_from_ucl (backend->read_servers, elt,
REDIS_DEFAULT_PORT, NULL)) {
msg_err_config ("cannot get read servers configuration");
return FALSE;
}
relt = elt;
elt = ucl_object_lookup (obj, "write_servers");
if (elt == NULL) {
/* Use read servers as write ones */
g_assert (relt != NULL);
backend->write_servers = rspamd_upstreams_create (cfg->ups_ctx);
if (!rspamd_upstreams_from_ucl (backend->write_servers, relt,
REDIS_DEFAULT_PORT, NULL)) {
msg_err_config ("cannot get write servers configuration");
return FALSE;
}
}
else {
backend->write_servers = rspamd_upstreams_create (cfg->ups_ctx);
if (!rspamd_upstreams_from_ucl (backend->write_servers, elt,
REDIS_DEFAULT_PORT, NULL)) {
msg_err_config ("cannot get write servers configuration");
rspamd_upstreams_destroy (backend->write_servers);
backend->write_servers = NULL;
}
}
elt = ucl_object_lookup (obj, "prefix");
if (elt == NULL || ucl_object_type (elt) != UCL_STRING) {
backend->redis_object = REDIS_DEFAULT_OBJECT;
}
else {
backend->redis_object = ucl_object_tostring (elt);
}
elt = ucl_object_lookup (obj, "timeout");
if (elt) {
backend->timeout = ucl_object_todouble (elt);
}
else {
backend->timeout = REDIS_DEFAULT_TIMEOUT;
}
elt = ucl_object_lookup (obj, "password");
if (elt) {
backend->password = ucl_object_tostring (elt);
}
else {
backend->password = NULL;
}
elt = ucl_object_lookup_any (obj, "db", "database", "dbname", NULL);
if (elt) {
backend->dbname = ucl_object_tostring (elt);
}
else {
backend->dbname = NULL;
}
return TRUE;
}
