static void parse_flags (struct fuzzy_rule *rule, struct rspamd_config *cfg, const ucl_object_t *val, gint cb_id) { const ucl_object_t *elt; struct fuzzy_mapping *map; const gchar *sym = NULL; if (val->type == UCL_STRING) { msg_err_config ( "string mappings are deprecated and no longer supported, use new style configuration"); } else if (val->type == UCL_OBJECT) { elt = ucl_object_find_key (val, "symbol"); if (elt == NULL || !ucl_object_tostring_safe (elt, &sym)) { sym = ucl_object_key (val); } if (sym != NULL) { map = rspamd_mempool_alloc (fuzzy_module_ctx->fuzzy_pool, sizeof (struct fuzzy_mapping)); map->symbol = sym; elt = ucl_object_find_key (val, "flag"); if (elt != NULL && ucl_obj_toint_safe (elt, &map->fuzzy_flag)) { elt = ucl_object_find_key (val, "max_score"); if (elt != NULL) { map->weight = ucl_obj_todouble (elt); } else { map->weight = rule->max_score; } /* Add flag to hash table */ g_hash_table_insert (rule->mappings, GINT_TO_POINTER (map->fuzzy_flag), map); rspamd_symbols_cache_add_symbol (cfg->cache, map->symbol, 0, NULL, NULL, SYMBOL_TYPE_VIRTUAL|SYMBOL_TYPE_FINE, cb_id); } else { msg_err_config ("fuzzy_map parameter has no flag definition"); } } else { msg_err_config ("fuzzy_map parameter has no symbol definition"); } } else { msg_err_config ("fuzzy_map parameter is of an unsupported type"); } }
static void rspamd_ucl_fin_cb (rspamd_mempool_t * pool, struct map_cb_data *data) { struct rspamd_ucl_map_cbdata *cbdata = data->cur_data, *prev = data->prev_data; ucl_object_t *obj; struct ucl_parser *parser; guint32 checksum; ucl_object_iter_t it = NULL; const ucl_object_t *cur; struct rspamd_config *cfg = data->map->cfg; if (prev != NULL) { if (prev->buf != NULL) { g_string_free (prev->buf, TRUE); } g_free (prev); } if (cbdata == NULL) { msg_err_config ("map fin error: new data is NULL"); return; } checksum = XXH64 (cbdata->buf->str, cbdata->buf->len, 0); if (data->map->checksum != checksum) { /* New data available */ parser = ucl_parser_new (0); if (!ucl_parser_add_chunk (parser, cbdata->buf->str, cbdata->buf->len)) { msg_err_config ("cannot parse map %s: %s", data->map->uri, ucl_parser_get_error (parser)); ucl_parser_free (parser); } else { obj = ucl_parser_get_object (parser); ucl_parser_free (parser); it = NULL; while ((cur = ucl_iterate_object (obj, &it, true))) { ucl_object_replace_key (cbdata->cfg->rcl_obj, (ucl_object_t *)cur, cur->key, cur->keylen, false); } ucl_object_unref (obj); data->map->checksum = checksum; } } else { msg_info_config ("do not reload map %s, checksum is the same: %d", data->map->uri, checksum); } }
gpointer rspamd_redis_init (struct rspamd_stat_ctx *ctx, struct rspamd_config *cfg, struct rspamd_statfile *st) { struct redis_stat_ctx *backend; struct rspamd_statfile_config *stf = st->stcf; struct rspamd_redis_stat_elt *st_elt; const ucl_object_t *obj; gboolean ret = FALSE; backend = g_slice_alloc0 (sizeof (*backend)); /* First search in backend configuration */ obj = ucl_object_lookup (st->classifier->cfg->opts, "backend"); if (obj != NULL && ucl_object_type (obj) == UCL_OBJECT) { ret = rspamd_redis_try_ucl (backend, obj, cfg, stf->symbol); } /* Now try statfiles config */ if (!ret) { ret = rspamd_redis_try_ucl (backend, stf->opts, cfg, stf->symbol); } /* Now try classifier config */ if (!ret) { ret = rspamd_redis_try_ucl (backend, st->classifier->cfg->opts, cfg, stf->symbol); } if (!ret) { msg_err_config ("cannot init redis backend for %s", stf->symbol); g_slice_free1 (sizeof (*backend), backend); return NULL; } stf->clcf->flags |= RSPAMD_FLAG_CLASSIFIER_INCREMENTING_BACKEND; backend->stcf = stf; st_elt = g_slice_alloc0 (sizeof (*st_elt)); st_elt->ev_base = ctx->ev_base; st_elt->ctx = backend; backend->stat_elt = rspamd_stat_ctx_register_async ( rspamd_redis_async_stat_cb, rspamd_redis_async_stat_fin, st_elt, REDIS_STAT_TIMEOUT); st_elt->async = backend->stat_elt; return (gpointer)backend; }
gboolean rspamd_config_calculate_checksum (struct rspamd_config *cfg) { gint fd; void *map; struct stat st; /* Compute checksum for config file that should be used by xml dumper */ if ((fd = open (cfg->cfg_name, O_RDONLY)) == -1) { msg_err_config ( "config file %s is no longer available, cannot calculate checksum"); return FALSE; } if (stat (cfg->cfg_name, &st) == -1) { msg_err_config ("cannot stat %s: %s", cfg->cfg_name, strerror (errno)); close (fd); return FALSE; } /* Now mmap this file to simplify reading process */ if ((map = mmap (NULL, st.st_size, PROT_READ, MAP_SHARED, fd, 0)) == MAP_FAILED) { msg_err_config ("cannot mmap %s: %s", cfg->cfg_name, strerror (errno)); close (fd); return FALSE; } close (fd); /* Get checksum for a file */ cfg->checksum = g_compute_checksum_for_string (G_CHECKSUM_MD5, map, st.st_size); munmap (map, st.st_size); return TRUE; }
gboolean rspamd_parse_bind_line (struct rspamd_config *cfg, struct rspamd_worker_conf *cf, const gchar *str) { struct rspamd_worker_bind_conf *cnf; gchar **tokens, *err; gboolean ret = TRUE; if (str == NULL) { return FALSE; } if (str[0] == '[') { /* This is an ipv6 address */ gsize len, ntok; const gchar *start, *ip_pos; start = str + 1; len = strcspn (start, "]"); if (start[len] != ']') { return FALSE; } ip_pos = start; start += len + 1; ntok = 1; if (*start == ':') { ntok = 2; start ++; } tokens = g_malloc_n (ntok + 1, sizeof (gchar *)); tokens[ntok] = NULL; tokens[0] = g_malloc (len + 1); rspamd_strlcpy (tokens[0], ip_pos, len + 1); if (ntok > 1) { tokens[1] = g_strdup (start); } } else { tokens = g_strsplit_set (str, ":", 0); } if (!tokens || !tokens[0]) { return FALSE; } cnf = rspamd_mempool_alloc0 (cfg->cfg_pool, sizeof (struct rspamd_worker_bind_conf)); cnf->cnt = 1024; if (strcmp (tokens[0], "systemd") == 0) { /* The actual socket will be passed by systemd environment */ cnf->is_systemd = TRUE; cnf->cnt = strtoul (tokens[1], &err, 10); cnf->addrs = NULL; if (err == NULL || *err == '\0') { cnf->name = rspamd_mempool_strdup (cfg->cfg_pool, str); LL_PREPEND (cf->bind_conf, cnf); } else { msg_err_config ("cannot parse bind line: %s", str); ret = FALSE; } } else { if (!rspamd_parse_host_port_priority_strv (tokens, &cnf->addrs, NULL, &cnf->name, DEFAULT_BIND_PORT, cfg->cfg_pool)) { msg_err_config ("cannot parse bind line: %s", str); ret = FALSE; } else { cnf->cnt = cnf->addrs->len; LL_PREPEND (cf->bind_conf, cnf); } } g_strfreev (tokens); return ret; }
/* * Perform post load actions */ void rspamd_config_post_load (struct rspamd_config *cfg) { #ifdef HAVE_CLOCK_GETTIME struct timespec ts; #endif struct metric *def_metric; #ifdef HAVE_CLOCK_GETTIME #ifdef HAVE_CLOCK_PROCESS_CPUTIME_ID clock_getres (CLOCK_PROCESS_CPUTIME_ID, &ts); # elif defined(HAVE_CLOCK_VIRTUAL) clock_getres (CLOCK_VIRTUAL, &ts); # else clock_getres (CLOCK_REALTIME, &ts); # endif cfg->clock_res = log10 (1000000. / ts.tv_nsec); if (cfg->clock_res < 0) { cfg->clock_res = 0; } if (cfg->clock_res > 3) { cfg->clock_res = 3; } #else /* For gettimeofday */ cfg->clock_res = 1; #endif rspamd_regexp_library_init (); if ((def_metric = g_hash_table_lookup (cfg->metrics, DEFAULT_METRIC)) == NULL) { def_metric = rspamd_config_new_metric (cfg, NULL, DEFAULT_METRIC); def_metric->actions[METRIC_ACTION_REJECT].score = DEFAULT_SCORE; } if (cfg->tld_file == NULL) { /* Try to guess tld file */ GString *fpath = g_string_new (NULL); rspamd_printf_gstring (fpath, "%s%c%s", RSPAMD_PLUGINSDIR, G_DIR_SEPARATOR, "effective_tld_names.dat"); if (access (fpath->str, R_OK)) { msg_warn_config ("url_tld option is not specified but %s is available," " therefore this file is assumed as TLD file for URL" " extraction", fpath->str); cfg->tld_file = rspamd_mempool_strdup (cfg->cfg_pool, fpath->str); } else { msg_err_config ("no url_tld option has been specified, URL's detection " "will be awfully broken"); } g_string_free (fpath, TRUE); } /* Lua options */ (void)rspamd_lua_post_load_config (cfg); init_dynamic_config (cfg); rspamd_url_init (cfg->tld_file); /* Insert classifiers symbols */ (void)rspamd_config_insert_classify_symbols (cfg); }
void rspamd_stat_init (struct rspamd_config *cfg, struct event_base *ev_base) { GList *cur, *curst; struct rspamd_classifier_config *clf; struct rspamd_statfile_config *stf; struct rspamd_stat_backend *bk; struct rspamd_statfile *st; struct rspamd_classifier *cl; const ucl_object_t *cache_obj = NULL, *cache_name_obj; const gchar *cache_name = NULL; if (stat_ctx == NULL) { stat_ctx = g_slice_alloc0 (sizeof (*stat_ctx)); } stat_ctx->backends_subrs = stat_backends; stat_ctx->backends_count = G_N_ELEMENTS (stat_backends); stat_ctx->classifiers_subrs = stat_classifiers; stat_ctx->classifiers_count = G_N_ELEMENTS (stat_classifiers); stat_ctx->tokenizers_subrs = stat_tokenizers; stat_ctx->tokenizers_count = G_N_ELEMENTS (stat_tokenizers); stat_ctx->caches_subrs = stat_caches; stat_ctx->caches_count = G_N_ELEMENTS (stat_caches); stat_ctx->cfg = cfg; stat_ctx->statfiles = g_ptr_array_new (); stat_ctx->classifiers = g_ptr_array_new (); stat_ctx->async_elts = g_queue_new (); stat_ctx->ev_base = ev_base; REF_RETAIN (stat_ctx->cfg); /* Create statfiles from the classifiers */ cur = cfg->classifiers; while (cur) { clf = cur->data; bk = rspamd_stat_get_backend (clf->backend); if (bk == NULL) { msg_err_config ("cannot get backend of type %s, so disable classifier" " %s completely", clf->backend, clf->name); cur = g_list_next (cur); continue; } /* XXX: * Here we get the first classifier tokenizer config as the only one * We NO LONGER support multiple tokenizers per rspamd instance */ if (stat_ctx->tkcf == NULL) { stat_ctx->tokenizer = rspamd_stat_get_tokenizer (clf->tokenizer->name); g_assert (stat_ctx->tokenizer != NULL); stat_ctx->tkcf = stat_ctx->tokenizer->get_config (cfg->cfg_pool, clf->tokenizer, NULL); } cl = g_slice_alloc0 (sizeof (*cl)); cl->cfg = clf; cl->ctx = stat_ctx; cl->statfiles_ids = g_array_new (FALSE, FALSE, sizeof (gint)); cl->subrs = rspamd_stat_get_classifier (clf->classifier); g_assert (cl->subrs != NULL); cl->subrs->init_func (cfg->cfg_pool, cl); /* Init classifier cache */ cache_name = NULL; if (clf->opts) { cache_obj = ucl_object_find_key (clf->opts, "cache"); cache_name_obj = NULL; if (cache_obj) { cache_name_obj = ucl_object_find_any_key (cache_obj, "name", "type", NULL); } if (cache_name_obj) { cache_name = ucl_object_tostring (cache_name_obj); } } if (cache_name == NULL) { /* We assume that learn cache is the same as backend */ cache_name = clf->backend; } curst = clf->statfiles; while (curst) { stf = curst->data; st = g_slice_alloc0 (sizeof (*st)); st->classifier = cl; st->stcf = stf; st->backend = bk; st->bkcf = bk->init (stat_ctx, cfg, st); msg_debug_config ("added backend %s for symbol %s", bk->name, stf->symbol); /* XXX: bad hack to pass statfiles configuration to cache */ if (cl->cache == NULL) { cl->cache = rspamd_stat_get_cache (cache_name); g_assert (cl->cache != NULL); cl->cachecf = cl->cache->init (stat_ctx, cfg, st, cache_obj); if (cl->cachecf == NULL) { msg_err_config ("error adding cache %s for symbol %s", cl->cache->name, stf->symbol); cl->cache = NULL; } else { msg_debug_config ("added cache %s for symbol %s", cl->cache->name, stf->symbol); } } if (st->bkcf == NULL) { msg_err_config ("cannot init backend %s for statfile %s", clf->backend, stf->symbol); g_slice_free1 (sizeof (*st), st); } else { st->id = stat_ctx->statfiles->len; g_ptr_array_add (stat_ctx->statfiles, st); g_array_append_val (cl->statfiles_ids, st->id); } curst = curst->next; } g_ptr_array_add (stat_ctx->classifiers, cl); cur = cur->next; } }
gint regexp_module_config (struct rspamd_config *cfg) { struct regexp_module_item *cur_item; const ucl_object_t *sec, *value, *elt; ucl_object_iter_t it = NULL; gint res = TRUE, id; if (!rspamd_config_is_module_enabled (cfg, "regexp")) { return TRUE; } sec = ucl_object_find_key (cfg->rcl_obj, "regexp"); if (sec == NULL) { msg_err_config ("regexp module enabled, but no rules are defined"); return TRUE; } regexp_module_ctx->max_size = 0; while ((value = ucl_iterate_object (sec, &it, true)) != NULL) { if (g_ascii_strncasecmp (ucl_object_key (value), "max_size", sizeof ("max_size") - 1) == 0) { regexp_module_ctx->max_size = ucl_obj_toint (value); rspamd_mime_expression_set_re_limit (regexp_module_ctx->max_size); } else if (g_ascii_strncasecmp (ucl_object_key (value), "max_threads", sizeof ("max_threads") - 1) == 0) { msg_warn_config ("regexp module is now single threaded, max_threads is ignored"); } else if (value->type == UCL_STRING) { cur_item = rspamd_mempool_alloc0 (regexp_module_ctx->regexp_pool, sizeof (struct regexp_module_item)); cur_item->symbol = ucl_object_key (value); if (!read_regexp_expression (regexp_module_ctx->regexp_pool, cur_item, ucl_object_key (value), ucl_obj_tostring (value), cfg)) { res = FALSE; } else { rspamd_symbols_cache_add_symbol (cfg->cache, cur_item->symbol, 0, process_regexp_item, cur_item, SYMBOL_TYPE_NORMAL, -1); } } else if (value->type == UCL_USERDATA) { /* Just a lua function */ cur_item = rspamd_mempool_alloc0 (regexp_module_ctx->regexp_pool, sizeof (struct regexp_module_item)); cur_item->symbol = ucl_object_key (value); cur_item->lua_function = ucl_object_toclosure (value); rspamd_symbols_cache_add_symbol (cfg->cache, cur_item->symbol, 0, process_regexp_item, cur_item, SYMBOL_TYPE_NORMAL, -1); } else if (value->type == UCL_OBJECT) { const gchar *description = NULL, *group = NULL, *metric = DEFAULT_METRIC; gdouble score = 0.0; gboolean one_shot = FALSE, is_lua = FALSE, valid_expression = TRUE; /* We have some lua table, extract its arguments */ elt = ucl_object_find_key (value, "callback"); if (elt == NULL || elt->type != UCL_USERDATA) { /* Try plain regexp expression */ elt = ucl_object_find_any_key (value, "regexp", "re", NULL); if (elt != NULL && ucl_object_type (elt) == UCL_STRING) { cur_item = rspamd_mempool_alloc0 (regexp_module_ctx->regexp_pool, sizeof (struct regexp_module_item)); cur_item->symbol = ucl_object_key (value); if (!read_regexp_expression (regexp_module_ctx->regexp_pool, cur_item, ucl_object_key (value), ucl_obj_tostring (elt), cfg)) { res = FALSE; } else { valid_expression = TRUE; } } else { msg_err_config ( "no callback/expression defined for regexp symbol: " "%s", ucl_object_key (value)); } } else { is_lua = TRUE; cur_item = rspamd_mempool_alloc0 ( regexp_module_ctx->regexp_pool, sizeof (struct regexp_module_item)); cur_item->symbol = ucl_object_key (value); cur_item->lua_function = ucl_object_toclosure (value); } if (cur_item && (is_lua || valid_expression)) { id = rspamd_symbols_cache_add_symbol (cfg->cache, cur_item->symbol, 0, process_regexp_item, cur_item, SYMBOL_TYPE_NORMAL, -1); elt = ucl_object_find_key (value, "condition"); if (elt != NULL && ucl_object_type (elt) == UCL_USERDATA) { struct ucl_lua_funcdata *conddata; conddata = ucl_object_toclosure (elt); rspamd_symbols_cache_add_condition (cfg->cache, id, conddata->L, conddata->idx); } elt = ucl_object_find_key (value, "metric"); if (elt) { metric = ucl_object_tostring (elt); } elt = ucl_object_find_key (value, "description"); if (elt) { description = ucl_object_tostring (elt); } elt = ucl_object_find_key (value, "group"); if (elt) { group = ucl_object_tostring (elt); } elt = ucl_object_find_key (value, "score"); if (elt) { score = ucl_object_todouble (elt); } elt = ucl_object_find_key (value, "one_shot"); if (elt) { one_shot = ucl_object_toboolean (elt); } rspamd_config_add_metric_symbol (cfg, metric, cur_item->symbol, score, description, group, one_shot, FALSE); } } else { msg_warn_config ("unknown type of attribute %s for regexp module", ucl_object_key (value)); } } return res; }
struct rspamd_map * rspamd_map_add (struct rspamd_config *cfg, const gchar *map_line, const gchar *description, map_cb_t read_callback, map_fin_cb_t fin_callback, void **user_data) { struct rspamd_map *new_map; const gchar *def; struct file_map_data *fdata; struct http_map_data *hdata; gchar *cksum_encoded, cksum[rspamd_cryptobox_HASHBYTES]; rspamd_mempool_t *pool; struct http_parser_url up; rspamd_ftok_t tok; if (cfg->map_pool == NULL) { cfg->map_pool = rspamd_mempool_new (rspamd_mempool_suggest_size (), "map"); memcpy (cfg->map_pool->tag.uid, cfg->cfg_pool->tag.uid, sizeof (cfg->map_pool->tag.uid)); } new_map = rspamd_mempool_alloc0 (cfg->map_pool, sizeof (struct rspamd_map)); /* First of all detect protocol line */ if (rspamd_map_check_proto (cfg, map_line, new_map) == NULL) { return NULL; } new_map->read_callback = read_callback; new_map->fin_callback = fin_callback; new_map->user_data = user_data; new_map->cfg = cfg; new_map->id = g_random_int (); new_map->locked = rspamd_mempool_alloc0_shared (cfg->cfg_pool, sizeof (gint)); def = new_map->uri; if (description != NULL) { new_map->description = rspamd_mempool_strdup (cfg->cfg_pool, description); } /* Now check for each proto separately */ if (new_map->protocol == MAP_PROTO_FILE) { fdata = rspamd_mempool_alloc0 (cfg->map_pool, sizeof (struct file_map_data)); if (access (def, R_OK) == -1) { if (errno != ENOENT) { msg_err_config ("cannot open file '%s': %s", def, strerror (errno)); return NULL; } msg_info_config ( "map '%s' is not found, but it can be loaded automatically later", def); /* We still can add this file */ fdata->st.st_mtime = -1; } else { stat (def, &fdata->st); } fdata->filename = rspamd_mempool_strdup (cfg->map_pool, def); new_map->map_data = fdata; } else if (new_map->protocol == MAP_PROTO_HTTP) { hdata = rspamd_mempool_alloc0 (cfg->map_pool, sizeof (struct http_map_data)); memset (&up, 0, sizeof (up)); if (http_parser_parse_url (new_map->uri, strlen (new_map->uri), FALSE, &up) != 0) { msg_err_config ("cannot parse HTTP url: %s", new_map->uri); return NULL; } else { if (!(up.field_set & 1 << UF_HOST)) { msg_err_config ("cannot parse HTTP url: %s: no host", new_map->uri); return NULL; } tok.begin = new_map->uri + up.field_data[UF_HOST].off; tok.len = up.field_data[UF_HOST].len; hdata->host = rspamd_mempool_ftokdup (cfg->map_pool, &tok); if (up.field_set & 1 << UF_PORT) { hdata->port = up.port; } else { hdata->port = 80; } if (up.field_set & 1 << UF_PATH) { tok.begin = new_map->uri + up.field_data[UF_PATH].off; tok.len = strlen (tok.begin); hdata->path = rspamd_mempool_ftokdup (cfg->map_pool, &tok); } } new_map->map_data = hdata; } /* Temp pool */ rspamd_cryptobox_hash (cksum, new_map->uri, strlen (new_map->uri), NULL, 0); cksum_encoded = rspamd_encode_base32 (cksum, sizeof (cksum)); new_map->pool = rspamd_mempool_new (rspamd_mempool_suggest_size (), "map"); rspamd_strlcpy (new_map->pool->tag.uid, cksum_encoded, sizeof (new_map->pool->tag.uid)); g_free (cksum_encoded); pool = new_map->pool; msg_info_pool ("added map %s", new_map->uri); cfg->maps = g_list_prepend (cfg->maps, new_map); return new_map; }
gboolean rspamd_map_add (struct rspamd_config *cfg, const gchar *map_line, const gchar *description, map_cb_t read_callback, map_fin_cb_t fin_callback, void **user_data) { struct rspamd_map *new_map; enum fetch_proto proto; const gchar *def, *p, *hostend; struct file_map_data *fdata; struct http_map_data *hdata; gchar portbuf[6], *cksum_encoded, cksum[BLAKE2B_OUTBYTES]; gint i, s, r; struct addrinfo hints, *res; rspamd_mempool_t *pool; /* First of all detect protocol line */ if (!rspamd_map_check_proto (map_line, (int *)&proto, &def)) { return FALSE; } /* Constant pool */ if (cfg->map_pool == NULL) { cfg->map_pool = rspamd_mempool_new (rspamd_mempool_suggest_size (), "map"); memcpy (cfg->map_pool->tag.uid, cfg->cfg_pool->tag.uid, sizeof (cfg->map_pool->tag.uid)); } new_map = rspamd_mempool_alloc0 (cfg->map_pool, sizeof (struct rspamd_map)); new_map->read_callback = read_callback; new_map->fin_callback = fin_callback; new_map->user_data = user_data; new_map->protocol = proto; new_map->cfg = cfg; new_map->id = g_random_int (); new_map->locked = rspamd_mempool_alloc0_shared (cfg->cfg_pool, sizeof (gint)); if (proto == MAP_PROTO_FILE) { new_map->uri = rspamd_mempool_strdup (cfg->cfg_pool, def); def = new_map->uri; } else { new_map->uri = rspamd_mempool_strdup (cfg->cfg_pool, map_line); } if (description != NULL) { new_map->description = rspamd_mempool_strdup (cfg->cfg_pool, description); } /* Now check for each proto separately */ if (proto == MAP_PROTO_FILE) { fdata = rspamd_mempool_alloc0 (cfg->map_pool, sizeof (struct file_map_data)); if (access (def, R_OK) == -1) { if (errno != ENOENT) { msg_err_config ("cannot open file '%s': %s", def, strerror (errno)); return FALSE; } msg_info_config ( "map '%s' is not found, but it can be loaded automatically later", def); /* We still can add this file */ fdata->st.st_mtime = -1; } else { stat (def, &fdata->st); } fdata->filename = rspamd_mempool_strdup (cfg->map_pool, def); new_map->map_data = fdata; } else if (proto == MAP_PROTO_HTTP) { hdata = rspamd_mempool_alloc0 (cfg->map_pool, sizeof (struct http_map_data)); /* Try to search port */ if ((p = strchr (def, ':')) != NULL) { hostend = p; i = 0; p++; while (g_ascii_isdigit (*p) && i < (gint)sizeof (portbuf) - 1) { portbuf[i++] = *p++; } if (*p != '/') { msg_info_config ("bad http map definition: %s", def); return FALSE; } portbuf[i] = '\0'; hdata->port = atoi (portbuf); } else { /* Default http port */ rspamd_snprintf (portbuf, sizeof (portbuf), "80"); hdata->port = 80; /* Now separate host from path */ if ((p = strchr (def, '/')) == NULL) { msg_info_config ("bad http map definition: %s", def); return FALSE; } hostend = p; } hdata->host = rspamd_mempool_alloc (cfg->map_pool, hostend - def + 1); rspamd_strlcpy (hdata->host, def, hostend - def + 1); hdata->path = rspamd_mempool_strdup (cfg->map_pool, p); /* Now try to resolve */ memset (&hints, 0, sizeof (hints)); hints.ai_family = AF_UNSPEC; /* Allow IPv4 or IPv6 */ hints.ai_socktype = SOCK_STREAM; /* Stream socket */ hints.ai_flags = 0; hints.ai_protocol = 0; /* Any protocol */ hints.ai_canonname = NULL; hints.ai_addr = NULL; hints.ai_next = NULL; if ((r = getaddrinfo (hdata->host, portbuf, &hints, &res)) == 0) { hdata->addr = res; rspamd_mempool_add_destructor (cfg->cfg_pool, (rspamd_mempool_destruct_t)freeaddrinfo, hdata->addr); } else { msg_err_config ("address resolution for %s failed: %s", hdata->host, gai_strerror (r)); return FALSE; } /* Now try to connect */ if ((s = rspamd_socket_tcp (hdata->addr, FALSE, FALSE)) == -1) { msg_info_config ("cannot connect to http server %s: %d, %s", hdata->host, errno, strerror (errno)); return FALSE; } close (s); hdata->conn = rspamd_http_connection_new (http_map_read, http_map_error, http_map_finish, RSPAMD_HTTP_BODY_PARTIAL | RSPAMD_HTTP_CLIENT_SIMPLE, RSPAMD_HTTP_CLIENT, NULL); new_map->map_data = hdata; } /* Temp pool */ blake2b (cksum, new_map->uri, NULL, sizeof (cksum), strlen (new_map->uri), 0); cksum_encoded = rspamd_encode_base32 (cksum, sizeof (cksum)); new_map->pool = rspamd_mempool_new (rspamd_mempool_suggest_size (), "map"); memcpy (new_map->pool->tag.uid, cksum_encoded, sizeof (new_map->pool->tag.uid)); g_free (cksum_encoded); pool = new_map->pool; msg_info_pool ("added map %s", new_map->uri); cfg->maps = g_list_prepend (cfg->maps, new_map); return TRUE; }
static struct rspamd_map_backend * rspamd_map_parse_backend (struct rspamd_config *cfg, const gchar *map_line) { struct rspamd_map_backend *bk; struct file_map_data *fdata = NULL; struct http_map_data *hdata = NULL; struct http_parser_url up; rspamd_ftok_t tok; bk = g_slice_alloc0 (sizeof (*bk)); REF_INIT_RETAIN (bk, rspamd_map_backend_dtor); if (!rspamd_map_check_proto (cfg, map_line, bk)) { goto err; } /* Now check for each proto separately */ if (bk->protocol == MAP_PROTO_FILE) { fdata = g_slice_alloc0 (sizeof (struct file_map_data)); fdata->st.st_mtime = -1; if (access (bk->uri, R_OK) == -1) { if (errno != ENOENT) { msg_err_config ("cannot open file '%s': %s", bk->uri, strerror (errno)); return NULL; } msg_info_config ( "map '%s' is not found, but it can be loaded automatically later", bk->uri); } fdata->filename = g_strdup (bk->uri); bk->data.fd = fdata; } else if (bk->protocol == MAP_PROTO_HTTP || bk->protocol == MAP_PROTO_HTTPS) { hdata = g_slice_alloc0 (sizeof (struct http_map_data)); memset (&up, 0, sizeof (up)); if (http_parser_parse_url (bk->uri, strlen (bk->uri), FALSE, &up) != 0) { msg_err_config ("cannot parse HTTP url: %s", bk->uri); goto err; } else { if (!(up.field_set & 1 << UF_HOST)) { msg_err_config ("cannot parse HTTP url: %s: no host", bk->uri); return NULL; } tok.begin = bk->uri + up.field_data[UF_HOST].off; tok.len = up.field_data[UF_HOST].len; hdata->host = rspamd_ftokdup (&tok); if (up.field_set & 1 << UF_PORT) { hdata->port = up.port; } else { if (bk->protocol == MAP_PROTO_HTTP) { hdata->port = 80; } else { hdata->port = 443; } } if (up.field_set & 1 << UF_PATH) { tok.begin = bk->uri + up.field_data[UF_PATH].off; tok.len = strlen (tok.begin); hdata->path = rspamd_ftokdup (&tok); } } bk->data.hd = hdata; } return bk; err: MAP_RELEASE (bk, "rspamd_map_backend"); if (hdata) { g_slice_free1 (sizeof (*hdata), hdata); } if (fdata) { g_slice_free1 (sizeof (*fdata), fdata); } return NULL; }
gboolean rspamd_lua_check_condition (struct rspamd_config *cfg, const gchar *condition) { lua_State *L = cfg->lua_state; gchar *hostbuf, *condbuf; gsize hostlen; gboolean res; #ifdef HAVE_SYS_UTSNAME_H struct utsname uts; #endif /* Set some globals for condition */ /* XXX: think what other variables can be useful */ hostlen = sysconf (_SC_HOST_NAME_MAX) + 1; hostbuf = alloca (hostlen); gethostname (hostbuf, hostlen); hostbuf[hostlen - 1] = '\0'; /* Hostname */ lua_pushstring (L, hostbuf); lua_setglobal (L, "hostname"); /* Config file name */ lua_pushstring (L, cfg->cfg_name); lua_setglobal (L, "cfg_name"); /* Check for uname */ #ifdef HAVE_SYS_UTSNAME_H uname (&uts); lua_pushstring (L, uts.sysname); lua_setglobal (L, "osname"); lua_pushstring (L, uts.release); lua_setglobal (L, "osrelease"); #else lua_pushstring (L, "unknown"); lua_setglobal (L, "osname"); lua_pushstring (L, ""); lua_setglobal (L, "osrelease"); #endif #ifdef HAVE_OPENSSL lua_pushboolean (L, TRUE); #else lua_pushboolean (L, FALSE); #endif lua_setglobal (L, "rspamd_supports_rsa"); /* Rspamd paths */ lua_newtable (L); rspamd_lua_table_set (L, "confdir", RSPAMD_CONFDIR); rspamd_lua_table_set (L, "rundir", RSPAMD_RUNDIR); rspamd_lua_table_set (L, "dbdir", RSPAMD_DBDIR); rspamd_lua_table_set (L, "logdir", RSPAMD_LOGDIR); rspamd_lua_table_set (L, "pluginsdir", RSPAMD_PLUGINSDIR); rspamd_lua_table_set (L, "prefix", RSPAMD_PREFIX); lua_setglobal (L, "rspamd_paths"); /* Make fake string */ hostlen = sizeof (FAKE_RES_VAR "=") + strlen (condition); condbuf = g_malloc (hostlen); rspamd_strlcpy (condbuf, FAKE_RES_VAR "=", sizeof (FAKE_RES_VAR "=")); g_strlcat (condbuf, condition, hostlen); /* Evaluate condition */ if (luaL_dostring (L, condbuf) != 0) { msg_err_config("eval of '%s' failed: '%s'", condition, lua_tostring (L, -1)); g_free (condbuf); return FALSE; } /* Get global variable res to get result */ lua_getglobal (L, FAKE_RES_VAR); if (!lua_isboolean (L, -1)) { msg_err_config("bad string evaluated: %s, type: %s", condbuf, lua_typename (L, lua_type (L, -1))); g_free (condbuf); return FALSE; } res = lua_toboolean (L, -1); g_free (condbuf); return res; }
/* Do post load initialization based on lua */ void rspamd_lua_post_load_config (struct rspamd_config *cfg) { lua_State *L = cfg->lua_state; const gchar *name, *val; gchar *sym; struct rspamd_expression *expr, *old_expr; ucl_object_t *obj; gsize keylen; GError *err = NULL; /* First check all module options that may be overriden in 'config' global */ lua_getglobal (L, "config"); if (lua_istable (L, -1)) { /* Iterate */ for (lua_pushnil (L); lua_next (L, -2); lua_pop (L, 1)) { /* 'key' is at index -2 and 'value' is at index -1 */ /* Key must be a string and value must be a table */ name = luaL_checklstring (L, -2, &keylen); if (name != NULL && lua_istable (L, -1)) { obj = ucl_object_lua_import (L, lua_gettop (L)); if (obj != NULL) { ucl_object_insert_key_merged (cfg->rcl_obj, obj, name, keylen, true); } } } } /* Check metrics settings */ lua_getglobal (L, "metrics"); if (lua_istable (L, -1)) { /* Iterate */ for (lua_pushnil (L); lua_next (L, -2); lua_pop (L, 1)) { /* 'key' is at index -2 and 'value' is at index -1 */ /* Key must be a string and value must be a table */ name = luaL_checkstring (L, -2); if (name != NULL && lua_istable (L, -1)) { lua_process_metric (L, name, cfg); } } } /* Check composites */ lua_getglobal (L, "composites"); if (lua_istable (L, -1)) { /* Iterate */ for (lua_pushnil (L); lua_next (L, -2); lua_pop (L, 1)) { /* 'key' is at index -2 and 'value' is at index -1 */ /* Key must be a string and value must be a table */ name = luaL_checkstring (L, -2); if (name != NULL && lua_isstring (L, -1)) { val = lua_tostring (L, -1); sym = rspamd_mempool_strdup (cfg->cfg_pool, name); if (!rspamd_parse_expression (val, 0, &composite_expr_subr, NULL, cfg->cfg_pool, &err, &expr)) { msg_err_config("cannot parse composite expression '%s': %s", val, err->message); g_error_free (err); err = NULL; continue; } /* Now check hash table for this composite */ if ((old_expr = g_hash_table_lookup (cfg->composite_symbols, name)) != NULL) { msg_info_config("replacing composite symbol %s", name); g_hash_table_replace (cfg->composite_symbols, sym, expr); } else { g_hash_table_insert (cfg->composite_symbols, sym, expr); rspamd_symbols_cache_add_symbol (cfg->cache, sym, 0, NULL, NULL, SYMBOL_TYPE_COMPOSITE, -1); } } } } }
gpointer rspamd_stat_cache_redis_init (struct rspamd_stat_ctx *ctx, struct rspamd_config *cfg, struct rspamd_statfile *st, const ucl_object_t *cf) { struct rspamd_redis_cache_ctx *cache_ctx; struct rspamd_statfile_config *stf = st->stcf; const ucl_object_t *obj; gboolean ret = FALSE; cache_ctx = g_slice_alloc0 (sizeof (*cache_ctx)); /* First search in backend configuration */ obj = ucl_object_lookup (st->classifier->cfg->opts, "backend"); if (obj != NULL && ucl_object_type (obj) == UCL_OBJECT) { ret = rspamd_redis_cache_try_ucl (cache_ctx, obj, cfg, stf->symbol); } /* Now try statfiles config */ if (!ret) { ret = rspamd_redis_cache_try_ucl (cache_ctx, stf->opts, cfg, stf->symbol); } /* Now try classifier config */ if (!ret) { ret = rspamd_redis_cache_try_ucl (cache_ctx, st->classifier->cfg->opts, cfg, stf->symbol); } /* Now try global redis settings */ if (!ret) { obj = ucl_object_lookup (cfg->rcl_obj, "redis"); if (obj) { const ucl_object_t *specific_obj; specific_obj = ucl_object_lookup (obj, "statistics"); if (specific_obj) { ret = rspamd_redis_cache_try_ucl (cache_ctx, specific_obj, cfg, stf->symbol); } else { ret = rspamd_redis_cache_try_ucl (cache_ctx, obj, cfg, stf->symbol); } } } if (!ret) { msg_err_config ("cannot init redis cache for %s", stf->symbol); g_slice_free1 (sizeof (*cache_ctx), cache_ctx); return NULL; } cache_ctx->stcf = stf; return (gpointer)cache_ctx; }
static gboolean rspamd_redis_try_ucl (struct redis_stat_ctx *backend, const ucl_object_t *obj, struct rspamd_config *cfg, const gchar *symbol) { const ucl_object_t *elt, *relt, *users_enabled; const gchar *lua_script; elt = ucl_object_lookup_any (obj, "read_servers", "servers", NULL); if (elt == NULL) { return FALSE; } backend->read_servers = rspamd_upstreams_create (cfg->ups_ctx); if (!rspamd_upstreams_from_ucl (backend->read_servers, elt, REDIS_DEFAULT_PORT, NULL)) { msg_err ("statfile %s cannot get read servers configuration", symbol); return FALSE; } relt = elt; elt = ucl_object_lookup (obj, "write_servers"); if (elt == NULL) { /* Use read servers as write ones */ g_assert (relt != NULL); backend->write_servers = rspamd_upstreams_create (cfg->ups_ctx); if (!rspamd_upstreams_from_ucl (backend->write_servers, relt, REDIS_DEFAULT_PORT, NULL)) { msg_err ("statfile %s cannot get write servers configuration", symbol); return FALSE; } } else { backend->write_servers = rspamd_upstreams_create (cfg->ups_ctx); if (!rspamd_upstreams_from_ucl (backend->write_servers, elt, REDIS_DEFAULT_PORT, NULL)) { msg_err ("statfile %s cannot get write servers configuration", symbol); rspamd_upstreams_destroy (backend->write_servers); backend->write_servers = NULL; } } elt = ucl_object_lookup (obj, "prefix"); if (elt == NULL || ucl_object_type (elt) != UCL_STRING) { /* Default non-users statistics */ backend->redis_object = REDIS_DEFAULT_OBJECT; /* * Make redis backend compatible with sqlite3 backend in users settings */ users_enabled = ucl_object_lookup_any (obj, "per_user", "users_enabled", NULL); if (users_enabled != NULL) { if (ucl_object_type (users_enabled) == UCL_BOOLEAN) { backend->enable_users = ucl_object_toboolean (users_enabled); backend->cbref_user = -1; if (backend->enable_users) { backend->redis_object = REDIS_DEFAULT_USERS_OBJECT; } } else if (ucl_object_type (users_enabled) == UCL_STRING) { lua_script = ucl_object_tostring (users_enabled); if (luaL_dostring (cfg->lua_state, lua_script) != 0) { msg_err_config ("cannot execute lua script for users " "extraction: %s", lua_tostring (cfg->lua_state, -1)); } else { if (lua_type (cfg->lua_state, -1) == LUA_TFUNCTION) { backend->enable_users = TRUE; backend->cbref_user = luaL_ref (cfg->lua_state, LUA_REGISTRYINDEX); } else { msg_err_config ("lua script must return " "function(task) and not %s", lua_typename (cfg->lua_state, lua_type ( cfg->lua_state, -1))); } } } } else { backend->enable_users = FALSE; } } else { /* XXX: sanity check */ backend->redis_object = ucl_object_tostring (elt); } elt = ucl_object_lookup (obj, "timeout"); if (elt) { backend->timeout = ucl_object_todouble (elt); } else { backend->timeout = REDIS_DEFAULT_TIMEOUT; } elt = ucl_object_lookup (obj, "password"); if (elt) { backend->password = ucl_object_tostring (elt); } else { backend->password = NULL; } elt = ucl_object_lookup_any (obj, "db", "database", "dbname", NULL); if (elt) { backend->dbname = ucl_object_tostring (elt); } else { backend->dbname = NULL; } return TRUE; }
static gint fuzzy_parse_rule (struct rspamd_config *cfg, const ucl_object_t *obj, gint cb_id) { const ucl_object_t *value, *cur; struct fuzzy_rule *rule; ucl_object_iter_t it = NULL; const char *k = NULL; if (obj->type != UCL_OBJECT) { msg_err_config ("invalid rule definition"); return -1; } rule = fuzzy_rule_new (fuzzy_module_ctx->default_symbol, fuzzy_module_ctx->fuzzy_pool); if ((value = ucl_object_find_key (obj, "mime_types")) != NULL) { it = NULL; while ((cur = ucl_iterate_object (value, &it, value->type == UCL_ARRAY)) != NULL) { rule->mime_types = g_list_concat (rule->mime_types, parse_mime_types (ucl_obj_tostring (cur))); } } if (rule->mime_types != NULL) { rspamd_mempool_add_destructor (fuzzy_module_ctx->fuzzy_pool, (rspamd_mempool_destruct_t)g_list_free, rule->mime_types); } if ((value = ucl_object_find_key (obj, "max_score")) != NULL) { rule->max_score = ucl_obj_todouble (value); } if ((value = ucl_object_find_key (obj, "symbol")) != NULL) { rule->symbol = ucl_obj_tostring (value); } if ((value = ucl_object_find_key (obj, "read_only")) != NULL) { rule->read_only = ucl_obj_toboolean (value); } if ((value = ucl_object_find_key (obj, "skip_unknown")) != NULL) { rule->skip_unknown = ucl_obj_toboolean (value); } if ((value = ucl_object_find_key (obj, "servers")) != NULL) { rule->servers = rspamd_upstreams_create (); rspamd_mempool_add_destructor (fuzzy_module_ctx->fuzzy_pool, (rspamd_mempool_destruct_t)rspamd_upstreams_destroy, rule->servers); rspamd_upstreams_from_ucl (rule->servers, value, DEFAULT_PORT, NULL); } if ((value = ucl_object_find_key (obj, "fuzzy_map")) != NULL) { it = NULL; while ((cur = ucl_iterate_object (value, &it, true)) != NULL) { parse_flags (rule, cfg, cur, cb_id); } } if ((value = ucl_object_find_key (obj, "encryption_key")) != NULL) { /* Create key from user's input */ k = ucl_object_tostring (value); if (k == NULL || (rule->peer_key = rspamd_http_connection_make_peer_key (k)) == NULL) { msg_err_config ("bad encryption key value: %s", k); return -1; } rule->local_key = rspamd_http_connection_gen_key (); } if ((value = ucl_object_find_key (obj, "fuzzy_key")) != NULL) { /* Create key from user's input */ k = ucl_object_tostring (value); } /* Setup keys */ if (k == NULL) { /* Use some default key for all ops */ k = "rspamd"; } rule->hash_key = g_string_sized_new (BLAKE2B_KEYBYTES); blake2 (rule->hash_key->str, k, NULL, BLAKE2B_KEYBYTES, strlen (k), 0); rule->hash_key->len = BLAKE2B_KEYBYTES; if ((value = ucl_object_find_key (obj, "fuzzy_shingles_key")) != NULL) { k = ucl_object_tostring (value); } if (k == NULL) { k = "rspamd"; } rule->shingles_key = g_string_sized_new (16); blake2 (rule->shingles_key->str, k, NULL, 16, strlen (k), 0); rule->shingles_key->len = 16; if (rspamd_upstreams_count (rule->servers) == 0) { msg_err_config ("no servers defined for fuzzy rule with symbol: %s", rule->symbol); return -1; } else { fuzzy_module_ctx->fuzzy_rules = g_list_prepend ( fuzzy_module_ctx->fuzzy_rules, rule); if (rule->symbol != fuzzy_module_ctx->default_symbol) { rspamd_symbols_cache_add_symbol (cfg->cache, rule->symbol, 0, NULL, NULL, SYMBOL_TYPE_VIRTUAL|SYMBOL_TYPE_FINE, cb_id); } } rspamd_mempool_add_destructor (fuzzy_module_ctx->fuzzy_pool, fuzzy_free_rule, rule); return 0; }
gboolean rspamd_config_add_metric_symbol (struct rspamd_config *cfg, const gchar *metric_name, const gchar *symbol, gdouble score, const gchar *description, const gchar *group, gboolean one_shot, gboolean rewrite_existing) { struct rspamd_symbols_group *sym_group; struct rspamd_symbol_def *sym_def; GList *metric_list; struct metric *metric; gdouble *score_ptr; g_assert (cfg != NULL); g_assert (symbol != NULL); if (metric_name == NULL) { metric_name = DEFAULT_METRIC; } metric = g_hash_table_lookup (cfg->metrics, metric_name); if (metric == NULL) { msg_err_config ("metric %s has not been found", metric_name); return FALSE; } if (g_hash_table_lookup (cfg->metrics_symbols, symbol) != NULL && !rewrite_existing) { msg_debug_config ("symbol %s has been already registered, do not override"); return FALSE; } sym_def = rspamd_mempool_alloc0 (cfg->cfg_pool, sizeof (struct rspamd_symbol_def)); score_ptr = rspamd_mempool_alloc (cfg->cfg_pool, sizeof (gdouble)); *score_ptr = score; sym_def->score = score; sym_def->weight_ptr = score_ptr; sym_def->name = rspamd_mempool_strdup (cfg->cfg_pool, symbol); sym_def->one_shot = one_shot; if (description) { sym_def->description = rspamd_mempool_strdup (cfg->cfg_pool, description); } msg_debug_config ("registered symbol %s with weight %.2f in metric %s and group %s", sym_def->name, score, metric->name, group); g_hash_table_insert (metric->symbols, sym_def->name, sym_def); if ((metric_list = g_hash_table_lookup (cfg->metrics_symbols, sym_def->name)) == NULL) { metric_list = g_list_prepend (NULL, metric); rspamd_mempool_add_destructor (cfg->cfg_pool, (rspamd_mempool_destruct_t)g_list_free, metric_list); g_hash_table_insert (cfg->metrics_symbols, sym_def->name, metric_list); } else { /* Slow but keep start element of list in safe */ if (!g_list_find (metric_list, metric)) { metric_list = g_list_append (metric_list, metric); } } /* Search for symbol group */ if (group == NULL) { group = "ungrouped"; } sym_group = g_hash_table_lookup (metric->groups, group); if (sym_group == NULL) { /* Create new group */ sym_group = rspamd_config_new_group (cfg, metric, group); } sym_def->gr = sym_group; g_hash_table_insert (sym_group->symbols, sym_def->name, sym_def); return TRUE; }
struct rspamd_map* rspamd_map_add_from_ucl (struct rspamd_config *cfg, const ucl_object_t *obj, const gchar *description, map_cb_t read_callback, map_fin_cb_t fin_callback, void **user_data) { ucl_object_iter_t it = NULL; const ucl_object_t *cur, *elt; struct rspamd_map *map; struct rspamd_map_backend *bk; g_assert (obj != NULL); if (ucl_object_type (obj) == UCL_STRING) { /* Just a plain string */ return rspamd_map_add (cfg, ucl_object_tostring (obj), NULL, read_callback, fin_callback, user_data); } map = g_slice_alloc0 (sizeof (struct rspamd_map)); map->read_callback = read_callback; map->fin_callback = fin_callback; map->user_data = user_data; map->cfg = cfg; map->id = g_random_int (); map->locked = rspamd_mempool_alloc0_shared (cfg->cfg_pool, sizeof (gint)); map->cache = rspamd_mempool_alloc0_shared (cfg->cfg_pool, sizeof (*map->cache)); map->backends = g_ptr_array_new (); map->poll_timeout = cfg->map_timeout; if (description) { map->description = g_strdup (description); } if (ucl_object_type (obj) == UCL_ARRAY) { /* Add array of maps as multiple backends */ while ((cur = ucl_object_iterate (obj, &it, true)) != NULL) { if (ucl_object_type (cur) == UCL_STRING) { bk = rspamd_map_parse_backend (cfg, ucl_object_tostring (cur)); if (bk != NULL) { g_ptr_array_add (map->backends, bk); if (!map->name) { map->name = g_strdup (ucl_object_tostring (cur)); } } } else { msg_err_config ("bad map element type: %s", ucl_object_type_to_string (ucl_object_type (cur))); } } if (map->backends->len == 0) { msg_err_config ("map has no urls to be loaded"); goto err; } } else if (ucl_object_type (obj) == UCL_OBJECT) { elt = ucl_object_lookup (obj, "name"); if (elt && ucl_object_type (elt) == UCL_STRING) { map->name = g_strdup (ucl_object_tostring (elt)); } elt = ucl_object_lookup (obj, "description"); if (elt && ucl_object_type (elt) == UCL_STRING) { if (map->description) { g_free (map->description); } map->description = g_strdup (ucl_object_tostring (elt)); } elt = ucl_object_lookup_any (obj, "timeout", "poll", "poll_time", "watch_interval", NULL); if (elt) { map->poll_timeout = ucl_object_todouble (elt); } elt = ucl_object_lookup_any (obj, "upstreams", "url", "urls", NULL); if (elt == NULL) { msg_err_config ("map has no urls to be loaded"); goto err; } if (ucl_object_type (obj) == UCL_ARRAY) { /* Add array of maps as multiple backends */ while ((cur = ucl_object_iterate (elt, &it, true)) != NULL) { if (ucl_object_type (cur) == UCL_STRING) { bk = rspamd_map_parse_backend (cfg, ucl_object_tostring (cur)); if (bk != NULL) { g_ptr_array_add (map->backends, bk); if (!map->name) { map->name = g_strdup (ucl_object_tostring (cur)); } } } else { msg_err_config ("bad map element type: %s", ucl_object_type_to_string (ucl_object_type (cur))); goto err; } } if (map->backends->len == 0) { msg_err_config ("map has no urls to be loaded"); goto err; } } else if (ucl_object_type (elt) == UCL_STRING) { bk = rspamd_map_parse_backend (cfg, ucl_object_tostring (elt)); if (bk != NULL) { g_ptr_array_add (map->backends, bk); if (!map->name) { map->name = g_strdup (ucl_object_tostring (cur)); } } } if (map->backends->len == 0) { msg_err_config ("map has no urls to be loaded"); goto err; } } rspamd_map_calculate_hash (map); msg_info_map ("added map from ucl"); cfg->maps = g_list_prepend (cfg->maps, map); return map; err: g_ptr_array_free (map->backends, TRUE); g_free (map->name); g_free (map->description); g_slice_free1 (sizeof (*map), map); return NULL; }
static const gchar * rspamd_map_check_proto (struct rspamd_config *cfg, const gchar *map_line, struct rspamd_map *map) { const gchar *pos = map_line, *end, *end_key; g_assert (map != NULL); g_assert (pos != NULL); end = pos + strlen (pos); if (g_ascii_strncasecmp (pos, "sign+", sizeof ("sign+") - 1) == 0) { map->is_signed = TRUE; pos += sizeof ("sign+") - 1; } if (g_ascii_strncasecmp (pos, "key=", sizeof ("key=") - 1) == 0) { pos += sizeof ("key=") - 1; end_key = memchr (pos, '+', end - pos); if (end_key != NULL) { map->trusted_pubkey = rspamd_pubkey_from_base32 (pos, end_key - pos, RSPAMD_KEYPAIR_SIGN, RSPAMD_CRYPTOBOX_MODE_25519); if (map->trusted_pubkey == NULL) { msg_err_config ("cannot read pubkey from map: %s", map_line); return NULL; } pos = end_key + 1; } else if (end - pos > 64) { /* Try hex encoding */ map->trusted_pubkey = rspamd_pubkey_from_hex (pos, 64, RSPAMD_KEYPAIR_SIGN, RSPAMD_CRYPTOBOX_MODE_25519); if (map->trusted_pubkey == NULL) { msg_err_config ("cannot read pubkey from map: %s", map_line); return NULL; } pos += 64; } else { msg_err_config ("cannot read pubkey from map: %s", map_line); return NULL; } if (*pos == '+' || *pos == ':') { pos ++; } } map->protocol = MAP_PROTO_FILE; if (g_ascii_strncasecmp (pos, "http://", sizeof ("http://") - 1) == 0) { map->protocol = MAP_PROTO_HTTP; /* Include http:// */ map->uri = rspamd_mempool_strdup (cfg->cfg_pool, pos); pos += sizeof ("http://") - 1; } else if (g_ascii_strncasecmp (pos, "file://", sizeof ("file://") - 1) == 0) { pos += sizeof ("file://") - 1; /* Exclude file:// */ map->uri = rspamd_mempool_strdup (cfg->cfg_pool, pos); } else if (*pos == '/') { /* Trivial file case */ map->uri = rspamd_mempool_strdup (cfg->cfg_pool, pos); } else { msg_err_config ("invalid map fetching protocol: %s", map_line); return NULL; } return pos; }
void* rspamd_fuzzy_backend_init_redis (struct rspamd_fuzzy_backend *bk, const ucl_object_t *obj, struct rspamd_config *cfg, GError **err) { struct rspamd_fuzzy_backend_redis *backend; const ucl_object_t *elt; gboolean ret = FALSE; guchar id_hash[rspamd_cryptobox_HASHBYTES]; rspamd_cryptobox_hash_state_t st; backend = g_slice_alloc0 (sizeof (*backend)); backend->timeout = REDIS_DEFAULT_TIMEOUT; backend->redis_object = REDIS_DEFAULT_OBJECT; ret = rspamd_fuzzy_backend_redis_try_ucl (backend, obj, cfg); /* Now try global redis settings */ if (!ret) { elt = ucl_object_lookup (cfg->rcl_obj, "redis"); if (elt) { const ucl_object_t *specific_obj; specific_obj = ucl_object_lookup_any (elt, "fuzzy", "fuzzy_storage", NULL); if (specific_obj) { ret = rspamd_fuzzy_backend_redis_try_ucl (backend, specific_obj, cfg); } else { ret = rspamd_fuzzy_backend_redis_try_ucl (backend, elt, cfg); } } } if (!ret) { msg_err_config ("cannot init redis backend for fuzzy storage"); g_slice_free1 (sizeof (*backend), backend); return NULL; } REF_INIT_RETAIN (backend, rspamd_fuzzy_backend_redis_dtor); backend->pool = cfg->redis_pool; rspamd_cryptobox_hash_init (&st, NULL, 0); rspamd_cryptobox_hash_update (&st, backend->redis_object, strlen (backend->redis_object)); if (backend->dbname) { rspamd_cryptobox_hash_update (&st, backend->dbname, strlen (backend->dbname)); } if (backend->password) { rspamd_cryptobox_hash_update (&st, backend->password, strlen (backend->password)); } rspamd_cryptobox_hash_final (&st, id_hash); backend->id = rspamd_encode_base32 (id_hash, sizeof (id_hash)); return backend; }
gint dkim_module_config (struct rspamd_config *cfg) { const ucl_object_t *value; gint res = TRUE, cb_id = -1, check_id = -1; guint cache_size; gboolean got_trusted = FALSE; if (!rspamd_config_is_module_enabled (cfg, "dkim")) { return TRUE; } dkim_module_ctx->whitelist_ip = radix_create_compressed (); if ((value = rspamd_config_get_module_opt (cfg, "dkim", "symbol_reject")) != NULL) { dkim_module_ctx->symbol_reject = ucl_obj_tostring (value); } else { dkim_module_ctx->symbol_reject = DEFAULT_SYMBOL_REJECT; } if ((value = rspamd_config_get_module_opt (cfg, "dkim", "symbol_tempfail")) != NULL) { dkim_module_ctx->symbol_tempfail = ucl_obj_tostring (value); } else { dkim_module_ctx->symbol_tempfail = DEFAULT_SYMBOL_TEMPFAIL; } if ((value = rspamd_config_get_module_opt (cfg, "dkim", "symbol_allow")) != NULL) { dkim_module_ctx->symbol_allow = ucl_obj_tostring (value); } else { dkim_module_ctx->symbol_allow = DEFAULT_SYMBOL_ALLOW; } if ((value = rspamd_config_get_module_opt (cfg, "dkim", "symbol_na")) != NULL) { dkim_module_ctx->symbol_na = ucl_obj_tostring (value); } else { dkim_module_ctx->symbol_na = DEFAULT_SYMBOL_NA; } if ((value = rspamd_config_get_module_opt (cfg, "dkim", "dkim_cache_size")) != NULL) { cache_size = ucl_obj_toint (value); } else { cache_size = DEFAULT_CACHE_SIZE; } if ((value = rspamd_config_get_module_opt (cfg, "dkim", "time_jitter")) != NULL) { dkim_module_ctx->time_jitter = ucl_obj_todouble (value); } else { dkim_module_ctx->time_jitter = DEFAULT_TIME_JITTER; } if ((value = rspamd_config_get_module_opt (cfg, "dkim", "max_sigs")) != NULL) { dkim_module_ctx->max_sigs = ucl_object_toint (value); } if ((value = rspamd_config_get_module_opt (cfg, "dkim", "whitelist")) != NULL) { rspamd_config_radix_from_ucl (cfg, value, "DKIM whitelist", &dkim_module_ctx->whitelist_ip, NULL); } if ((value = rspamd_config_get_module_opt (cfg, "dkim", "domains")) != NULL) { if (!rspamd_map_add_from_ucl (cfg, value, "DKIM domains", rspamd_kv_list_read, rspamd_kv_list_fin, (void **)&dkim_module_ctx->dkim_domains)) { msg_warn_config ("cannot load dkim domains list from %s", ucl_obj_tostring (value)); } else { got_trusted = TRUE; } } if (!got_trusted && (value = rspamd_config_get_module_opt (cfg, "dkim", "trusted_domains")) != NULL) { if (!rspamd_map_add_from_ucl (cfg, value, "DKIM domains", rspamd_kv_list_read, rspamd_kv_list_fin, (void **)&dkim_module_ctx->dkim_domains)) { msg_warn_config ("cannot load dkim domains list from %s", ucl_obj_tostring (value)); } else { got_trusted = TRUE; } } if ((value = rspamd_config_get_module_opt (cfg, "dkim", "strict_multiplier")) != NULL) { dkim_module_ctx->strict_multiplier = ucl_obj_toint (value); } else { dkim_module_ctx->strict_multiplier = 1; } if ((value = rspamd_config_get_module_opt (cfg, "dkim", "trusted_only")) != NULL) { dkim_module_ctx->trusted_only = ucl_obj_toboolean (value); } else { dkim_module_ctx->trusted_only = FALSE; } if ((value = rspamd_config_get_module_opt (cfg, "dkim", "skip_multi")) != NULL) { dkim_module_ctx->skip_multi = ucl_obj_toboolean (value); } else { dkim_module_ctx->skip_multi = FALSE; } if (dkim_module_ctx->trusted_only && !got_trusted) { msg_err_config ( "trusted_only option is set and no trusted domains are defined; disabling dkim module completely as it is useless in this case"); } else { cb_id = rspamd_symbols_cache_add_symbol (cfg->cache, dkim_module_ctx->symbol_reject, 0, dkim_symbol_callback, NULL, SYMBOL_TYPE_NORMAL|SYMBOL_TYPE_FINE, -1); rspamd_symbols_cache_add_symbol (cfg->cache, dkim_module_ctx->symbol_na, 0, NULL, NULL, SYMBOL_TYPE_VIRTUAL|SYMBOL_TYPE_FINE, cb_id); rspamd_symbols_cache_add_symbol (cfg->cache, dkim_module_ctx->symbol_tempfail, 0, NULL, NULL, SYMBOL_TYPE_VIRTUAL|SYMBOL_TYPE_FINE, cb_id); rspamd_symbols_cache_add_symbol (cfg->cache, dkim_module_ctx->symbol_allow, 0, NULL, NULL, SYMBOL_TYPE_VIRTUAL|SYMBOL_TYPE_FINE, cb_id); dkim_module_ctx->dkim_hash = rspamd_lru_hash_new ( cache_size, g_free, /* Keys are just C-strings */ dkim_module_key_dtor); msg_info_config ("init internal dkim module"); #ifndef HAVE_OPENSSL msg_warn_config ( "openssl is not found so dkim rsa check is disabled, only check body hash, it is NOT safe to trust these results"); #endif } if ((value = rspamd_config_get_module_opt (cfg, "dkim", "sign_condition")) != NULL) { const gchar *lua_script; lua_script = ucl_object_tostring (value); if (lua_script) { if (luaL_dostring (cfg->lua_state, lua_script) != 0) { msg_err_config ("cannot execute lua script for fuzzy " "learn condition: %s", lua_tostring (cfg->lua_state, -1)); } else { if (lua_type (cfg->lua_state, -1) == LUA_TFUNCTION) { dkim_module_ctx->sign_condition_ref = luaL_ref (cfg->lua_state, LUA_REGISTRYINDEX); dkim_module_ctx->dkim_sign_hash = rspamd_lru_hash_new ( 128, g_free, /* Keys are just C-strings */ (GDestroyNotify)rspamd_dkim_sign_key_unref); check_id = rspamd_symbols_cache_add_symbol (cfg->cache, "DKIM_SIGN", 0, dkim_sign_callback, NULL, SYMBOL_TYPE_CALLBACK|SYMBOL_TYPE_FINE, -1); msg_info_config ("init condition script for DKIM signing"); /* * Allow dkim signing to be executed only after dkim check */ if (cb_id > 0) { rspamd_symbols_cache_add_delayed_dependency (cfg->cache, "DKIM_SIGN", dkim_module_ctx->symbol_reject); } rspamd_config_add_metric_symbol (cfg, DEFAULT_METRIC, "DKIM_SIGN", 0.0, "DKIM signature fake symbol", "dkim", RSPAMD_SYMBOL_FLAG_IGNORE, 1); } else { msg_err_config ("lua script must return " "function(task) and not %s", lua_typename (cfg->lua_state, lua_type (cfg->lua_state, -1))); } } } } return res; }
static gboolean rspamd_fuzzy_backend_redis_try_ucl (struct rspamd_fuzzy_backend_redis *backend, const ucl_object_t *obj, struct rspamd_config *cfg) { const ucl_object_t *elt, *relt; elt = ucl_object_lookup_any (obj, "read_servers", "servers", NULL); if (elt == NULL) { return FALSE; } backend->read_servers = rspamd_upstreams_create (cfg->ups_ctx); if (!rspamd_upstreams_from_ucl (backend->read_servers, elt, REDIS_DEFAULT_PORT, NULL)) { msg_err_config ("cannot get read servers configuration"); return FALSE; } relt = elt; elt = ucl_object_lookup (obj, "write_servers"); if (elt == NULL) { /* Use read servers as write ones */ g_assert (relt != NULL); backend->write_servers = rspamd_upstreams_create (cfg->ups_ctx); if (!rspamd_upstreams_from_ucl (backend->write_servers, relt, REDIS_DEFAULT_PORT, NULL)) { msg_err_config ("cannot get write servers configuration"); return FALSE; } } else { backend->write_servers = rspamd_upstreams_create (cfg->ups_ctx); if (!rspamd_upstreams_from_ucl (backend->write_servers, elt, REDIS_DEFAULT_PORT, NULL)) { msg_err_config ("cannot get write servers configuration"); rspamd_upstreams_destroy (backend->write_servers); backend->write_servers = NULL; } } elt = ucl_object_lookup (obj, "prefix"); if (elt == NULL || ucl_object_type (elt) != UCL_STRING) { backend->redis_object = REDIS_DEFAULT_OBJECT; } else { backend->redis_object = ucl_object_tostring (elt); } elt = ucl_object_lookup (obj, "timeout"); if (elt) { backend->timeout = ucl_object_todouble (elt); } else { backend->timeout = REDIS_DEFAULT_TIMEOUT; } elt = ucl_object_lookup (obj, "password"); if (elt) { backend->password = ucl_object_tostring (elt); } else { backend->password = NULL; } elt = ucl_object_lookup_any (obj, "db", "database", "dbname", NULL); if (elt) { backend->dbname = ucl_object_tostring (elt); } else { backend->dbname = NULL; } return TRUE; }