static struct cache_savepoint *
rspamd_symbols_cache_make_checkpoint (struct rspamd_task *task,
struct symbols_cache *cache)
{
struct cache_savepoint *checkpoint;
if (cache->items_by_id->len != cache->items_by_order->d->len) {
/*
* Cache has been modified, need to resort it
*/
msg_info_cache ("symbols cache has been modified since last check:"
" old items: %ud, new items: %ud",
cache->items_by_order->d->len, cache->items_by_id->len);
rspamd_symbols_cache_resort (cache);
}
checkpoint = rspamd_mempool_alloc0 (task->task_pool, sizeof (*checkpoint));
/* Bit 0: check started, Bit 1: check finished */
checkpoint->processed_bits = rspamd_mempool_alloc0 (task->task_pool,
NBYTES (cache->used_items) * 2);
checkpoint->waitq = g_ptr_array_new ();
g_assert (cache->items_by_order != NULL);
checkpoint->version = cache->items_by_order->d->len;
checkpoint->order = cache->items_by_order;
REF_RETAIN (checkpoint->order);
rspamd_mempool_add_destructor (task->task_pool,
rspamd_symbols_cache_order_unref, checkpoint->order);
rspamd_mempool_add_destructor (task->task_pool,
rspamd_ptr_array_free_hard, checkpoint->waitq);
task->checkpoint = checkpoint;
rspamd_create_metric_result (task, DEFAULT_METRIC);
return checkpoint;
}
static void
rspamd_register_symbol_fromlua (lua_State *L,
struct rspamd_config *cfg,
const gchar *name,
gint ref,
gdouble weight,
gint priority,
enum rspamd_symbol_type type)
{
struct lua_callback_data *cd;
cd = rspamd_mempool_alloc0 (cfg->cfg_pool,
sizeof (struct lua_callback_data));
cd->cb_is_ref = TRUE;
cd->callback.ref = ref;
cd->L = L;
if (name) {
cd->symbol = rspamd_mempool_strdup (cfg->cfg_pool, name);
}
register_symbol_common (&cfg->cache,
name,
weight,
priority,
lua_metric_symbol_callback,
cd,
type);
rspamd_mempool_add_destructor (cfg->cfg_pool,
(rspamd_mempool_destruct_t)lua_destroy_cfg_symbol,
cd);
}
struct rspamd_worker_conf *
rspamd_config_new_worker (struct rspamd_config *cfg,
struct rspamd_worker_conf *c)
{
if (c == NULL) {
c =
rspamd_mempool_alloc0 (cfg->cfg_pool,
sizeof (struct rspamd_worker_conf));
c->params = g_hash_table_new (rspamd_str_hash, rspamd_str_equal);
c->active_workers = g_queue_new ();
rspamd_mempool_add_destructor (cfg->cfg_pool,
(rspamd_mempool_destruct_t)g_hash_table_destroy,
c->params);
rspamd_mempool_add_destructor (cfg->cfg_pool,
(rspamd_mempool_destruct_t)g_queue_free,
c->active_workers);
#ifdef HAVE_SC_NPROCESSORS_ONLN
c->count = sysconf (_SC_NPROCESSORS_ONLN);
#else
c->count = DEFAULT_WORKERS_NUM;
#endif
c->rlimit_nofile = 0;
c->rlimit_maxcore = 0;
}
return c;
}
static rspamd_expression_atom_t *
lua_atom_parse (const gchar *line, gsize len,
rspamd_mempool_t *pool, gpointer ud, GError **err)
{
struct lua_expression *e = (struct lua_expression *)ud;
rspamd_expression_atom_t *atom;
gsize rlen;
const gchar *tok;
lua_rawgeti (e->L, LUA_REGISTRYINDEX, e->parse_idx);
lua_pushlstring (e->L, line, len);
if (lua_pcall (e->L, 1, 1, 0) != 0) {
msg_info ("callback call failed: %s", lua_tostring (e->L, -1));
}
if (lua_type (e->L, -1) != LUA_TSTRING) {
g_set_error (err, lua_expr_quark(), 500, "cannot parse lua atom");
lua_pop (e->L, 1);
return NULL;
}
tok = lua_tolstring (e->L, -1, &rlen);
atom = rspamd_mempool_alloc0 (e->pool, sizeof (*atom));
atom->str = rspamd_mempool_strdup (e->pool, tok);
atom->len = rlen;
atom->data = ud;
lua_pop (e->L, 1);
return atom;
}
gpointer
rspamd_redis_runtime (struct rspamd_task *task,
struct rspamd_statfile_config *stcf,
gboolean learn, gpointer c)
{
struct redis_stat_ctx *ctx = REDIS_CTX (c);
struct redis_stat_runtime *rt;
struct upstream *up;
rspamd_inet_addr_t *addr;
g_assert (ctx != NULL);
g_assert (stcf != NULL);
if (learn && ctx->write_servers == NULL) {
msg_err_task ("no write servers defined for %s, cannot learn", stcf->symbol);
return NULL;
}
if (learn) {
up = rspamd_upstream_get (ctx->write_servers,
RSPAMD_UPSTREAM_MASTER_SLAVE,
NULL,
0);
}
else {
up = rspamd_upstream_get (ctx->read_servers,
RSPAMD_UPSTREAM_ROUND_ROBIN,
NULL,
0);
}
if (up == NULL) {
msg_err_task ("no upstreams reachable");
return NULL;
}
rt = rspamd_mempool_alloc0 (task->task_pool, sizeof (*rt));
rspamd_redis_expand_object (ctx->redis_object, ctx, task,
&rt->redis_object_expanded);
rt->selected = up;
rt->task = task;
rt->ctx = ctx;
rt->stcf = stcf;
addr = rspamd_upstream_addr (up);
g_assert (addr != NULL);
rt->redis = redisAsyncConnect (rspamd_inet_address_to_string (addr),
rspamd_inet_address_get_port (addr));
if (rt->redis == NULL) {
msg_err_task ("cannot connect redis");
return NULL;
}
redisLibeventAttach (rt->redis, task->ev_base);
rspamd_redis_maybe_auth (ctx, rt->redis);
return rt;
}
struct rspamd_statfile_config *
rspamd_config_new_statfile (struct rspamd_config *cfg,
struct rspamd_statfile_config *c)
{
if (c == NULL) {
c =
rspamd_mempool_alloc0 (cfg->cfg_pool,
sizeof (struct rspamd_statfile_config));
}
return c;
}
static struct fuzzy_rule *
fuzzy_rule_new (const char *default_symbol, rspamd_mempool_t *pool)
{
struct fuzzy_rule *rule;
rule = rspamd_mempool_alloc0 (pool, sizeof (struct fuzzy_rule));
rule->mappings = g_hash_table_new (g_direct_hash, g_direct_equal);
rule->symbol = default_symbol;
rspamd_mempool_add_destructor (pool,
(rspamd_mempool_destruct_t)g_hash_table_unref,
rule->mappings);
rule->read_only = FALSE;
return rule;
}
struct rspamd_symbols_group *
rspamd_config_new_group (struct rspamd_config *cfg, struct metric *metric,
const gchar *name)
{
struct rspamd_symbols_group *gr;
gr = rspamd_mempool_alloc0 (cfg->cfg_pool, sizeof (*gr));
gr->symbols = g_hash_table_new (rspamd_strcase_hash,
rspamd_strcase_equal);
rspamd_mempool_add_destructor (cfg->cfg_pool,
(rspamd_mempool_destruct_t)g_hash_table_unref, gr->symbols);
gr->name = rspamd_mempool_strdup (cfg->cfg_pool, name);
g_hash_table_insert (metric->groups, gr->name, gr);
return gr;
}
gint
lua_config_radix_from_config (lua_State *L)
{
struct rspamd_config *cfg = lua_check_config (L, 1);
const gchar *mname, *optname;
const ucl_object_t *obj;
struct rspamd_lua_map *map, **pmap;
if (!cfg) {
return luaL_error (L, "invalid arguments");
}
mname = luaL_checkstring (L, 2);
optname = luaL_checkstring (L, 3);
if (mname && optname) {
obj = rspamd_config_get_module_opt (cfg, mname, optname);
if (obj) {
map = rspamd_mempool_alloc0 (cfg->cfg_pool, sizeof (*map));
map->data.radix = radix_create_compressed ();
map->type = RSPAMD_LUA_MAP_RADIX;
map->data.radix = radix_create_compressed ();
map->flags |= RSPAMD_LUA_MAP_FLAG_EMBEDDED;
radix_add_generic_iplist (ucl_obj_tostring (obj), &map->data.radix,
TRUE);
pmap = lua_newuserdata (L, sizeof (void *));
*pmap = map;
rspamd_lua_setclass (L, "rspamd{map}", -1);
} else {
msg_warn_config ("Couldnt find config option [%s][%s]", mname,
optname);
lua_pushnil (L);
}
}
else {
return luaL_error (L, "invalid arguments");
}
return 1;
}
struct rspamd_classifier_config *
rspamd_config_new_classifier (struct rspamd_config *cfg,
struct rspamd_classifier_config *c)
{
if (c == NULL) {
c =
rspamd_mempool_alloc0 (cfg->cfg_pool,
sizeof (struct rspamd_classifier_config));
}
if (c->labels == NULL) {
c->labels = g_hash_table_new_full (rspamd_str_hash,
rspamd_str_equal,
NULL,
(GDestroyNotify)g_list_free);
rspamd_mempool_add_destructor (cfg->cfg_pool,
(rspamd_mempool_destruct_t) g_hash_table_destroy,
c->labels);
}
return c;
}
static void
rspamd_html_url_is_phished (rspamd_mempool_t *pool,
struct rspamd_url *href_url,
const guchar *url_text,
gsize len,
gboolean *url_found)
{
struct rspamd_url *text_url;
gint rc;
gchar *url_str = NULL;
*url_found = FALSE;
if (rspamd_url_find (pool, url_text, len, &url_str, TRUE) && url_str != NULL) {
text_url = rspamd_mempool_alloc0 (pool, sizeof (struct rspamd_url));
rc = rspamd_url_parse (text_url, url_str, strlen (url_str), pool);
if (rc == URI_ERRNO_OK) {
if (href_url->hostlen != text_url->hostlen || memcmp (href_url->host,
text_url->host, href_url->hostlen) != 0) {
if (href_url->tldlen != text_url->tldlen || memcmp (href_url->tld,
text_url->tld, href_url->tldlen) != 0) {
href_url->flags |= RSPAMD_URL_FLAG_PHISHED;
href_url->phished_url = text_url;
}
}
*url_found = TRUE;
}
else {
msg_info_pool ("extract of url '%s' failed: %s",
url_str,
rspamd_url_strerror (rc));
}
}
}
gint
lua_config_add_hash_map (lua_State *L)
{
struct rspamd_config *cfg = lua_check_config (L, 1);
const gchar *map_line, *description;
struct rspamd_lua_map *map, **pmap;
struct rspamd_map *m;
if (cfg) {
map_line = luaL_checkstring (L, 2);
description = lua_tostring (L, 3);
map = rspamd_mempool_alloc0 (cfg->cfg_pool, sizeof (*map));
map->data.hash = g_hash_table_new (rspamd_strcase_hash,
rspamd_strcase_equal);
map->type = RSPAMD_LUA_MAP_SET;
if ((m = rspamd_map_add (cfg, map_line, description,
rspamd_hosts_read,
rspamd_hosts_fin,
(void **)&map->data.hash)) == NULL) {
msg_warn_config ("invalid set map %s", map_line);
g_hash_table_destroy (map->data.hash);
lua_pushnil (L);
return 1;
}
map->map = m;
pmap = lua_newuserdata (L, sizeof (void *));
*pmap = map;
rspamd_lua_setclass (L, "rspamd{map}", -1);
}
else {
return luaL_error (L, "invalid arguments");
}
return 1;
}
gint
lua_config_add_radix_map (lua_State *L)
{
struct rspamd_config *cfg = lua_check_config (L, 1);
const gchar *map_line, *description;
struct rspamd_lua_map *map, **pmap;
struct rspamd_map *m;
if (cfg) {
map_line = luaL_checkstring (L, 2);
description = lua_tostring (L, 3);
map = rspamd_mempool_alloc0 (cfg->cfg_pool, sizeof (*map));
map->data.radix = radix_create_compressed ();
map->type = RSPAMD_LUA_MAP_RADIX;
if ((m = rspamd_map_add (cfg, map_line, description,
rspamd_radix_read,
rspamd_radix_fin,
(void **)&map->data.radix)) == NULL) {
msg_warn_config ("invalid radix map %s", map_line);
radix_destroy_compressed (map->data.radix);
lua_pushnil (L);
return 1;
}
map->map = m;
pmap = lua_newuserdata (L, sizeof (void *));
*pmap = map;
rspamd_lua_setclass (L, "rspamd{map}", -1);
}
else {
return luaL_error (L, "invalid arguments");
}
return 1;
}
struct metric *
rspamd_config_new_metric (struct rspamd_config *cfg, struct metric *c,
const gchar *name)
{
int i;
if (c == NULL) {
c = rspamd_mempool_alloc0 (cfg->cfg_pool, sizeof (struct metric));
c->grow_factor = 1.0;
c->symbols = g_hash_table_new (rspamd_str_hash, rspamd_str_equal);
c->groups = g_hash_table_new (rspamd_str_hash, rspamd_str_equal);
for (i = METRIC_ACTION_REJECT; i < METRIC_ACTION_MAX; i++) {
c->actions[i].score = -1.0;
c->actions[i].action = i;
}
c->subject = SPAM_SUBJECT;
c->name = rspamd_mempool_strdup (cfg->cfg_pool, name);
rspamd_mempool_add_destructor (cfg->cfg_pool,
(rspamd_mempool_destruct_t) g_hash_table_unref,
c->symbols);
rspamd_mempool_add_destructor (cfg->cfg_pool,
(rspamd_mempool_destruct_t) g_hash_table_unref,
c->groups);
g_hash_table_insert (cfg->metrics, (void *)c->name, c);
cfg->metrics_list = g_list_prepend (cfg->metrics_list, c);
if (strcmp (c->name, DEFAULT_METRIC) == 0) {
cfg->default_metric = c;
}
}
return c;
}
static void
rspamd_archive_process_rar (struct rspamd_task *task,
struct rspamd_mime_part *part)
{
const guchar *p, *end, *section_start;
const guchar rar_v5_magic[] = {0x52, 0x61, 0x72, 0x21, 0x1A, 0x07, 0x01, 0x00},
rar_v4_magic[] = {0x52, 0x61, 0x72, 0x21, 0x1A, 0x07, 0x00};
const guint rar_encrypted_header = 4, rar_main_header = 1,
rar_file_header = 2;
guint64 vint, sz, comp_sz = 0, uncomp_sz = 0, flags = 0, type = 0;
struct rspamd_archive *arch;
struct rspamd_archive_file *f;
gint r;
p = part->parsed_data.begin;
end = p + part->parsed_data.len;
if ((gsize)(end - p) <= sizeof (rar_v5_magic)) {
msg_debug_task ("rar archive is invalid (too small)");
return;
}
if (memcmp (p, rar_v5_magic, sizeof (rar_v5_magic)) == 0) {
p += sizeof (rar_v5_magic);
}
else if (memcmp (p, rar_v4_magic, sizeof (rar_v4_magic)) == 0) {
p += sizeof (rar_v4_magic);
rspamd_archive_process_rar_v4 (task, p, end, part);
return;
}
else {
msg_debug_task ("rar archive is invalid (no rar magic)");
return;
}
/* Rar v5 format */
arch = rspamd_mempool_alloc0 (task->task_pool, sizeof (*arch));
arch->files = g_ptr_array_new ();
arch->type = RSPAMD_ARCHIVE_RAR;
rspamd_mempool_add_destructor (task->task_pool, rspamd_archive_dtor,
arch);
/* Now we can have either encryption header or archive header */
/* Crc 32 */
RAR_SKIP_BYTES (sizeof (guint32));
/* Size */
RAR_READ_VINT_SKIP ();
sz = vint;
/* Type */
section_start = p;
RAR_READ_VINT_SKIP ();
type = vint;
/* Header flags */
RAR_READ_VINT_SKIP ();
flags = vint;
if (flags & 0x1) {
/* Have extra zone */
RAR_READ_VINT_SKIP ();
}
if (flags & 0x2) {
/* Data zone is presented */
RAR_READ_VINT_SKIP ();
sz += vint;
}
if (type == rar_encrypted_header) {
/* We can't read any further information as archive is encrypted */
arch->flags |= RSPAMD_ARCHIVE_ENCRYPTED;
goto end;
}
else if (type != rar_main_header) {
msg_debug_task ("rar archive is invalid (bad main header)");
return;
}
/* Nothing useful in main header */
p = section_start;
RAR_SKIP_BYTES (sz);
while (p < end) {
/* Read the next header */
/* Crc 32 */
RAR_SKIP_BYTES (sizeof (guint32));
/* Size */
RAR_READ_VINT_SKIP ();
sz = vint;
if (sz == 0) {
/* Zero sized block - error */
msg_debug_task ("rar archive is invalid (zero size block)");
return;
}
section_start = p;
/* Type */
RAR_READ_VINT_SKIP ();
type = vint;
/* Header flags */
RAR_READ_VINT_SKIP ();
flags = vint;
if (flags & 0x1) {
/* Have extra zone */
RAR_READ_VINT_SKIP ();
}
if (flags & 0x2) {
/* Data zone is presented */
RAR_READ_VINT_SKIP ();
sz += vint;
comp_sz = vint;
}
if (type != rar_file_header) {
p = section_start;
RAR_SKIP_BYTES (sz);
}
else {
/* We have a file header, go forward */
guint64 fname_len;
/* File header specific flags */
RAR_READ_VINT_SKIP ();
flags = vint;
/* Unpacked size */
RAR_READ_VINT_SKIP ();
uncomp_sz = vint;
/* Attributes */
RAR_READ_VINT_SKIP ();
if (flags & 0x2) {
/* Unix mtime */
RAR_SKIP_BYTES (sizeof (guint32));
}
if (flags & 0x4) {
/* Crc32 */
RAR_SKIP_BYTES (sizeof (guint32));
}
/* Compression */
RAR_READ_VINT_SKIP ();
/* Host OS */
RAR_READ_VINT_SKIP ();
/* Filename length (finally!) */
RAR_READ_VINT_SKIP ();
fname_len = vint;
if (fname_len == 0 || fname_len > (gsize)(end - p)) {
msg_debug_task ("rar archive is invalid (bad fileame size)");
return;
}
f = g_slice_alloc0 (sizeof (*f));
f->uncompressed_size = uncomp_sz;
f->compressed_size = comp_sz;
f->fname = g_string_new_len (p, fname_len);
g_ptr_array_add (arch->files, f);
/* Restore p to the beginning of the header */
p = section_start;
RAR_SKIP_BYTES (sz);
}
}
end:
part->flags |= RSPAMD_MIME_PART_ARCHIVE;
part->specific.arch = arch;
if (part->cd != NULL) {
arch->archive_name = &part->cd->filename;
}
arch->size = part->parsed_data.len;
}
static void
rspamd_archive_process_rar_v4 (struct rspamd_task *task, const guchar *start,
const guchar *end, struct rspamd_mime_part *part)
{
const guchar *p = start, *start_section;
guint8 type;
guint flags;
guint64 sz, comp_sz = 0, uncomp_sz = 0;
struct rspamd_archive *arch;
struct rspamd_archive_file *f;
arch = rspamd_mempool_alloc0 (task->task_pool, sizeof (*arch));
arch->files = g_ptr_array_new ();
arch->type = RSPAMD_ARCHIVE_RAR;
rspamd_mempool_add_destructor (task->task_pool, rspamd_archive_dtor,
arch);
while (p < end) {
/* Crc16 */
start_section = p;
RAR_SKIP_BYTES (sizeof (guint16));
type = *p;
p ++;
RAR_READ_UINT16 (flags);
if (type == 0x73) {
/* Main header, check for encryption */
if (flags & 0x80) {
arch->flags |= RSPAMD_ARCHIVE_ENCRYPTED;
goto end;
}
}
RAR_READ_UINT16 (sz);
if (flags & 0x8000) {
/* We also need to read ADD_SIZE element */
guint32 tmp;
RAR_READ_UINT32 (tmp);
sz += tmp;
/* This is also used as PACK_SIZE */
comp_sz = tmp;
}
if (sz == 0) {
/* Zero sized block - error */
msg_debug_task ("rar archive is invalid (zero size block)");
return;
}
if (type == 0x74) {
guint fname_len;
/* File header */
/* Uncompressed size */
RAR_READ_UINT32 (uncomp_sz);
/* Skip to NAME_SIZE element */
RAR_SKIP_BYTES (11);
RAR_READ_UINT16 (fname_len);
if (fname_len == 0 || fname_len > (gsize)(end - p)) {
msg_debug_task ("rar archive is invalid (bad fileame size)");
return;
}
/* Attrs */
RAR_SKIP_BYTES (4);
if (flags & 0x100) {
/* We also need to read HIGH_PACK_SIZE */
guint32 tmp;
RAR_READ_UINT32 (tmp);
sz += tmp;
comp_sz += tmp;
/* HIGH_UNP_SIZE */
RAR_READ_UINT32 (tmp);
uncomp_sz += tmp;
}
f = g_slice_alloc0 (sizeof (*f));
if (flags & 0x200) {
/* We have unicode + normal version */
guchar *tmp;
tmp = memchr (p, '\0', fname_len);
if (tmp != NULL) {
/* Just use ASCII version */
f->fname = g_string_new_len (p, tmp - p);
}
else {
/* We have UTF8 filename, use it as is */
f->fname = g_string_new_len (p, fname_len);
}
}
else {
f->fname = g_string_new_len (p, fname_len);
}
f->compressed_size = comp_sz;
f->uncompressed_size = uncomp_sz;
if (flags & 0x4) {
f->flags |= RSPAMD_ARCHIVE_FILE_ENCRYPTED;
}
g_ptr_array_add (arch->files, f);
}
p = start_section;
RAR_SKIP_BYTES (sz);
}
end:
part->flags |= RSPAMD_MIME_PART_ARCHIVE;
part->specific.arch = arch;
arch->archive_name = &part->cd->filename;
arch->size = part->parsed_data.len;
}
gboolean
rspamd_map_add (struct rspamd_config *cfg,
const gchar *map_line,
const gchar *description,
map_cb_t read_callback,
map_fin_cb_t fin_callback,
void **user_data)
{
struct rspamd_map *new_map;
enum fetch_proto proto;
const gchar *def, *p, *hostend;
struct file_map_data *fdata;
struct http_map_data *hdata;
gchar portbuf[6], *cksum_encoded, cksum[BLAKE2B_OUTBYTES];
gint i, s, r;
struct addrinfo hints, *res;
rspamd_mempool_t *pool;
/* First of all detect protocol line */
if (!rspamd_map_check_proto (map_line, (int *)&proto, &def)) {
return FALSE;
}
/* Constant pool */
if (cfg->map_pool == NULL) {
cfg->map_pool = rspamd_mempool_new (rspamd_mempool_suggest_size (),
"map");
memcpy (cfg->map_pool->tag.uid, cfg->cfg_pool->tag.uid,
sizeof (cfg->map_pool->tag.uid));
}
new_map = rspamd_mempool_alloc0 (cfg->map_pool, sizeof (struct rspamd_map));
new_map->read_callback = read_callback;
new_map->fin_callback = fin_callback;
new_map->user_data = user_data;
new_map->protocol = proto;
new_map->cfg = cfg;
new_map->id = g_random_int ();
new_map->locked =
rspamd_mempool_alloc0_shared (cfg->cfg_pool, sizeof (gint));
if (proto == MAP_PROTO_FILE) {
new_map->uri = rspamd_mempool_strdup (cfg->cfg_pool, def);
def = new_map->uri;
}
else {
new_map->uri = rspamd_mempool_strdup (cfg->cfg_pool, map_line);
}
if (description != NULL) {
new_map->description =
rspamd_mempool_strdup (cfg->cfg_pool, description);
}
/* Now check for each proto separately */
if (proto == MAP_PROTO_FILE) {
fdata =
rspamd_mempool_alloc0 (cfg->map_pool,
sizeof (struct file_map_data));
if (access (def, R_OK) == -1) {
if (errno != ENOENT) {
msg_err_config ("cannot open file '%s': %s", def, strerror
(errno));
return FALSE;
}
msg_info_config (
"map '%s' is not found, but it can be loaded automatically later",
def);
/* We still can add this file */
fdata->st.st_mtime = -1;
}
else {
stat (def, &fdata->st);
}
fdata->filename = rspamd_mempool_strdup (cfg->map_pool, def);
new_map->map_data = fdata;
}
else if (proto == MAP_PROTO_HTTP) {
hdata =
rspamd_mempool_alloc0 (cfg->map_pool,
sizeof (struct http_map_data));
/* Try to search port */
if ((p = strchr (def, ':')) != NULL) {
hostend = p;
i = 0;
p++;
while (g_ascii_isdigit (*p) && i < (gint)sizeof (portbuf) - 1) {
portbuf[i++] = *p++;
}
if (*p != '/') {
msg_info_config ("bad http map definition: %s", def);
return FALSE;
}
portbuf[i] = '\0';
hdata->port = atoi (portbuf);
}
else {
/* Default http port */
rspamd_snprintf (portbuf, sizeof (portbuf), "80");
hdata->port = 80;
/* Now separate host from path */
if ((p = strchr (def, '/')) == NULL) {
msg_info_config ("bad http map definition: %s", def);
return FALSE;
}
hostend = p;
}
hdata->host = rspamd_mempool_alloc (cfg->map_pool, hostend - def + 1);
rspamd_strlcpy (hdata->host, def, hostend - def + 1);
hdata->path = rspamd_mempool_strdup (cfg->map_pool, p);
/* Now try to resolve */
memset (&hints, 0, sizeof (hints));
hints.ai_family = AF_UNSPEC; /* Allow IPv4 or IPv6 */
hints.ai_socktype = SOCK_STREAM; /* Stream socket */
hints.ai_flags = 0;
hints.ai_protocol = 0; /* Any protocol */
hints.ai_canonname = NULL;
hints.ai_addr = NULL;
hints.ai_next = NULL;
if ((r = getaddrinfo (hdata->host, portbuf, &hints, &res)) == 0) {
hdata->addr = res;
rspamd_mempool_add_destructor (cfg->cfg_pool,
(rspamd_mempool_destruct_t)freeaddrinfo, hdata->addr);
}
else {
msg_err_config ("address resolution for %s failed: %s",
hdata->host,
gai_strerror (r));
return FALSE;
}
/* Now try to connect */
if ((s = rspamd_socket_tcp (hdata->addr, FALSE, FALSE)) == -1) {
msg_info_config ("cannot connect to http server %s: %d, %s",
hdata->host,
errno,
strerror (errno));
return FALSE;
}
close (s);
hdata->conn = rspamd_http_connection_new (http_map_read, http_map_error,
http_map_finish,
RSPAMD_HTTP_BODY_PARTIAL | RSPAMD_HTTP_CLIENT_SIMPLE,
RSPAMD_HTTP_CLIENT, NULL);
new_map->map_data = hdata;
}
/* Temp pool */
blake2b (cksum, new_map->uri, NULL, sizeof (cksum), strlen (new_map->uri), 0);
cksum_encoded = rspamd_encode_base32 (cksum, sizeof (cksum));
new_map->pool = rspamd_mempool_new (rspamd_mempool_suggest_size (), "map");
memcpy (new_map->pool->tag.uid, cksum_encoded,
sizeof (new_map->pool->tag.uid));
g_free (cksum_encoded);
pool = new_map->pool;
msg_info_pool ("added map %s", new_map->uri);
cfg->maps = g_list_prepend (cfg->maps, new_map);
return TRUE;
}
gint
rspamd_symbols_cache_add_symbol (struct symbols_cache *cache,
const gchar *name,
double weight,
gint priority,
symbol_func_t func,
gpointer user_data,
enum rspamd_symbol_type type,
gint parent)
{
struct cache_item *item = NULL;
g_assert (cache != NULL);
if (name == NULL && type != SYMBOL_TYPE_CALLBACK) {
msg_warn ("no name for non-callback symbol!");
}
else if (type == SYMBOL_TYPE_VIRTUAL && parent == -1) {
msg_warn ("no parent symbol is associated with virtual symbol %s",
name);
}
if (name != NULL) {
if (g_hash_table_lookup (cache->items_by_symbol, name) != NULL) {
msg_err ("skip duplicate symbol registration for %s", name);
return -1;
}
}
item = rspamd_mempool_alloc0_shared (cache->static_pool,
sizeof (struct cache_item));
/*
* We do not share cd to skip locking, instead we'll just calculate it on
* save or accumulate
*/
item->cd = rspamd_mempool_alloc0 (cache->static_pool,
sizeof (struct counter_data));
if (name != NULL) {
item->symbol = rspamd_mempool_strdup (cache->static_pool, name);
}
item->func = func;
item->user_data = user_data;
item->priority = priority;
item->type = type;
item->weight = weight;
if (item->weight < 0 && item->priority == 0) {
/* Make priority for negative weighted symbols */
item->priority = 1;
}
item->id = cache->used_items;
item->parent = parent;
cache->used_items ++;
msg_debug ("used items: %d, added symbol: %s", cache->used_items, name);
rspamd_set_counter (item, 0);
g_ptr_array_add (cache->items_by_id, item);
g_ptr_array_add (cache->items_by_order, item);
item->deps = g_ptr_array_new ();
item->rdeps = g_ptr_array_new ();
rspamd_mempool_add_destructor (cache->static_pool,
rspamd_ptr_array_free_hard, item->deps);
rspamd_mempool_add_destructor (cache->static_pool,
rspamd_ptr_array_free_hard, item->rdeps);
if (name != NULL) {
g_hash_table_insert (cache->items_by_symbol, item->symbol, item);
}
return item->id;
}
gboolean
rspamd_config_add_metric_symbol (struct rspamd_config *cfg,
const gchar *metric_name, const gchar *symbol,
gdouble score, const gchar *description, const gchar *group,
gboolean one_shot, gboolean rewrite_existing)
{
struct rspamd_symbols_group *sym_group;
struct rspamd_symbol_def *sym_def;
GList *metric_list;
struct metric *metric;
gdouble *score_ptr;
g_assert (cfg != NULL);
g_assert (symbol != NULL);
if (metric_name == NULL) {
metric_name = DEFAULT_METRIC;
}
metric = g_hash_table_lookup (cfg->metrics, metric_name);
if (metric == NULL) {
msg_err_config ("metric %s has not been found", metric_name);
return FALSE;
}
if (g_hash_table_lookup (cfg->metrics_symbols, symbol) != NULL &&
!rewrite_existing) {
msg_debug_config ("symbol %s has been already registered, do not override");
return FALSE;
}
sym_def =
rspamd_mempool_alloc0 (cfg->cfg_pool, sizeof (struct rspamd_symbol_def));
score_ptr = rspamd_mempool_alloc (cfg->cfg_pool, sizeof (gdouble));
*score_ptr = score;
sym_def->score = score;
sym_def->weight_ptr = score_ptr;
sym_def->name = rspamd_mempool_strdup (cfg->cfg_pool, symbol);
sym_def->one_shot = one_shot;
if (description) {
sym_def->description = rspamd_mempool_strdup (cfg->cfg_pool, description);
}
msg_debug_config ("registered symbol %s with weight %.2f in metric %s and group %s",
sym_def->name, score, metric->name, group);
g_hash_table_insert (metric->symbols, sym_def->name, sym_def);
if ((metric_list =
g_hash_table_lookup (cfg->metrics_symbols, sym_def->name)) == NULL) {
metric_list = g_list_prepend (NULL, metric);
rspamd_mempool_add_destructor (cfg->cfg_pool,
(rspamd_mempool_destruct_t)g_list_free,
metric_list);
g_hash_table_insert (cfg->metrics_symbols, sym_def->name, metric_list);
}
else {
/* Slow but keep start element of list in safe */
if (!g_list_find (metric_list, metric)) {
metric_list = g_list_append (metric_list, metric);
}
}
/* Search for symbol group */
if (group == NULL) {
group = "ungrouped";
}
sym_group = g_hash_table_lookup (metric->groups, group);
if (sym_group == NULL) {
/* Create new group */
sym_group = rspamd_config_new_group (cfg, metric, group);
}
sym_def->gr = sym_group;
g_hash_table_insert (sym_group->symbols, sym_def->name, sym_def);
return TRUE;
}
gpointer
rspamd_stat_cache_redis_runtime (struct rspamd_task *task,
gpointer c, gboolean learn)
{
struct rspamd_redis_cache_ctx *ctx = c;
struct rspamd_redis_cache_runtime *rt;
struct upstream *up;
rspamd_inet_addr_t *addr;
g_assert (ctx != NULL);
if (learn && ctx->write_servers == NULL) {
msg_err_task ("no write servers defined for %s, cannot learn",
ctx->stcf->symbol);
return NULL;
}
if (task->tokens == NULL || task->tokens->len == 0) {
return NULL;
}
if (learn) {
up = rspamd_upstream_get (ctx->write_servers,
RSPAMD_UPSTREAM_MASTER_SLAVE,
NULL,
0);
}
else {
up = rspamd_upstream_get (ctx->read_servers,
RSPAMD_UPSTREAM_ROUND_ROBIN,
NULL,
0);
}
if (up == NULL) {
msg_err_task ("no upstreams reachable");
return NULL;
}
rt = rspamd_mempool_alloc0 (task->task_pool, sizeof (*rt));
rt->selected = up;
rt->task = task;
rt->ctx = ctx;
addr = rspamd_upstream_addr (up);
g_assert (addr != NULL);
rt->redis = redisAsyncConnect (rspamd_inet_address_to_string (addr),
rspamd_inet_address_get_port (addr));
g_assert (rt->redis != NULL);
redisLibeventAttach (rt->redis, task->ev_base);
/* Now check stats */
event_set (&rt->timeout_event, -1, EV_TIMEOUT, rspamd_redis_cache_timeout, rt);
event_base_set (task->ev_base, &rt->timeout_event);
rspamd_redis_cache_maybe_auth (ctx, rt->redis);
if (!learn) {
rspamd_stat_cache_redis_generate_id (task);
}
return rt;
}
struct rspamd_map *
rspamd_map_add (struct rspamd_config *cfg,
const gchar *map_line,
const gchar *description,
map_cb_t read_callback,
map_fin_cb_t fin_callback,
void **user_data)
{
struct rspamd_map *new_map;
const gchar *def;
struct file_map_data *fdata;
struct http_map_data *hdata;
gchar *cksum_encoded, cksum[rspamd_cryptobox_HASHBYTES];
rspamd_mempool_t *pool;
struct http_parser_url up;
rspamd_ftok_t tok;
if (cfg->map_pool == NULL) {
cfg->map_pool = rspamd_mempool_new (rspamd_mempool_suggest_size (),
"map");
memcpy (cfg->map_pool->tag.uid, cfg->cfg_pool->tag.uid,
sizeof (cfg->map_pool->tag.uid));
}
new_map = rspamd_mempool_alloc0 (cfg->map_pool, sizeof (struct rspamd_map));
/* First of all detect protocol line */
if (rspamd_map_check_proto (cfg, map_line, new_map) == NULL) {
return NULL;
}
new_map->read_callback = read_callback;
new_map->fin_callback = fin_callback;
new_map->user_data = user_data;
new_map->cfg = cfg;
new_map->id = g_random_int ();
new_map->locked =
rspamd_mempool_alloc0_shared (cfg->cfg_pool, sizeof (gint));
def = new_map->uri;
if (description != NULL) {
new_map->description =
rspamd_mempool_strdup (cfg->cfg_pool, description);
}
/* Now check for each proto separately */
if (new_map->protocol == MAP_PROTO_FILE) {
fdata =
rspamd_mempool_alloc0 (cfg->map_pool,
sizeof (struct file_map_data));
if (access (def, R_OK) == -1) {
if (errno != ENOENT) {
msg_err_config ("cannot open file '%s': %s", def, strerror
(errno));
return NULL;
}
msg_info_config (
"map '%s' is not found, but it can be loaded automatically later",
def);
/* We still can add this file */
fdata->st.st_mtime = -1;
}
else {
stat (def, &fdata->st);
}
fdata->filename = rspamd_mempool_strdup (cfg->map_pool, def);
new_map->map_data = fdata;
}
else if (new_map->protocol == MAP_PROTO_HTTP) {
hdata =
rspamd_mempool_alloc0 (cfg->map_pool,
sizeof (struct http_map_data));
memset (&up, 0, sizeof (up));
if (http_parser_parse_url (new_map->uri, strlen (new_map->uri), FALSE,
&up) != 0) {
msg_err_config ("cannot parse HTTP url: %s", new_map->uri);
return NULL;
}
else {
if (!(up.field_set & 1 << UF_HOST)) {
msg_err_config ("cannot parse HTTP url: %s: no host", new_map->uri);
return NULL;
}
tok.begin = new_map->uri + up.field_data[UF_HOST].off;
tok.len = up.field_data[UF_HOST].len;
hdata->host = rspamd_mempool_ftokdup (cfg->map_pool, &tok);
if (up.field_set & 1 << UF_PORT) {
hdata->port = up.port;
}
else {
hdata->port = 80;
}
if (up.field_set & 1 << UF_PATH) {
tok.begin = new_map->uri + up.field_data[UF_PATH].off;
tok.len = strlen (tok.begin);
hdata->path = rspamd_mempool_ftokdup (cfg->map_pool, &tok);
}
}
new_map->map_data = hdata;
}
/* Temp pool */
rspamd_cryptobox_hash (cksum, new_map->uri, strlen (new_map->uri), NULL, 0);
cksum_encoded = rspamd_encode_base32 (cksum, sizeof (cksum));
new_map->pool = rspamd_mempool_new (rspamd_mempool_suggest_size (), "map");
rspamd_strlcpy (new_map->pool->tag.uid, cksum_encoded,
sizeof (new_map->pool->tag.uid));
g_free (cksum_encoded);
pool = new_map->pool;
msg_info_pool ("added map %s", new_map->uri);
cfg->maps = g_list_prepend (cfg->maps, new_map);
return new_map;
}
static struct rspamd_osb_tokenizer_config *
rspamd_tokenizer_osb_config_from_ucl (rspamd_mempool_t * pool,
const ucl_object_t *obj)
{
const ucl_object_t *elt;
struct rspamd_osb_tokenizer_config *cf, *def;
guchar *key = NULL;
gsize keylen;
if (pool != NULL) {
cf = rspamd_mempool_alloc0 (pool, sizeof (*cf));
}
else {
cf = g_malloc0 (sizeof (*cf));
}
/* Use default config */
def = rspamd_tokenizer_osb_default_config ();
memcpy (cf, def, sizeof (*cf));
elt = ucl_object_lookup (obj, "hash");
if (elt != NULL && ucl_object_type (elt) == UCL_STRING) {
if (g_ascii_strncasecmp (ucl_object_tostring (elt), "xxh", 3)
== 0) {
cf->ht = RSPAMD_OSB_HASH_XXHASH;
elt = ucl_object_lookup (obj, "seed");
if (elt != NULL && ucl_object_type (elt) == UCL_INT) {
cf->seed = ucl_object_toint (elt);
}
}
else if (g_ascii_strncasecmp (ucl_object_tostring (elt), "sip", 3)
== 0) {
cf->ht = RSPAMD_OSB_HASH_SIPHASH;
elt = ucl_object_lookup (obj, "key");
if (elt != NULL && ucl_object_type (elt) == UCL_STRING) {
key = rspamd_decode_base32 (ucl_object_tostring (elt),
0, &keylen);
if (keylen < sizeof (rspamd_sipkey_t)) {
msg_warn ("siphash key is too short: %z", keylen);
g_free (key);
}
else {
memcpy (cf->sk, key, sizeof (cf->sk));
g_free (key);
}
}
else {
msg_warn_pool ("siphash cannot be used without key");
}
}
}
else {
elt = ucl_object_lookup (obj, "compat");
if (elt != NULL && ucl_object_toboolean (elt)) {
cf->ht = RSPAMD_OSB_HASH_COMPAT;
}
}
elt = ucl_object_lookup (obj, "window");
if (elt != NULL && ucl_object_type (elt) == UCL_INT) {
cf->window_size = ucl_object_toint (elt);
if (cf->window_size > DEFAULT_FEATURE_WINDOW_SIZE * 4) {
msg_err_pool ("too large window size: %d", cf->window_size);
cf->window_size = DEFAULT_FEATURE_WINDOW_SIZE;
}
}
return cf;
}
gint
rspamd_tokenizer_osb (struct rspamd_stat_ctx *ctx,
rspamd_mempool_t *pool,
GArray *words,
gboolean is_utf,
const gchar *prefix,
GPtrArray *result)
{
rspamd_token_t *new_tok = NULL;
rspamd_stat_token_t *token;
struct rspamd_osb_tokenizer_config *osb_cf;
guint64 cur, seed;
struct token_pipe_entry *hashpipe;
guint32 h1, h2;
gsize token_size;
guint processed = 0, i, w, window_size, token_flags = 0;
if (words == NULL) {
return FALSE;
}
osb_cf = ctx->tkcf;
window_size = osb_cf->window_size;
if (prefix) {
seed = rspamd_cryptobox_fast_hash_specific (RSPAMD_CRYPTOBOX_XXHASH64,
prefix, strlen (prefix), osb_cf->seed);
}
else {
seed = osb_cf->seed;
}
hashpipe = g_alloca (window_size * sizeof (hashpipe[0]));
for (i = 0; i < window_size; i++) {
hashpipe[i].h = 0xfe;
hashpipe[i].t = NULL;
}
token_size = sizeof (rspamd_token_t) +
sizeof (gdouble) * ctx->statfiles->len;
g_assert (token_size > 0);
for (w = 0; w < words->len; w ++) {
token = &g_array_index (words, rspamd_stat_token_t, w);
token_flags = token->flags;
if (osb_cf->ht == RSPAMD_OSB_HASH_COMPAT) {
rspamd_ftok_t ftok;
ftok.begin = token->begin;
ftok.len = token->len;
cur = rspamd_fstrhash_lc (&ftok, is_utf);
}
else {
/* We know that the words are normalized */
if (osb_cf->ht == RSPAMD_OSB_HASH_XXHASH) {
cur = rspamd_cryptobox_fast_hash_specific (RSPAMD_CRYPTOBOX_XXHASH64,
token->begin, token->len, osb_cf->seed);
}
else {
rspamd_cryptobox_siphash ((guchar *)&cur, token->begin,
token->len, osb_cf->sk);
if (prefix) {
cur ^= seed;
}
}
}
if (token_flags & RSPAMD_STAT_TOKEN_FLAG_UNIGRAM) {
new_tok = rspamd_mempool_alloc0 (pool, token_size);
new_tok->flags = token_flags;
new_tok->t1 = token;
new_tok->t2 = token;
new_tok->data = cur;
new_tok->window_idx = 0;
g_ptr_array_add (result, new_tok);
continue;
}
#define ADD_TOKEN do {\
new_tok = rspamd_mempool_alloc0 (pool, token_size); \
new_tok->flags = token_flags; \
new_tok->t1 = hashpipe[0].t; \
new_tok->t2 = hashpipe[i].t; \
if (osb_cf->ht == RSPAMD_OSB_HASH_COMPAT) { \
h1 = ((guint32)hashpipe[0].h) * primes[0] + \
((guint32)hashpipe[i].h) * primes[i << 1]; \
h2 = ((guint32)hashpipe[0].h) * primes[1] + \
((guint32)hashpipe[i].h) * primes[(i << 1) - 1]; \
memcpy((guchar *)&new_tok->data, &h1, sizeof (h1)); \
memcpy(((guchar *)&new_tok->data) + sizeof (h1), &h2, sizeof (h2)); \
} \
else { \
new_tok->data = hashpipe[0].h * primes[0] + hashpipe[i].h * primes[i << 1]; \
} \
new_tok->window_idx = i + 1; \
g_ptr_array_add (result, new_tok); \
} while(0)
if (processed < window_size) {
/* Just fill a hashpipe */
++processed;
hashpipe[window_size - processed].h = cur;
hashpipe[window_size - processed].t = token;
}
else {
/* Shift hashpipe */
for (i = window_size - 1; i > 0; i--) {
hashpipe[i] = hashpipe[i - 1];
}
hashpipe[0].h = cur;
hashpipe[0].t = token;
processed++;
for (i = 1; i < window_size; i++) {
ADD_TOKEN;
}
}
}
if (processed > 1 && processed <= window_size) {
processed --;
memmove (hashpipe, &hashpipe[window_size - processed],
processed * sizeof (hashpipe[0]));
for (i = 1; i < processed; i++) {
ADD_TOKEN;
}
}
#undef ADD_TOKEN
return TRUE;
}
static void
dkim_symbol_callback (struct rspamd_task *task, void *unused)
{
GList *hlist;
rspamd_dkim_context_t *ctx;
rspamd_dkim_key_t *key;
GError *err = NULL;
struct raw_header *rh;
struct dkim_check_result *res = NULL, *cur;
/* First check if a message has its signature */
hlist = rspamd_message_get_header (task,
DKIM_SIGNHEADER,
FALSE);
if (hlist != NULL) {
/* Check whitelist */
msg_debug ("dkim signature found");
if (radix_find_compressed_addr (dkim_module_ctx->whitelist_ip,
task->from_addr) == RADIX_NO_VALUE) {
/* Parse signature */
msg_debug ("create dkim signature");
while (hlist != NULL) {
rh = (struct raw_header *)hlist->data;
if (res == NULL) {
res = rspamd_mempool_alloc0 (task->task_pool, sizeof (*res));
res->prev = res;
res->w = rspamd_session_get_watcher (task->s);
cur = res;
}
else {
cur = rspamd_mempool_alloc0 (task->task_pool, sizeof (*res));
}
cur->first = res;
cur->res = -1;
cur->task = task;
cur->mult_allow = 1.0;
cur->mult_deny = 1.0;
ctx = rspamd_create_dkim_context (rh->decoded,
task->task_pool,
dkim_module_ctx->time_jitter,
&err);
if (ctx == NULL) {
if (err != NULL) {
msg_info ("<%s> cannot parse DKIM context: %s",
task->message_id, err->message);
g_error_free (err);
}
else {
msg_info ("<%s> cannot parse DKIM context: unknown error",
task->message_id);
}
hlist = g_list_next (hlist);
continue;
}
else {
/* Get key */
cur->ctx = ctx;
if (dkim_module_ctx->trusted_only &&
(dkim_module_ctx->dkim_domains == NULL ||
g_hash_table_lookup (dkim_module_ctx->dkim_domains,
ctx->domain) == NULL)) {
msg_debug ("skip dkim check for %s domain", ctx->domain);
hlist = g_list_next (hlist);
continue;
}
key = rspamd_lru_hash_lookup (dkim_module_ctx->dkim_hash,
ctx->dns_key,
task->tv.tv_sec);
if (key != NULL) {
debug_task ("found key for %s in cache", ctx->dns_key);
cur->key = key;
}
else {
debug_task ("request key for %s from DNS", ctx->dns_key);
rspamd_get_dkim_key (ctx,
task,
dkim_module_key_handler,
cur);
}
}
if (res != cur) {
DL_APPEND (res, cur);
}
hlist = g_list_next (hlist);
}
}
}
if (res != NULL) {
rspamd_session_watcher_push (task->s);
dkim_module_check (res);
}
}
static void
rspamd_archive_process_zip (struct rspamd_task *task,
struct rspamd_mime_part *part)
{
const guchar *p, *start, *end, *eocd = NULL, *cd;
const guint32 eocd_magic = 0x06054b50, cd_basic_len = 46;
const guchar cd_magic[] = {0x50, 0x4b, 0x01, 0x02};
guint32 cd_offset, cd_size, comp_size, uncomp_size;
guint16 extra_len, fname_len, comment_len;
struct rspamd_archive *arch;
struct rspamd_archive_file *f;
/* Zip files have interesting data at the end of archive */
p = part->parsed_data.begin + part->parsed_data.len - 1;
start = part->parsed_data.begin;
end = p;
/* Search for EOCD:
* 22 bytes is a typical size of eocd without a comment and
* end points one byte after the last character
*/
p -= 21;
while (p > start + sizeof (guint32)) {
guint32 t;
/* XXX: not an efficient approach */
memcpy (&t, p, sizeof (t));
if (GUINT32_FROM_LE (t) == eocd_magic) {
eocd = p;
break;
}
p --;
}
if (eocd == NULL) {
/* Not a zip file */
msg_debug_task ("zip archive is invalid (no EOCD)");
return;
}
if (end - eocd < 21) {
msg_debug_task ("zip archive is invalid (short EOCD)");
return;
}
memcpy (&cd_size, eocd + 12, sizeof (cd_size));
cd_size = GUINT32_FROM_LE (cd_size);
memcpy (&cd_offset, eocd + 16, sizeof (cd_offset));
cd_offset = GUINT32_FROM_LE (cd_offset);
/* We need to check sanity as well */
if (cd_offset + cd_size != (guint)(eocd - start)) {
msg_debug_task ("zip archive is invalid (bad size/offset for CD)");
return;
}
cd = start + cd_offset;
arch = rspamd_mempool_alloc0 (task->task_pool, sizeof (*arch));
arch->files = g_ptr_array_new ();
arch->type = RSPAMD_ARCHIVE_ZIP;
rspamd_mempool_add_destructor (task->task_pool, rspamd_archive_dtor,
arch);
while (cd < eocd) {
/* Read central directory record */
if (eocd - cd < cd_basic_len ||
memcmp (cd, cd_magic, sizeof (cd_magic)) != 0) {
msg_debug_task ("zip archive is invalid (bad cd record)");
return;
}
memcpy (&comp_size, cd + 20, sizeof (guint32));
comp_size = GUINT32_FROM_LE (comp_size);
memcpy (&uncomp_size, cd + 24, sizeof (guint32));
uncomp_size = GUINT32_FROM_LE (uncomp_size);
memcpy (&fname_len, cd + 28, sizeof (fname_len));
fname_len = GUINT16_FROM_LE (fname_len);
memcpy (&extra_len, cd + 30, sizeof (extra_len));
extra_len = GUINT16_FROM_LE (extra_len);
memcpy (&comment_len, cd + 32, sizeof (comment_len));
comment_len = GUINT16_FROM_LE (comment_len);
if (cd + fname_len + comment_len + extra_len + cd_basic_len > eocd) {
msg_debug_task ("zip archive is invalid (too large cd record)");
return;
}
f = g_slice_alloc0 (sizeof (*f));
f->fname = g_string_new_len (cd + cd_basic_len, fname_len);
f->compressed_size = comp_size;
f->uncompressed_size = uncomp_size;
g_ptr_array_add (arch->files, f);
msg_debug_task ("found file in zip archive: %v", f->fname);
cd += fname_len + comment_len + extra_len + cd_basic_len;
}
part->flags |= RSPAMD_MIME_PART_ARCHIVE;
part->specific.arch = arch;
if (part->cd) {
arch->archive_name = &part->cd->filename;
}
arch->size = part->parsed_data.len;
}
static void
rspamd_symbols_cache_validate_cb (gpointer k, gpointer v, gpointer ud)
{
struct cache_item *item = v, *parent;
struct symbols_cache *cache = (struct symbols_cache *)ud;
GList *cur;
struct metric *m;
struct rspamd_symbol_def *s;
gboolean skipped, ghost;
gint p1, p2;
ghost = item->weight == 0 ? TRUE : FALSE;
/* Check whether this item is skipped */
skipped = !ghost;
if (item->type == SYMBOL_TYPE_NORMAL &&
g_hash_table_lookup (cache->cfg->metrics_symbols, item->symbol) == NULL) {
cur = g_list_first (cache->cfg->metrics_list);
while (cur) {
m = cur->data;
if (m->accept_unknown_symbols) {
GList *mlist;
skipped = FALSE;
item->weight = item->weight * (m->unknown_weight);
s = rspamd_mempool_alloc0 (cache->static_pool,
sizeof (*s));
s->name = item->symbol;
s->weight_ptr = &item->weight;
g_hash_table_insert (m->symbols, item->symbol, s);
mlist = g_hash_table_lookup (cache->cfg->metrics_symbols,
item->symbol);
mlist = g_list_prepend (mlist, m);
g_hash_table_insert (cache->cfg->metrics_symbols,
item->symbol, mlist);
msg_info ("adding unknown symbol %s to metric %s", item->symbol,
m->name);
}
cur = g_list_next (cur);
}
}
else {
skipped = FALSE;
}
if (skipped) {
item->type = SYMBOL_TYPE_SKIPPED;
msg_warn ("symbol %s is not registered in any metric, so skip its check",
item->symbol);
}
if (ghost) {
msg_debug ("symbol %s is registered as ghost symbol, it won't be inserted "
"to any metric", item->symbol);
}
if (item->weight < 0 && item->priority == 0) {
item->priority ++;
}
if (item->type == SYMBOL_TYPE_VIRTUAL && item->parent != -1) {
g_assert (item->parent < (gint)cache->items_by_id->len);
parent = g_ptr_array_index (cache->items_by_id, item->parent);
if (fabs (parent->weight) < fabs (item->weight)) {
parent->weight = item->weight;
}
p1 = abs (item->priority);
p2 = abs (parent->priority);
if (p1 != p2) {
parent->priority = MAX (p1, p2);
item->priority = parent->priority;
}
}
if (fabs (item->weight) > cache->max_weight) {
cache->max_weight = fabs (item->weight);
}
}
gboolean
parse_smtp_command (struct smtp_session *session,
rspamd_ftok_t *line,
struct smtp_command **cmd)
{
enum {
SMTP_PARSE_START = 0,
SMTP_PARSE_SPACES,
SMTP_PARSE_ARGUMENT,
SMTP_PARSE_DONE
} state;
const gchar *p, *c;
gchar ch, cmd_buf[4];
guint i;
rspamd_ftok_t *arg = NULL;
struct smtp_command *pcmd;
if (line->len == 0) {
return FALSE;
}
state = SMTP_PARSE_START;
c = line->begin;
p = c;
*cmd = rspamd_mempool_alloc0 (session->pool, sizeof (struct smtp_command));
pcmd = *cmd;
for (i = 0; i < line->len; i++, p++) {
ch = *p;
switch (state) {
case SMTP_PARSE_START:
if (ch == ' ' || ch == ':' || ch == CR || ch == LF || i ==
line->len - 1) {
if (i == line->len - 1) {
p++;
}
if (p - c == 4) {
cmd_buf[0] = g_ascii_toupper (c[0]);
cmd_buf[1] = g_ascii_toupper (c[1]);
cmd_buf[2] = g_ascii_toupper (c[2]);
cmd_buf[3] = g_ascii_toupper (c[3]);
if (memcmp (cmd_buf, "HELO", 4) == 0) {
pcmd->command = SMTP_COMMAND_HELO;
}
else if (memcmp (cmd_buf, "EHLO", 4) == 0) {
pcmd->command = SMTP_COMMAND_EHLO;
}
else if (memcmp (cmd_buf, "MAIL", 4) == 0) {
pcmd->command = SMTP_COMMAND_MAIL;
}
else if (memcmp (cmd_buf, "RCPT", 4) == 0) {
pcmd->command = SMTP_COMMAND_RCPT;
}
else if (memcmp (cmd_buf, "DATA", 4) == 0) {
pcmd->command = SMTP_COMMAND_DATA;
}
else if (memcmp (cmd_buf, "QUIT", 4) == 0) {
pcmd->command = SMTP_COMMAND_QUIT;
}
else if (memcmp (cmd_buf, "NOOP", 4) == 0) {
pcmd->command = SMTP_COMMAND_NOOP;
}
else if (memcmp (cmd_buf, "EXPN", 4) == 0) {
pcmd->command = SMTP_COMMAND_EXPN;
}
else if (memcmp (cmd_buf, "RSET", 4) == 0) {
pcmd->command = SMTP_COMMAND_RSET;
}
else if (memcmp (cmd_buf, "HELP", 4) == 0) {
pcmd->command = SMTP_COMMAND_HELP;
}
else if (memcmp (cmd_buf, "VRFY", 4) == 0) {
pcmd->command = SMTP_COMMAND_VRFY;
}
else {
msg_info ("invalid command: %*s", 4, cmd_buf);
return FALSE;
}
}
else {
/* Invalid command */
msg_info ("invalid command: %*s", 4, c);
return FALSE;
}
/* Now check what we have */
if (ch == ' ' || ch == ':') {
state = SMTP_PARSE_SPACES;
}
else if (ch == CR) {
state = SMTP_PARSE_DONE;
}
else if (ch == LF) {
return TRUE;
}
}
else if ((ch < 'A' || ch > 'Z') && (ch < 'a' || ch > 'z')) {
msg_info ("invalid letter code in SMTP command: %d", (gint)ch);
return FALSE;
}
break;
case SMTP_PARSE_SPACES:
if (ch == CR) {
state = SMTP_PARSE_DONE;
}
else if (ch == LF) {
goto end;
}
else if (ch != ' ' && ch != ':') {
state = SMTP_PARSE_ARGUMENT;
arg = rspamd_mempool_alloc (session->pool, sizeof (rspamd_fstring_t));
c = p;
}
break;
case SMTP_PARSE_ARGUMENT:
if (ch == ' ' || ch == ':' || ch == CR || ch == LF || i ==
line->len - 1) {
if (i == line->len - 1 && (ch != ' ' && ch != CR && ch != LF)) {
p++;
}
arg->len = p - c;
arg->begin = rspamd_mempool_alloc (session->pool, arg->len);
memcpy ((gchar *)arg->begin, c, arg->len);
pcmd->args = g_list_prepend (pcmd->args, arg);
if (ch == ' ' || ch == ':') {
state = SMTP_PARSE_SPACES;
}
else if (ch == CR) {
state = SMTP_PARSE_DONE;
}
else {
goto end;
}
}
break;
case SMTP_PARSE_DONE:
if (ch == LF) {
goto end;
}
msg_info ("CR without LF in SMTP command");
return FALSE;
}
}
end:
if (pcmd->args) {
pcmd->args = g_list_reverse (pcmd->args);
rspamd_mempool_add_destructor (session->pool,
(rspamd_mempool_destruct_t)g_list_free,
pcmd->args);
}
return TRUE;
}
gboolean
rspamd_parse_bind_line (struct rspamd_config *cfg,
struct rspamd_worker_conf *cf,
const gchar *str)
{
struct rspamd_worker_bind_conf *cnf;
gchar **tokens, *err;
gboolean ret = TRUE;
if (str == NULL) {
return FALSE;
}
if (str[0] == '[') {
/* This is an ipv6 address */
gsize len, ntok;
const gchar *start, *ip_pos;
start = str + 1;
len = strcspn (start, "]");
if (start[len] != ']') {
return FALSE;
}
ip_pos = start;
start += len + 1;
ntok = 1;
if (*start == ':') {
ntok = 2;
start ++;
}
tokens = g_malloc_n (ntok + 1, sizeof (gchar *));
tokens[ntok] = NULL;
tokens[0] = g_malloc (len + 1);
rspamd_strlcpy (tokens[0], ip_pos, len + 1);
if (ntok > 1) {
tokens[1] = g_strdup (start);
}
}
else {
tokens = g_strsplit_set (str, ":", 0);
}
if (!tokens || !tokens[0]) {
return FALSE;
}
cnf =
rspamd_mempool_alloc0 (cfg->cfg_pool,
sizeof (struct rspamd_worker_bind_conf));
cnf->cnt = 1024;
if (strcmp (tokens[0], "systemd") == 0) {
/* The actual socket will be passed by systemd environment */
cnf->is_systemd = TRUE;
cnf->cnt = strtoul (tokens[1], &err, 10);
cnf->addrs = NULL;
if (err == NULL || *err == '\0') {
cnf->name = rspamd_mempool_strdup (cfg->cfg_pool, str);
LL_PREPEND (cf->bind_conf, cnf);
}
else {
msg_err_config ("cannot parse bind line: %s", str);
ret = FALSE;
}
}
else {
if (!rspamd_parse_host_port_priority_strv (tokens, &cnf->addrs,
NULL, &cnf->name, DEFAULT_BIND_PORT, cfg->cfg_pool)) {
msg_err_config ("cannot parse bind line: %s", str);
ret = FALSE;
}
else {
cnf->cnt = cnf->addrs->len;
LL_PREPEND (cf->bind_conf, cnf);
}
}
g_strfreev (tokens);
return ret;
}
static void
dkim_symbol_callback (struct rspamd_task *task, void *unused)
{
GList *hlist;
rspamd_dkim_context_t *ctx;
rspamd_dkim_key_t *key;
GError *err = NULL;
struct raw_header *rh;
struct dkim_check_result *res = NULL, *cur;
guint checked = 0;
/* First check if plugin should be enabled */
if (task->user != NULL || rspamd_inet_address_is_local (task->from_addr)) {
msg_info_task ("skip DKIM checks for local networks and authorized users");
return;
}
/* Check whitelist */
if (radix_find_compressed_addr (dkim_module_ctx->whitelist_ip,
task->from_addr) != RADIX_NO_VALUE) {
msg_info_task ("skip DKIM checks for whitelisted address");
return;
}
/* Now check if a message has its signature */
hlist = rspamd_message_get_header (task,
DKIM_SIGNHEADER,
FALSE);
if (hlist != NULL) {
msg_debug_task ("dkim signature found");
while (hlist != NULL) {
rh = (struct raw_header *)hlist->data;
if (rh->decoded == NULL || rh->decoded[0] == '\0') {
msg_info_task ("<%s> cannot load empty DKIM context",
task->message_id);
hlist = g_list_next (hlist);
continue;
}
if (res == NULL) {
res = rspamd_mempool_alloc0 (task->task_pool, sizeof (*res));
res->prev = res;
res->w = rspamd_session_get_watcher (task->s);
cur = res;
}
else {
cur = rspamd_mempool_alloc0 (task->task_pool, sizeof (*res));
}
cur->first = res;
cur->res = -1;
cur->task = task;
cur->mult_allow = 1.0;
cur->mult_deny = 1.0;
ctx = rspamd_create_dkim_context (rh->decoded,
task->task_pool,
dkim_module_ctx->time_jitter,
&err);
if (ctx == NULL) {
if (err != NULL) {
msg_info_task ("<%s> cannot parse DKIM context: %e",
task->message_id, err);
g_error_free (err);
err = NULL;
}
else {
msg_info_task ("<%s> cannot parse DKIM context: "
"unknown error",
task->message_id);
}
hlist = g_list_next (hlist);
continue;
}
else {
/* Get key */
cur->ctx = ctx;
if (dkim_module_ctx->trusted_only &&
(dkim_module_ctx->dkim_domains == NULL ||
g_hash_table_lookup (dkim_module_ctx->dkim_domains,
rspamd_dkim_get_domain (ctx)) == NULL)) {
msg_debug_task ("skip dkim check for %s domain",
rspamd_dkim_get_domain (ctx));
hlist = g_list_next (hlist);
continue;
}
key = rspamd_lru_hash_lookup (dkim_module_ctx->dkim_hash,
rspamd_dkim_get_dns_key (ctx),
task->tv.tv_sec);
if (key != NULL) {
cur->key = rspamd_dkim_key_ref (key);
/* Release key when task is processed */
rspamd_mempool_add_destructor (task->task_pool,
dkim_module_key_dtor, cur->key);
}
else {
rspamd_get_dkim_key (ctx,
task,
dkim_module_key_handler,
cur);
}
}
if (res != cur) {
DL_APPEND (res, cur);
}
if (dkim_module_ctx->skip_multi) {
if (hlist->next) {
msg_info_task ("message has multiple signatures but we"
" check only one as 'skip_multi' is set");
}
break;
}
checked ++;
if (checked > dkim_module_ctx->max_sigs) {
msg_info_task ("message has multiple signatures but we"
" stopped after %d checked signatures as limit"
" is reached", checked);
break;
}
hlist = g_list_next (hlist);
}
}
else {
rspamd_task_insert_result (task,
dkim_module_ctx->symbol_na,
1.0,
NULL);
}
if (res != NULL) {
rspamd_session_watcher_push (task->s);
dkim_module_check (res);
}
}
gint
regexp_module_config (struct rspamd_config *cfg)
{
struct regexp_module_item *cur_item;
const ucl_object_t *sec, *value, *elt;
ucl_object_iter_t it = NULL;
gint res = TRUE, id;
if (!rspamd_config_is_module_enabled (cfg, "regexp")) {
return TRUE;
}
sec = ucl_object_find_key (cfg->rcl_obj, "regexp");
if (sec == NULL) {
msg_err_config ("regexp module enabled, but no rules are defined");
return TRUE;
}
regexp_module_ctx->max_size = 0;
while ((value = ucl_iterate_object (sec, &it, true)) != NULL) {
if (g_ascii_strncasecmp (ucl_object_key (value), "max_size",
sizeof ("max_size") - 1) == 0) {
regexp_module_ctx->max_size = ucl_obj_toint (value);
rspamd_mime_expression_set_re_limit (regexp_module_ctx->max_size);
}
else if (g_ascii_strncasecmp (ucl_object_key (value), "max_threads",
sizeof ("max_threads") - 1) == 0) {
msg_warn_config ("regexp module is now single threaded, max_threads is ignored");
}
else if (value->type == UCL_STRING) {
cur_item = rspamd_mempool_alloc0 (regexp_module_ctx->regexp_pool,
sizeof (struct regexp_module_item));
cur_item->symbol = ucl_object_key (value);
if (!read_regexp_expression (regexp_module_ctx->regexp_pool,
cur_item, ucl_object_key (value),
ucl_obj_tostring (value), cfg)) {
res = FALSE;
}
else {
rspamd_symbols_cache_add_symbol (cfg->cache,
cur_item->symbol,
0,
process_regexp_item,
cur_item,
SYMBOL_TYPE_NORMAL, -1);
}
}
else if (value->type == UCL_USERDATA) {
/* Just a lua function */
cur_item = rspamd_mempool_alloc0 (regexp_module_ctx->regexp_pool,
sizeof (struct regexp_module_item));
cur_item->symbol = ucl_object_key (value);
cur_item->lua_function = ucl_object_toclosure (value);
rspamd_symbols_cache_add_symbol (cfg->cache,
cur_item->symbol,
0,
process_regexp_item,
cur_item,
SYMBOL_TYPE_NORMAL, -1);
}
else if (value->type == UCL_OBJECT) {
const gchar *description = NULL, *group = NULL,
*metric = DEFAULT_METRIC;
gdouble score = 0.0;
gboolean one_shot = FALSE, is_lua = FALSE, valid_expression = TRUE;
/* We have some lua table, extract its arguments */
elt = ucl_object_find_key (value, "callback");
if (elt == NULL || elt->type != UCL_USERDATA) {
/* Try plain regexp expression */
elt = ucl_object_find_any_key (value, "regexp", "re", NULL);
if (elt != NULL && ucl_object_type (elt) == UCL_STRING) {
cur_item = rspamd_mempool_alloc0 (regexp_module_ctx->regexp_pool,
sizeof (struct regexp_module_item));
cur_item->symbol = ucl_object_key (value);
if (!read_regexp_expression (regexp_module_ctx->regexp_pool,
cur_item, ucl_object_key (value),
ucl_obj_tostring (elt), cfg)) {
res = FALSE;
}
else {
valid_expression = TRUE;
}
}
else {
msg_err_config (
"no callback/expression defined for regexp symbol: "
"%s", ucl_object_key (value));
}
}
else {
is_lua = TRUE;
cur_item = rspamd_mempool_alloc0 (
regexp_module_ctx->regexp_pool,
sizeof (struct regexp_module_item));
cur_item->symbol = ucl_object_key (value);
cur_item->lua_function = ucl_object_toclosure (value);
}
if (cur_item && (is_lua || valid_expression)) {
id = rspamd_symbols_cache_add_symbol (cfg->cache,
cur_item->symbol,
0,
process_regexp_item,
cur_item,
SYMBOL_TYPE_NORMAL, -1);
elt = ucl_object_find_key (value, "condition");
if (elt != NULL && ucl_object_type (elt) == UCL_USERDATA) {
struct ucl_lua_funcdata *conddata;
conddata = ucl_object_toclosure (elt);
rspamd_symbols_cache_add_condition (cfg->cache, id,
conddata->L, conddata->idx);
}
elt = ucl_object_find_key (value, "metric");
if (elt) {
metric = ucl_object_tostring (elt);
}
elt = ucl_object_find_key (value, "description");
if (elt) {
description = ucl_object_tostring (elt);
}
elt = ucl_object_find_key (value, "group");
if (elt) {
group = ucl_object_tostring (elt);
}
elt = ucl_object_find_key (value, "score");
if (elt) {
score = ucl_object_todouble (elt);
}
elt = ucl_object_find_key (value, "one_shot");
if (elt) {
one_shot = ucl_object_toboolean (elt);
}
rspamd_config_add_metric_symbol (cfg, metric, cur_item->symbol,
score, description, group, one_shot, FALSE);
}
}
else {
msg_warn_config ("unknown type of attribute %s for regexp module",
ucl_object_key (value));
}
}
return res;
}