static struct cache_savepoint * rspamd_symbols_cache_make_checkpoint (struct rspamd_task *task, struct symbols_cache *cache) { struct cache_savepoint *checkpoint; if (cache->items_by_id->len != cache->items_by_order->d->len) { /* * Cache has been modified, need to resort it */ msg_info_cache ("symbols cache has been modified since last check:" " old items: %ud, new items: %ud", cache->items_by_order->d->len, cache->items_by_id->len); rspamd_symbols_cache_resort (cache); } checkpoint = rspamd_mempool_alloc0 (task->task_pool, sizeof (*checkpoint)); /* Bit 0: check started, Bit 1: check finished */ checkpoint->processed_bits = rspamd_mempool_alloc0 (task->task_pool, NBYTES (cache->used_items) * 2); checkpoint->waitq = g_ptr_array_new (); g_assert (cache->items_by_order != NULL); checkpoint->version = cache->items_by_order->d->len; checkpoint->order = cache->items_by_order; REF_RETAIN (checkpoint->order); rspamd_mempool_add_destructor (task->task_pool, rspamd_symbols_cache_order_unref, checkpoint->order); rspamd_mempool_add_destructor (task->task_pool, rspamd_ptr_array_free_hard, checkpoint->waitq); task->checkpoint = checkpoint; rspamd_create_metric_result (task, DEFAULT_METRIC); return checkpoint; }
static void rspamd_register_symbol_fromlua (lua_State *L, struct rspamd_config *cfg, const gchar *name, gint ref, gdouble weight, gint priority, enum rspamd_symbol_type type) { struct lua_callback_data *cd; cd = rspamd_mempool_alloc0 (cfg->cfg_pool, sizeof (struct lua_callback_data)); cd->cb_is_ref = TRUE; cd->callback.ref = ref; cd->L = L; if (name) { cd->symbol = rspamd_mempool_strdup (cfg->cfg_pool, name); } register_symbol_common (&cfg->cache, name, weight, priority, lua_metric_symbol_callback, cd, type); rspamd_mempool_add_destructor (cfg->cfg_pool, (rspamd_mempool_destruct_t)lua_destroy_cfg_symbol, cd); }
struct rspamd_worker_conf * rspamd_config_new_worker (struct rspamd_config *cfg, struct rspamd_worker_conf *c) { if (c == NULL) { c = rspamd_mempool_alloc0 (cfg->cfg_pool, sizeof (struct rspamd_worker_conf)); c->params = g_hash_table_new (rspamd_str_hash, rspamd_str_equal); c->active_workers = g_queue_new (); rspamd_mempool_add_destructor (cfg->cfg_pool, (rspamd_mempool_destruct_t)g_hash_table_destroy, c->params); rspamd_mempool_add_destructor (cfg->cfg_pool, (rspamd_mempool_destruct_t)g_queue_free, c->active_workers); #ifdef HAVE_SC_NPROCESSORS_ONLN c->count = sysconf (_SC_NPROCESSORS_ONLN); #else c->count = DEFAULT_WORKERS_NUM; #endif c->rlimit_nofile = 0; c->rlimit_maxcore = 0; } return c; }
static rspamd_expression_atom_t * lua_atom_parse (const gchar *line, gsize len, rspamd_mempool_t *pool, gpointer ud, GError **err) { struct lua_expression *e = (struct lua_expression *)ud; rspamd_expression_atom_t *atom; gsize rlen; const gchar *tok; lua_rawgeti (e->L, LUA_REGISTRYINDEX, e->parse_idx); lua_pushlstring (e->L, line, len); if (lua_pcall (e->L, 1, 1, 0) != 0) { msg_info ("callback call failed: %s", lua_tostring (e->L, -1)); } if (lua_type (e->L, -1) != LUA_TSTRING) { g_set_error (err, lua_expr_quark(), 500, "cannot parse lua atom"); lua_pop (e->L, 1); return NULL; } tok = lua_tolstring (e->L, -1, &rlen); atom = rspamd_mempool_alloc0 (e->pool, sizeof (*atom)); atom->str = rspamd_mempool_strdup (e->pool, tok); atom->len = rlen; atom->data = ud; lua_pop (e->L, 1); return atom; }
gpointer rspamd_redis_runtime (struct rspamd_task *task, struct rspamd_statfile_config *stcf, gboolean learn, gpointer c) { struct redis_stat_ctx *ctx = REDIS_CTX (c); struct redis_stat_runtime *rt; struct upstream *up; rspamd_inet_addr_t *addr; g_assert (ctx != NULL); g_assert (stcf != NULL); if (learn && ctx->write_servers == NULL) { msg_err_task ("no write servers defined for %s, cannot learn", stcf->symbol); return NULL; } if (learn) { up = rspamd_upstream_get (ctx->write_servers, RSPAMD_UPSTREAM_MASTER_SLAVE, NULL, 0); } else { up = rspamd_upstream_get (ctx->read_servers, RSPAMD_UPSTREAM_ROUND_ROBIN, NULL, 0); } if (up == NULL) { msg_err_task ("no upstreams reachable"); return NULL; } rt = rspamd_mempool_alloc0 (task->task_pool, sizeof (*rt)); rspamd_redis_expand_object (ctx->redis_object, ctx, task, &rt->redis_object_expanded); rt->selected = up; rt->task = task; rt->ctx = ctx; rt->stcf = stcf; addr = rspamd_upstream_addr (up); g_assert (addr != NULL); rt->redis = redisAsyncConnect (rspamd_inet_address_to_string (addr), rspamd_inet_address_get_port (addr)); if (rt->redis == NULL) { msg_err_task ("cannot connect redis"); return NULL; } redisLibeventAttach (rt->redis, task->ev_base); rspamd_redis_maybe_auth (ctx, rt->redis); return rt; }
struct rspamd_statfile_config * rspamd_config_new_statfile (struct rspamd_config *cfg, struct rspamd_statfile_config *c) { if (c == NULL) { c = rspamd_mempool_alloc0 (cfg->cfg_pool, sizeof (struct rspamd_statfile_config)); } return c; }
static struct fuzzy_rule * fuzzy_rule_new (const char *default_symbol, rspamd_mempool_t *pool) { struct fuzzy_rule *rule; rule = rspamd_mempool_alloc0 (pool, sizeof (struct fuzzy_rule)); rule->mappings = g_hash_table_new (g_direct_hash, g_direct_equal); rule->symbol = default_symbol; rspamd_mempool_add_destructor (pool, (rspamd_mempool_destruct_t)g_hash_table_unref, rule->mappings); rule->read_only = FALSE; return rule; }
struct rspamd_symbols_group * rspamd_config_new_group (struct rspamd_config *cfg, struct metric *metric, const gchar *name) { struct rspamd_symbols_group *gr; gr = rspamd_mempool_alloc0 (cfg->cfg_pool, sizeof (*gr)); gr->symbols = g_hash_table_new (rspamd_strcase_hash, rspamd_strcase_equal); rspamd_mempool_add_destructor (cfg->cfg_pool, (rspamd_mempool_destruct_t)g_hash_table_unref, gr->symbols); gr->name = rspamd_mempool_strdup (cfg->cfg_pool, name); g_hash_table_insert (metric->groups, gr->name, gr); return gr; }
gint lua_config_radix_from_config (lua_State *L) { struct rspamd_config *cfg = lua_check_config (L, 1); const gchar *mname, *optname; const ucl_object_t *obj; struct rspamd_lua_map *map, **pmap; if (!cfg) { return luaL_error (L, "invalid arguments"); } mname = luaL_checkstring (L, 2); optname = luaL_checkstring (L, 3); if (mname && optname) { obj = rspamd_config_get_module_opt (cfg, mname, optname); if (obj) { map = rspamd_mempool_alloc0 (cfg->cfg_pool, sizeof (*map)); map->data.radix = radix_create_compressed (); map->type = RSPAMD_LUA_MAP_RADIX; map->data.radix = radix_create_compressed (); map->flags |= RSPAMD_LUA_MAP_FLAG_EMBEDDED; radix_add_generic_iplist (ucl_obj_tostring (obj), &map->data.radix, TRUE); pmap = lua_newuserdata (L, sizeof (void *)); *pmap = map; rspamd_lua_setclass (L, "rspamd{map}", -1); } else { msg_warn_config ("Couldnt find config option [%s][%s]", mname, optname); lua_pushnil (L); } } else { return luaL_error (L, "invalid arguments"); } return 1; }
struct rspamd_classifier_config * rspamd_config_new_classifier (struct rspamd_config *cfg, struct rspamd_classifier_config *c) { if (c == NULL) { c = rspamd_mempool_alloc0 (cfg->cfg_pool, sizeof (struct rspamd_classifier_config)); } if (c->labels == NULL) { c->labels = g_hash_table_new_full (rspamd_str_hash, rspamd_str_equal, NULL, (GDestroyNotify)g_list_free); rspamd_mempool_add_destructor (cfg->cfg_pool, (rspamd_mempool_destruct_t) g_hash_table_destroy, c->labels); } return c; }
static void rspamd_html_url_is_phished (rspamd_mempool_t *pool, struct rspamd_url *href_url, const guchar *url_text, gsize len, gboolean *url_found) { struct rspamd_url *text_url; gint rc; gchar *url_str = NULL; *url_found = FALSE; if (rspamd_url_find (pool, url_text, len, &url_str, TRUE) && url_str != NULL) { text_url = rspamd_mempool_alloc0 (pool, sizeof (struct rspamd_url)); rc = rspamd_url_parse (text_url, url_str, strlen (url_str), pool); if (rc == URI_ERRNO_OK) { if (href_url->hostlen != text_url->hostlen || memcmp (href_url->host, text_url->host, href_url->hostlen) != 0) { if (href_url->tldlen != text_url->tldlen || memcmp (href_url->tld, text_url->tld, href_url->tldlen) != 0) { href_url->flags |= RSPAMD_URL_FLAG_PHISHED; href_url->phished_url = text_url; } } *url_found = TRUE; } else { msg_info_pool ("extract of url '%s' failed: %s", url_str, rspamd_url_strerror (rc)); } } }
gint lua_config_add_hash_map (lua_State *L) { struct rspamd_config *cfg = lua_check_config (L, 1); const gchar *map_line, *description; struct rspamd_lua_map *map, **pmap; struct rspamd_map *m; if (cfg) { map_line = luaL_checkstring (L, 2); description = lua_tostring (L, 3); map = rspamd_mempool_alloc0 (cfg->cfg_pool, sizeof (*map)); map->data.hash = g_hash_table_new (rspamd_strcase_hash, rspamd_strcase_equal); map->type = RSPAMD_LUA_MAP_SET; if ((m = rspamd_map_add (cfg, map_line, description, rspamd_hosts_read, rspamd_hosts_fin, (void **)&map->data.hash)) == NULL) { msg_warn_config ("invalid set map %s", map_line); g_hash_table_destroy (map->data.hash); lua_pushnil (L); return 1; } map->map = m; pmap = lua_newuserdata (L, sizeof (void *)); *pmap = map; rspamd_lua_setclass (L, "rspamd{map}", -1); } else { return luaL_error (L, "invalid arguments"); } return 1; }
gint lua_config_add_radix_map (lua_State *L) { struct rspamd_config *cfg = lua_check_config (L, 1); const gchar *map_line, *description; struct rspamd_lua_map *map, **pmap; struct rspamd_map *m; if (cfg) { map_line = luaL_checkstring (L, 2); description = lua_tostring (L, 3); map = rspamd_mempool_alloc0 (cfg->cfg_pool, sizeof (*map)); map->data.radix = radix_create_compressed (); map->type = RSPAMD_LUA_MAP_RADIX; if ((m = rspamd_map_add (cfg, map_line, description, rspamd_radix_read, rspamd_radix_fin, (void **)&map->data.radix)) == NULL) { msg_warn_config ("invalid radix map %s", map_line); radix_destroy_compressed (map->data.radix); lua_pushnil (L); return 1; } map->map = m; pmap = lua_newuserdata (L, sizeof (void *)); *pmap = map; rspamd_lua_setclass (L, "rspamd{map}", -1); } else { return luaL_error (L, "invalid arguments"); } return 1; }
struct metric * rspamd_config_new_metric (struct rspamd_config *cfg, struct metric *c, const gchar *name) { int i; if (c == NULL) { c = rspamd_mempool_alloc0 (cfg->cfg_pool, sizeof (struct metric)); c->grow_factor = 1.0; c->symbols = g_hash_table_new (rspamd_str_hash, rspamd_str_equal); c->groups = g_hash_table_new (rspamd_str_hash, rspamd_str_equal); for (i = METRIC_ACTION_REJECT; i < METRIC_ACTION_MAX; i++) { c->actions[i].score = -1.0; c->actions[i].action = i; } c->subject = SPAM_SUBJECT; c->name = rspamd_mempool_strdup (cfg->cfg_pool, name); rspamd_mempool_add_destructor (cfg->cfg_pool, (rspamd_mempool_destruct_t) g_hash_table_unref, c->symbols); rspamd_mempool_add_destructor (cfg->cfg_pool, (rspamd_mempool_destruct_t) g_hash_table_unref, c->groups); g_hash_table_insert (cfg->metrics, (void *)c->name, c); cfg->metrics_list = g_list_prepend (cfg->metrics_list, c); if (strcmp (c->name, DEFAULT_METRIC) == 0) { cfg->default_metric = c; } } return c; }
static void rspamd_archive_process_rar (struct rspamd_task *task, struct rspamd_mime_part *part) { const guchar *p, *end, *section_start; const guchar rar_v5_magic[] = {0x52, 0x61, 0x72, 0x21, 0x1A, 0x07, 0x01, 0x00}, rar_v4_magic[] = {0x52, 0x61, 0x72, 0x21, 0x1A, 0x07, 0x00}; const guint rar_encrypted_header = 4, rar_main_header = 1, rar_file_header = 2; guint64 vint, sz, comp_sz = 0, uncomp_sz = 0, flags = 0, type = 0; struct rspamd_archive *arch; struct rspamd_archive_file *f; gint r; p = part->parsed_data.begin; end = p + part->parsed_data.len; if ((gsize)(end - p) <= sizeof (rar_v5_magic)) { msg_debug_task ("rar archive is invalid (too small)"); return; } if (memcmp (p, rar_v5_magic, sizeof (rar_v5_magic)) == 0) { p += sizeof (rar_v5_magic); } else if (memcmp (p, rar_v4_magic, sizeof (rar_v4_magic)) == 0) { p += sizeof (rar_v4_magic); rspamd_archive_process_rar_v4 (task, p, end, part); return; } else { msg_debug_task ("rar archive is invalid (no rar magic)"); return; } /* Rar v5 format */ arch = rspamd_mempool_alloc0 (task->task_pool, sizeof (*arch)); arch->files = g_ptr_array_new (); arch->type = RSPAMD_ARCHIVE_RAR; rspamd_mempool_add_destructor (task->task_pool, rspamd_archive_dtor, arch); /* Now we can have either encryption header or archive header */ /* Crc 32 */ RAR_SKIP_BYTES (sizeof (guint32)); /* Size */ RAR_READ_VINT_SKIP (); sz = vint; /* Type */ section_start = p; RAR_READ_VINT_SKIP (); type = vint; /* Header flags */ RAR_READ_VINT_SKIP (); flags = vint; if (flags & 0x1) { /* Have extra zone */ RAR_READ_VINT_SKIP (); } if (flags & 0x2) { /* Data zone is presented */ RAR_READ_VINT_SKIP (); sz += vint; } if (type == rar_encrypted_header) { /* We can't read any further information as archive is encrypted */ arch->flags |= RSPAMD_ARCHIVE_ENCRYPTED; goto end; } else if (type != rar_main_header) { msg_debug_task ("rar archive is invalid (bad main header)"); return; } /* Nothing useful in main header */ p = section_start; RAR_SKIP_BYTES (sz); while (p < end) { /* Read the next header */ /* Crc 32 */ RAR_SKIP_BYTES (sizeof (guint32)); /* Size */ RAR_READ_VINT_SKIP (); sz = vint; if (sz == 0) { /* Zero sized block - error */ msg_debug_task ("rar archive is invalid (zero size block)"); return; } section_start = p; /* Type */ RAR_READ_VINT_SKIP (); type = vint; /* Header flags */ RAR_READ_VINT_SKIP (); flags = vint; if (flags & 0x1) { /* Have extra zone */ RAR_READ_VINT_SKIP (); } if (flags & 0x2) { /* Data zone is presented */ RAR_READ_VINT_SKIP (); sz += vint; comp_sz = vint; } if (type != rar_file_header) { p = section_start; RAR_SKIP_BYTES (sz); } else { /* We have a file header, go forward */ guint64 fname_len; /* File header specific flags */ RAR_READ_VINT_SKIP (); flags = vint; /* Unpacked size */ RAR_READ_VINT_SKIP (); uncomp_sz = vint; /* Attributes */ RAR_READ_VINT_SKIP (); if (flags & 0x2) { /* Unix mtime */ RAR_SKIP_BYTES (sizeof (guint32)); } if (flags & 0x4) { /* Crc32 */ RAR_SKIP_BYTES (sizeof (guint32)); } /* Compression */ RAR_READ_VINT_SKIP (); /* Host OS */ RAR_READ_VINT_SKIP (); /* Filename length (finally!) */ RAR_READ_VINT_SKIP (); fname_len = vint; if (fname_len == 0 || fname_len > (gsize)(end - p)) { msg_debug_task ("rar archive is invalid (bad fileame size)"); return; } f = g_slice_alloc0 (sizeof (*f)); f->uncompressed_size = uncomp_sz; f->compressed_size = comp_sz; f->fname = g_string_new_len (p, fname_len); g_ptr_array_add (arch->files, f); /* Restore p to the beginning of the header */ p = section_start; RAR_SKIP_BYTES (sz); } } end: part->flags |= RSPAMD_MIME_PART_ARCHIVE; part->specific.arch = arch; if (part->cd != NULL) { arch->archive_name = &part->cd->filename; } arch->size = part->parsed_data.len; }
static void rspamd_archive_process_rar_v4 (struct rspamd_task *task, const guchar *start, const guchar *end, struct rspamd_mime_part *part) { const guchar *p = start, *start_section; guint8 type; guint flags; guint64 sz, comp_sz = 0, uncomp_sz = 0; struct rspamd_archive *arch; struct rspamd_archive_file *f; arch = rspamd_mempool_alloc0 (task->task_pool, sizeof (*arch)); arch->files = g_ptr_array_new (); arch->type = RSPAMD_ARCHIVE_RAR; rspamd_mempool_add_destructor (task->task_pool, rspamd_archive_dtor, arch); while (p < end) { /* Crc16 */ start_section = p; RAR_SKIP_BYTES (sizeof (guint16)); type = *p; p ++; RAR_READ_UINT16 (flags); if (type == 0x73) { /* Main header, check for encryption */ if (flags & 0x80) { arch->flags |= RSPAMD_ARCHIVE_ENCRYPTED; goto end; } } RAR_READ_UINT16 (sz); if (flags & 0x8000) { /* We also need to read ADD_SIZE element */ guint32 tmp; RAR_READ_UINT32 (tmp); sz += tmp; /* This is also used as PACK_SIZE */ comp_sz = tmp; } if (sz == 0) { /* Zero sized block - error */ msg_debug_task ("rar archive is invalid (zero size block)"); return; } if (type == 0x74) { guint fname_len; /* File header */ /* Uncompressed size */ RAR_READ_UINT32 (uncomp_sz); /* Skip to NAME_SIZE element */ RAR_SKIP_BYTES (11); RAR_READ_UINT16 (fname_len); if (fname_len == 0 || fname_len > (gsize)(end - p)) { msg_debug_task ("rar archive is invalid (bad fileame size)"); return; } /* Attrs */ RAR_SKIP_BYTES (4); if (flags & 0x100) { /* We also need to read HIGH_PACK_SIZE */ guint32 tmp; RAR_READ_UINT32 (tmp); sz += tmp; comp_sz += tmp; /* HIGH_UNP_SIZE */ RAR_READ_UINT32 (tmp); uncomp_sz += tmp; } f = g_slice_alloc0 (sizeof (*f)); if (flags & 0x200) { /* We have unicode + normal version */ guchar *tmp; tmp = memchr (p, '\0', fname_len); if (tmp != NULL) { /* Just use ASCII version */ f->fname = g_string_new_len (p, tmp - p); } else { /* We have UTF8 filename, use it as is */ f->fname = g_string_new_len (p, fname_len); } } else { f->fname = g_string_new_len (p, fname_len); } f->compressed_size = comp_sz; f->uncompressed_size = uncomp_sz; if (flags & 0x4) { f->flags |= RSPAMD_ARCHIVE_FILE_ENCRYPTED; } g_ptr_array_add (arch->files, f); } p = start_section; RAR_SKIP_BYTES (sz); } end: part->flags |= RSPAMD_MIME_PART_ARCHIVE; part->specific.arch = arch; arch->archive_name = &part->cd->filename; arch->size = part->parsed_data.len; }
gboolean rspamd_map_add (struct rspamd_config *cfg, const gchar *map_line, const gchar *description, map_cb_t read_callback, map_fin_cb_t fin_callback, void **user_data) { struct rspamd_map *new_map; enum fetch_proto proto; const gchar *def, *p, *hostend; struct file_map_data *fdata; struct http_map_data *hdata; gchar portbuf[6], *cksum_encoded, cksum[BLAKE2B_OUTBYTES]; gint i, s, r; struct addrinfo hints, *res; rspamd_mempool_t *pool; /* First of all detect protocol line */ if (!rspamd_map_check_proto (map_line, (int *)&proto, &def)) { return FALSE; } /* Constant pool */ if (cfg->map_pool == NULL) { cfg->map_pool = rspamd_mempool_new (rspamd_mempool_suggest_size (), "map"); memcpy (cfg->map_pool->tag.uid, cfg->cfg_pool->tag.uid, sizeof (cfg->map_pool->tag.uid)); } new_map = rspamd_mempool_alloc0 (cfg->map_pool, sizeof (struct rspamd_map)); new_map->read_callback = read_callback; new_map->fin_callback = fin_callback; new_map->user_data = user_data; new_map->protocol = proto; new_map->cfg = cfg; new_map->id = g_random_int (); new_map->locked = rspamd_mempool_alloc0_shared (cfg->cfg_pool, sizeof (gint)); if (proto == MAP_PROTO_FILE) { new_map->uri = rspamd_mempool_strdup (cfg->cfg_pool, def); def = new_map->uri; } else { new_map->uri = rspamd_mempool_strdup (cfg->cfg_pool, map_line); } if (description != NULL) { new_map->description = rspamd_mempool_strdup (cfg->cfg_pool, description); } /* Now check for each proto separately */ if (proto == MAP_PROTO_FILE) { fdata = rspamd_mempool_alloc0 (cfg->map_pool, sizeof (struct file_map_data)); if (access (def, R_OK) == -1) { if (errno != ENOENT) { msg_err_config ("cannot open file '%s': %s", def, strerror (errno)); return FALSE; } msg_info_config ( "map '%s' is not found, but it can be loaded automatically later", def); /* We still can add this file */ fdata->st.st_mtime = -1; } else { stat (def, &fdata->st); } fdata->filename = rspamd_mempool_strdup (cfg->map_pool, def); new_map->map_data = fdata; } else if (proto == MAP_PROTO_HTTP) { hdata = rspamd_mempool_alloc0 (cfg->map_pool, sizeof (struct http_map_data)); /* Try to search port */ if ((p = strchr (def, ':')) != NULL) { hostend = p; i = 0; p++; while (g_ascii_isdigit (*p) && i < (gint)sizeof (portbuf) - 1) { portbuf[i++] = *p++; } if (*p != '/') { msg_info_config ("bad http map definition: %s", def); return FALSE; } portbuf[i] = '\0'; hdata->port = atoi (portbuf); } else { /* Default http port */ rspamd_snprintf (portbuf, sizeof (portbuf), "80"); hdata->port = 80; /* Now separate host from path */ if ((p = strchr (def, '/')) == NULL) { msg_info_config ("bad http map definition: %s", def); return FALSE; } hostend = p; } hdata->host = rspamd_mempool_alloc (cfg->map_pool, hostend - def + 1); rspamd_strlcpy (hdata->host, def, hostend - def + 1); hdata->path = rspamd_mempool_strdup (cfg->map_pool, p); /* Now try to resolve */ memset (&hints, 0, sizeof (hints)); hints.ai_family = AF_UNSPEC; /* Allow IPv4 or IPv6 */ hints.ai_socktype = SOCK_STREAM; /* Stream socket */ hints.ai_flags = 0; hints.ai_protocol = 0; /* Any protocol */ hints.ai_canonname = NULL; hints.ai_addr = NULL; hints.ai_next = NULL; if ((r = getaddrinfo (hdata->host, portbuf, &hints, &res)) == 0) { hdata->addr = res; rspamd_mempool_add_destructor (cfg->cfg_pool, (rspamd_mempool_destruct_t)freeaddrinfo, hdata->addr); } else { msg_err_config ("address resolution for %s failed: %s", hdata->host, gai_strerror (r)); return FALSE; } /* Now try to connect */ if ((s = rspamd_socket_tcp (hdata->addr, FALSE, FALSE)) == -1) { msg_info_config ("cannot connect to http server %s: %d, %s", hdata->host, errno, strerror (errno)); return FALSE; } close (s); hdata->conn = rspamd_http_connection_new (http_map_read, http_map_error, http_map_finish, RSPAMD_HTTP_BODY_PARTIAL | RSPAMD_HTTP_CLIENT_SIMPLE, RSPAMD_HTTP_CLIENT, NULL); new_map->map_data = hdata; } /* Temp pool */ blake2b (cksum, new_map->uri, NULL, sizeof (cksum), strlen (new_map->uri), 0); cksum_encoded = rspamd_encode_base32 (cksum, sizeof (cksum)); new_map->pool = rspamd_mempool_new (rspamd_mempool_suggest_size (), "map"); memcpy (new_map->pool->tag.uid, cksum_encoded, sizeof (new_map->pool->tag.uid)); g_free (cksum_encoded); pool = new_map->pool; msg_info_pool ("added map %s", new_map->uri); cfg->maps = g_list_prepend (cfg->maps, new_map); return TRUE; }
gint rspamd_symbols_cache_add_symbol (struct symbols_cache *cache, const gchar *name, double weight, gint priority, symbol_func_t func, gpointer user_data, enum rspamd_symbol_type type, gint parent) { struct cache_item *item = NULL; g_assert (cache != NULL); if (name == NULL && type != SYMBOL_TYPE_CALLBACK) { msg_warn ("no name for non-callback symbol!"); } else if (type == SYMBOL_TYPE_VIRTUAL && parent == -1) { msg_warn ("no parent symbol is associated with virtual symbol %s", name); } if (name != NULL) { if (g_hash_table_lookup (cache->items_by_symbol, name) != NULL) { msg_err ("skip duplicate symbol registration for %s", name); return -1; } } item = rspamd_mempool_alloc0_shared (cache->static_pool, sizeof (struct cache_item)); /* * We do not share cd to skip locking, instead we'll just calculate it on * save or accumulate */ item->cd = rspamd_mempool_alloc0 (cache->static_pool, sizeof (struct counter_data)); if (name != NULL) { item->symbol = rspamd_mempool_strdup (cache->static_pool, name); } item->func = func; item->user_data = user_data; item->priority = priority; item->type = type; item->weight = weight; if (item->weight < 0 && item->priority == 0) { /* Make priority for negative weighted symbols */ item->priority = 1; } item->id = cache->used_items; item->parent = parent; cache->used_items ++; msg_debug ("used items: %d, added symbol: %s", cache->used_items, name); rspamd_set_counter (item, 0); g_ptr_array_add (cache->items_by_id, item); g_ptr_array_add (cache->items_by_order, item); item->deps = g_ptr_array_new (); item->rdeps = g_ptr_array_new (); rspamd_mempool_add_destructor (cache->static_pool, rspamd_ptr_array_free_hard, item->deps); rspamd_mempool_add_destructor (cache->static_pool, rspamd_ptr_array_free_hard, item->rdeps); if (name != NULL) { g_hash_table_insert (cache->items_by_symbol, item->symbol, item); } return item->id; }
gboolean rspamd_config_add_metric_symbol (struct rspamd_config *cfg, const gchar *metric_name, const gchar *symbol, gdouble score, const gchar *description, const gchar *group, gboolean one_shot, gboolean rewrite_existing) { struct rspamd_symbols_group *sym_group; struct rspamd_symbol_def *sym_def; GList *metric_list; struct metric *metric; gdouble *score_ptr; g_assert (cfg != NULL); g_assert (symbol != NULL); if (metric_name == NULL) { metric_name = DEFAULT_METRIC; } metric = g_hash_table_lookup (cfg->metrics, metric_name); if (metric == NULL) { msg_err_config ("metric %s has not been found", metric_name); return FALSE; } if (g_hash_table_lookup (cfg->metrics_symbols, symbol) != NULL && !rewrite_existing) { msg_debug_config ("symbol %s has been already registered, do not override"); return FALSE; } sym_def = rspamd_mempool_alloc0 (cfg->cfg_pool, sizeof (struct rspamd_symbol_def)); score_ptr = rspamd_mempool_alloc (cfg->cfg_pool, sizeof (gdouble)); *score_ptr = score; sym_def->score = score; sym_def->weight_ptr = score_ptr; sym_def->name = rspamd_mempool_strdup (cfg->cfg_pool, symbol); sym_def->one_shot = one_shot; if (description) { sym_def->description = rspamd_mempool_strdup (cfg->cfg_pool, description); } msg_debug_config ("registered symbol %s with weight %.2f in metric %s and group %s", sym_def->name, score, metric->name, group); g_hash_table_insert (metric->symbols, sym_def->name, sym_def); if ((metric_list = g_hash_table_lookup (cfg->metrics_symbols, sym_def->name)) == NULL) { metric_list = g_list_prepend (NULL, metric); rspamd_mempool_add_destructor (cfg->cfg_pool, (rspamd_mempool_destruct_t)g_list_free, metric_list); g_hash_table_insert (cfg->metrics_symbols, sym_def->name, metric_list); } else { /* Slow but keep start element of list in safe */ if (!g_list_find (metric_list, metric)) { metric_list = g_list_append (metric_list, metric); } } /* Search for symbol group */ if (group == NULL) { group = "ungrouped"; } sym_group = g_hash_table_lookup (metric->groups, group); if (sym_group == NULL) { /* Create new group */ sym_group = rspamd_config_new_group (cfg, metric, group); } sym_def->gr = sym_group; g_hash_table_insert (sym_group->symbols, sym_def->name, sym_def); return TRUE; }
gpointer rspamd_stat_cache_redis_runtime (struct rspamd_task *task, gpointer c, gboolean learn) { struct rspamd_redis_cache_ctx *ctx = c; struct rspamd_redis_cache_runtime *rt; struct upstream *up; rspamd_inet_addr_t *addr; g_assert (ctx != NULL); if (learn && ctx->write_servers == NULL) { msg_err_task ("no write servers defined for %s, cannot learn", ctx->stcf->symbol); return NULL; } if (task->tokens == NULL || task->tokens->len == 0) { return NULL; } if (learn) { up = rspamd_upstream_get (ctx->write_servers, RSPAMD_UPSTREAM_MASTER_SLAVE, NULL, 0); } else { up = rspamd_upstream_get (ctx->read_servers, RSPAMD_UPSTREAM_ROUND_ROBIN, NULL, 0); } if (up == NULL) { msg_err_task ("no upstreams reachable"); return NULL; } rt = rspamd_mempool_alloc0 (task->task_pool, sizeof (*rt)); rt->selected = up; rt->task = task; rt->ctx = ctx; addr = rspamd_upstream_addr (up); g_assert (addr != NULL); rt->redis = redisAsyncConnect (rspamd_inet_address_to_string (addr), rspamd_inet_address_get_port (addr)); g_assert (rt->redis != NULL); redisLibeventAttach (rt->redis, task->ev_base); /* Now check stats */ event_set (&rt->timeout_event, -1, EV_TIMEOUT, rspamd_redis_cache_timeout, rt); event_base_set (task->ev_base, &rt->timeout_event); rspamd_redis_cache_maybe_auth (ctx, rt->redis); if (!learn) { rspamd_stat_cache_redis_generate_id (task); } return rt; }
struct rspamd_map * rspamd_map_add (struct rspamd_config *cfg, const gchar *map_line, const gchar *description, map_cb_t read_callback, map_fin_cb_t fin_callback, void **user_data) { struct rspamd_map *new_map; const gchar *def; struct file_map_data *fdata; struct http_map_data *hdata; gchar *cksum_encoded, cksum[rspamd_cryptobox_HASHBYTES]; rspamd_mempool_t *pool; struct http_parser_url up; rspamd_ftok_t tok; if (cfg->map_pool == NULL) { cfg->map_pool = rspamd_mempool_new (rspamd_mempool_suggest_size (), "map"); memcpy (cfg->map_pool->tag.uid, cfg->cfg_pool->tag.uid, sizeof (cfg->map_pool->tag.uid)); } new_map = rspamd_mempool_alloc0 (cfg->map_pool, sizeof (struct rspamd_map)); /* First of all detect protocol line */ if (rspamd_map_check_proto (cfg, map_line, new_map) == NULL) { return NULL; } new_map->read_callback = read_callback; new_map->fin_callback = fin_callback; new_map->user_data = user_data; new_map->cfg = cfg; new_map->id = g_random_int (); new_map->locked = rspamd_mempool_alloc0_shared (cfg->cfg_pool, sizeof (gint)); def = new_map->uri; if (description != NULL) { new_map->description = rspamd_mempool_strdup (cfg->cfg_pool, description); } /* Now check for each proto separately */ if (new_map->protocol == MAP_PROTO_FILE) { fdata = rspamd_mempool_alloc0 (cfg->map_pool, sizeof (struct file_map_data)); if (access (def, R_OK) == -1) { if (errno != ENOENT) { msg_err_config ("cannot open file '%s': %s", def, strerror (errno)); return NULL; } msg_info_config ( "map '%s' is not found, but it can be loaded automatically later", def); /* We still can add this file */ fdata->st.st_mtime = -1; } else { stat (def, &fdata->st); } fdata->filename = rspamd_mempool_strdup (cfg->map_pool, def); new_map->map_data = fdata; } else if (new_map->protocol == MAP_PROTO_HTTP) { hdata = rspamd_mempool_alloc0 (cfg->map_pool, sizeof (struct http_map_data)); memset (&up, 0, sizeof (up)); if (http_parser_parse_url (new_map->uri, strlen (new_map->uri), FALSE, &up) != 0) { msg_err_config ("cannot parse HTTP url: %s", new_map->uri); return NULL; } else { if (!(up.field_set & 1 << UF_HOST)) { msg_err_config ("cannot parse HTTP url: %s: no host", new_map->uri); return NULL; } tok.begin = new_map->uri + up.field_data[UF_HOST].off; tok.len = up.field_data[UF_HOST].len; hdata->host = rspamd_mempool_ftokdup (cfg->map_pool, &tok); if (up.field_set & 1 << UF_PORT) { hdata->port = up.port; } else { hdata->port = 80; } if (up.field_set & 1 << UF_PATH) { tok.begin = new_map->uri + up.field_data[UF_PATH].off; tok.len = strlen (tok.begin); hdata->path = rspamd_mempool_ftokdup (cfg->map_pool, &tok); } } new_map->map_data = hdata; } /* Temp pool */ rspamd_cryptobox_hash (cksum, new_map->uri, strlen (new_map->uri), NULL, 0); cksum_encoded = rspamd_encode_base32 (cksum, sizeof (cksum)); new_map->pool = rspamd_mempool_new (rspamd_mempool_suggest_size (), "map"); rspamd_strlcpy (new_map->pool->tag.uid, cksum_encoded, sizeof (new_map->pool->tag.uid)); g_free (cksum_encoded); pool = new_map->pool; msg_info_pool ("added map %s", new_map->uri); cfg->maps = g_list_prepend (cfg->maps, new_map); return new_map; }
static struct rspamd_osb_tokenizer_config * rspamd_tokenizer_osb_config_from_ucl (rspamd_mempool_t * pool, const ucl_object_t *obj) { const ucl_object_t *elt; struct rspamd_osb_tokenizer_config *cf, *def; guchar *key = NULL; gsize keylen; if (pool != NULL) { cf = rspamd_mempool_alloc0 (pool, sizeof (*cf)); } else { cf = g_malloc0 (sizeof (*cf)); } /* Use default config */ def = rspamd_tokenizer_osb_default_config (); memcpy (cf, def, sizeof (*cf)); elt = ucl_object_lookup (obj, "hash"); if (elt != NULL && ucl_object_type (elt) == UCL_STRING) { if (g_ascii_strncasecmp (ucl_object_tostring (elt), "xxh", 3) == 0) { cf->ht = RSPAMD_OSB_HASH_XXHASH; elt = ucl_object_lookup (obj, "seed"); if (elt != NULL && ucl_object_type (elt) == UCL_INT) { cf->seed = ucl_object_toint (elt); } } else if (g_ascii_strncasecmp (ucl_object_tostring (elt), "sip", 3) == 0) { cf->ht = RSPAMD_OSB_HASH_SIPHASH; elt = ucl_object_lookup (obj, "key"); if (elt != NULL && ucl_object_type (elt) == UCL_STRING) { key = rspamd_decode_base32 (ucl_object_tostring (elt), 0, &keylen); if (keylen < sizeof (rspamd_sipkey_t)) { msg_warn ("siphash key is too short: %z", keylen); g_free (key); } else { memcpy (cf->sk, key, sizeof (cf->sk)); g_free (key); } } else { msg_warn_pool ("siphash cannot be used without key"); } } } else { elt = ucl_object_lookup (obj, "compat"); if (elt != NULL && ucl_object_toboolean (elt)) { cf->ht = RSPAMD_OSB_HASH_COMPAT; } } elt = ucl_object_lookup (obj, "window"); if (elt != NULL && ucl_object_type (elt) == UCL_INT) { cf->window_size = ucl_object_toint (elt); if (cf->window_size > DEFAULT_FEATURE_WINDOW_SIZE * 4) { msg_err_pool ("too large window size: %d", cf->window_size); cf->window_size = DEFAULT_FEATURE_WINDOW_SIZE; } } return cf; }
gint rspamd_tokenizer_osb (struct rspamd_stat_ctx *ctx, rspamd_mempool_t *pool, GArray *words, gboolean is_utf, const gchar *prefix, GPtrArray *result) { rspamd_token_t *new_tok = NULL; rspamd_stat_token_t *token; struct rspamd_osb_tokenizer_config *osb_cf; guint64 cur, seed; struct token_pipe_entry *hashpipe; guint32 h1, h2; gsize token_size; guint processed = 0, i, w, window_size, token_flags = 0; if (words == NULL) { return FALSE; } osb_cf = ctx->tkcf; window_size = osb_cf->window_size; if (prefix) { seed = rspamd_cryptobox_fast_hash_specific (RSPAMD_CRYPTOBOX_XXHASH64, prefix, strlen (prefix), osb_cf->seed); } else { seed = osb_cf->seed; } hashpipe = g_alloca (window_size * sizeof (hashpipe[0])); for (i = 0; i < window_size; i++) { hashpipe[i].h = 0xfe; hashpipe[i].t = NULL; } token_size = sizeof (rspamd_token_t) + sizeof (gdouble) * ctx->statfiles->len; g_assert (token_size > 0); for (w = 0; w < words->len; w ++) { token = &g_array_index (words, rspamd_stat_token_t, w); token_flags = token->flags; if (osb_cf->ht == RSPAMD_OSB_HASH_COMPAT) { rspamd_ftok_t ftok; ftok.begin = token->begin; ftok.len = token->len; cur = rspamd_fstrhash_lc (&ftok, is_utf); } else { /* We know that the words are normalized */ if (osb_cf->ht == RSPAMD_OSB_HASH_XXHASH) { cur = rspamd_cryptobox_fast_hash_specific (RSPAMD_CRYPTOBOX_XXHASH64, token->begin, token->len, osb_cf->seed); } else { rspamd_cryptobox_siphash ((guchar *)&cur, token->begin, token->len, osb_cf->sk); if (prefix) { cur ^= seed; } } } if (token_flags & RSPAMD_STAT_TOKEN_FLAG_UNIGRAM) { new_tok = rspamd_mempool_alloc0 (pool, token_size); new_tok->flags = token_flags; new_tok->t1 = token; new_tok->t2 = token; new_tok->data = cur; new_tok->window_idx = 0; g_ptr_array_add (result, new_tok); continue; } #define ADD_TOKEN do {\ new_tok = rspamd_mempool_alloc0 (pool, token_size); \ new_tok->flags = token_flags; \ new_tok->t1 = hashpipe[0].t; \ new_tok->t2 = hashpipe[i].t; \ if (osb_cf->ht == RSPAMD_OSB_HASH_COMPAT) { \ h1 = ((guint32)hashpipe[0].h) * primes[0] + \ ((guint32)hashpipe[i].h) * primes[i << 1]; \ h2 = ((guint32)hashpipe[0].h) * primes[1] + \ ((guint32)hashpipe[i].h) * primes[(i << 1) - 1]; \ memcpy((guchar *)&new_tok->data, &h1, sizeof (h1)); \ memcpy(((guchar *)&new_tok->data) + sizeof (h1), &h2, sizeof (h2)); \ } \ else { \ new_tok->data = hashpipe[0].h * primes[0] + hashpipe[i].h * primes[i << 1]; \ } \ new_tok->window_idx = i + 1; \ g_ptr_array_add (result, new_tok); \ } while(0) if (processed < window_size) { /* Just fill a hashpipe */ ++processed; hashpipe[window_size - processed].h = cur; hashpipe[window_size - processed].t = token; } else { /* Shift hashpipe */ for (i = window_size - 1; i > 0; i--) { hashpipe[i] = hashpipe[i - 1]; } hashpipe[0].h = cur; hashpipe[0].t = token; processed++; for (i = 1; i < window_size; i++) { ADD_TOKEN; } } } if (processed > 1 && processed <= window_size) { processed --; memmove (hashpipe, &hashpipe[window_size - processed], processed * sizeof (hashpipe[0])); for (i = 1; i < processed; i++) { ADD_TOKEN; } } #undef ADD_TOKEN return TRUE; }
static void dkim_symbol_callback (struct rspamd_task *task, void *unused) { GList *hlist; rspamd_dkim_context_t *ctx; rspamd_dkim_key_t *key; GError *err = NULL; struct raw_header *rh; struct dkim_check_result *res = NULL, *cur; /* First check if a message has its signature */ hlist = rspamd_message_get_header (task, DKIM_SIGNHEADER, FALSE); if (hlist != NULL) { /* Check whitelist */ msg_debug ("dkim signature found"); if (radix_find_compressed_addr (dkim_module_ctx->whitelist_ip, task->from_addr) == RADIX_NO_VALUE) { /* Parse signature */ msg_debug ("create dkim signature"); while (hlist != NULL) { rh = (struct raw_header *)hlist->data; if (res == NULL) { res = rspamd_mempool_alloc0 (task->task_pool, sizeof (*res)); res->prev = res; res->w = rspamd_session_get_watcher (task->s); cur = res; } else { cur = rspamd_mempool_alloc0 (task->task_pool, sizeof (*res)); } cur->first = res; cur->res = -1; cur->task = task; cur->mult_allow = 1.0; cur->mult_deny = 1.0; ctx = rspamd_create_dkim_context (rh->decoded, task->task_pool, dkim_module_ctx->time_jitter, &err); if (ctx == NULL) { if (err != NULL) { msg_info ("<%s> cannot parse DKIM context: %s", task->message_id, err->message); g_error_free (err); } else { msg_info ("<%s> cannot parse DKIM context: unknown error", task->message_id); } hlist = g_list_next (hlist); continue; } else { /* Get key */ cur->ctx = ctx; if (dkim_module_ctx->trusted_only && (dkim_module_ctx->dkim_domains == NULL || g_hash_table_lookup (dkim_module_ctx->dkim_domains, ctx->domain) == NULL)) { msg_debug ("skip dkim check for %s domain", ctx->domain); hlist = g_list_next (hlist); continue; } key = rspamd_lru_hash_lookup (dkim_module_ctx->dkim_hash, ctx->dns_key, task->tv.tv_sec); if (key != NULL) { debug_task ("found key for %s in cache", ctx->dns_key); cur->key = key; } else { debug_task ("request key for %s from DNS", ctx->dns_key); rspamd_get_dkim_key (ctx, task, dkim_module_key_handler, cur); } } if (res != cur) { DL_APPEND (res, cur); } hlist = g_list_next (hlist); } } } if (res != NULL) { rspamd_session_watcher_push (task->s); dkim_module_check (res); } }
static void rspamd_archive_process_zip (struct rspamd_task *task, struct rspamd_mime_part *part) { const guchar *p, *start, *end, *eocd = NULL, *cd; const guint32 eocd_magic = 0x06054b50, cd_basic_len = 46; const guchar cd_magic[] = {0x50, 0x4b, 0x01, 0x02}; guint32 cd_offset, cd_size, comp_size, uncomp_size; guint16 extra_len, fname_len, comment_len; struct rspamd_archive *arch; struct rspamd_archive_file *f; /* Zip files have interesting data at the end of archive */ p = part->parsed_data.begin + part->parsed_data.len - 1; start = part->parsed_data.begin; end = p; /* Search for EOCD: * 22 bytes is a typical size of eocd without a comment and * end points one byte after the last character */ p -= 21; while (p > start + sizeof (guint32)) { guint32 t; /* XXX: not an efficient approach */ memcpy (&t, p, sizeof (t)); if (GUINT32_FROM_LE (t) == eocd_magic) { eocd = p; break; } p --; } if (eocd == NULL) { /* Not a zip file */ msg_debug_task ("zip archive is invalid (no EOCD)"); return; } if (end - eocd < 21) { msg_debug_task ("zip archive is invalid (short EOCD)"); return; } memcpy (&cd_size, eocd + 12, sizeof (cd_size)); cd_size = GUINT32_FROM_LE (cd_size); memcpy (&cd_offset, eocd + 16, sizeof (cd_offset)); cd_offset = GUINT32_FROM_LE (cd_offset); /* We need to check sanity as well */ if (cd_offset + cd_size != (guint)(eocd - start)) { msg_debug_task ("zip archive is invalid (bad size/offset for CD)"); return; } cd = start + cd_offset; arch = rspamd_mempool_alloc0 (task->task_pool, sizeof (*arch)); arch->files = g_ptr_array_new (); arch->type = RSPAMD_ARCHIVE_ZIP; rspamd_mempool_add_destructor (task->task_pool, rspamd_archive_dtor, arch); while (cd < eocd) { /* Read central directory record */ if (eocd - cd < cd_basic_len || memcmp (cd, cd_magic, sizeof (cd_magic)) != 0) { msg_debug_task ("zip archive is invalid (bad cd record)"); return; } memcpy (&comp_size, cd + 20, sizeof (guint32)); comp_size = GUINT32_FROM_LE (comp_size); memcpy (&uncomp_size, cd + 24, sizeof (guint32)); uncomp_size = GUINT32_FROM_LE (uncomp_size); memcpy (&fname_len, cd + 28, sizeof (fname_len)); fname_len = GUINT16_FROM_LE (fname_len); memcpy (&extra_len, cd + 30, sizeof (extra_len)); extra_len = GUINT16_FROM_LE (extra_len); memcpy (&comment_len, cd + 32, sizeof (comment_len)); comment_len = GUINT16_FROM_LE (comment_len); if (cd + fname_len + comment_len + extra_len + cd_basic_len > eocd) { msg_debug_task ("zip archive is invalid (too large cd record)"); return; } f = g_slice_alloc0 (sizeof (*f)); f->fname = g_string_new_len (cd + cd_basic_len, fname_len); f->compressed_size = comp_size; f->uncompressed_size = uncomp_size; g_ptr_array_add (arch->files, f); msg_debug_task ("found file in zip archive: %v", f->fname); cd += fname_len + comment_len + extra_len + cd_basic_len; } part->flags |= RSPAMD_MIME_PART_ARCHIVE; part->specific.arch = arch; if (part->cd) { arch->archive_name = &part->cd->filename; } arch->size = part->parsed_data.len; }
static void rspamd_symbols_cache_validate_cb (gpointer k, gpointer v, gpointer ud) { struct cache_item *item = v, *parent; struct symbols_cache *cache = (struct symbols_cache *)ud; GList *cur; struct metric *m; struct rspamd_symbol_def *s; gboolean skipped, ghost; gint p1, p2; ghost = item->weight == 0 ? TRUE : FALSE; /* Check whether this item is skipped */ skipped = !ghost; if (item->type == SYMBOL_TYPE_NORMAL && g_hash_table_lookup (cache->cfg->metrics_symbols, item->symbol) == NULL) { cur = g_list_first (cache->cfg->metrics_list); while (cur) { m = cur->data; if (m->accept_unknown_symbols) { GList *mlist; skipped = FALSE; item->weight = item->weight * (m->unknown_weight); s = rspamd_mempool_alloc0 (cache->static_pool, sizeof (*s)); s->name = item->symbol; s->weight_ptr = &item->weight; g_hash_table_insert (m->symbols, item->symbol, s); mlist = g_hash_table_lookup (cache->cfg->metrics_symbols, item->symbol); mlist = g_list_prepend (mlist, m); g_hash_table_insert (cache->cfg->metrics_symbols, item->symbol, mlist); msg_info ("adding unknown symbol %s to metric %s", item->symbol, m->name); } cur = g_list_next (cur); } } else { skipped = FALSE; } if (skipped) { item->type = SYMBOL_TYPE_SKIPPED; msg_warn ("symbol %s is not registered in any metric, so skip its check", item->symbol); } if (ghost) { msg_debug ("symbol %s is registered as ghost symbol, it won't be inserted " "to any metric", item->symbol); } if (item->weight < 0 && item->priority == 0) { item->priority ++; } if (item->type == SYMBOL_TYPE_VIRTUAL && item->parent != -1) { g_assert (item->parent < (gint)cache->items_by_id->len); parent = g_ptr_array_index (cache->items_by_id, item->parent); if (fabs (parent->weight) < fabs (item->weight)) { parent->weight = item->weight; } p1 = abs (item->priority); p2 = abs (parent->priority); if (p1 != p2) { parent->priority = MAX (p1, p2); item->priority = parent->priority; } } if (fabs (item->weight) > cache->max_weight) { cache->max_weight = fabs (item->weight); } }
gboolean parse_smtp_command (struct smtp_session *session, rspamd_ftok_t *line, struct smtp_command **cmd) { enum { SMTP_PARSE_START = 0, SMTP_PARSE_SPACES, SMTP_PARSE_ARGUMENT, SMTP_PARSE_DONE } state; const gchar *p, *c; gchar ch, cmd_buf[4]; guint i; rspamd_ftok_t *arg = NULL; struct smtp_command *pcmd; if (line->len == 0) { return FALSE; } state = SMTP_PARSE_START; c = line->begin; p = c; *cmd = rspamd_mempool_alloc0 (session->pool, sizeof (struct smtp_command)); pcmd = *cmd; for (i = 0; i < line->len; i++, p++) { ch = *p; switch (state) { case SMTP_PARSE_START: if (ch == ' ' || ch == ':' || ch == CR || ch == LF || i == line->len - 1) { if (i == line->len - 1) { p++; } if (p - c == 4) { cmd_buf[0] = g_ascii_toupper (c[0]); cmd_buf[1] = g_ascii_toupper (c[1]); cmd_buf[2] = g_ascii_toupper (c[2]); cmd_buf[3] = g_ascii_toupper (c[3]); if (memcmp (cmd_buf, "HELO", 4) == 0) { pcmd->command = SMTP_COMMAND_HELO; } else if (memcmp (cmd_buf, "EHLO", 4) == 0) { pcmd->command = SMTP_COMMAND_EHLO; } else if (memcmp (cmd_buf, "MAIL", 4) == 0) { pcmd->command = SMTP_COMMAND_MAIL; } else if (memcmp (cmd_buf, "RCPT", 4) == 0) { pcmd->command = SMTP_COMMAND_RCPT; } else if (memcmp (cmd_buf, "DATA", 4) == 0) { pcmd->command = SMTP_COMMAND_DATA; } else if (memcmp (cmd_buf, "QUIT", 4) == 0) { pcmd->command = SMTP_COMMAND_QUIT; } else if (memcmp (cmd_buf, "NOOP", 4) == 0) { pcmd->command = SMTP_COMMAND_NOOP; } else if (memcmp (cmd_buf, "EXPN", 4) == 0) { pcmd->command = SMTP_COMMAND_EXPN; } else if (memcmp (cmd_buf, "RSET", 4) == 0) { pcmd->command = SMTP_COMMAND_RSET; } else if (memcmp (cmd_buf, "HELP", 4) == 0) { pcmd->command = SMTP_COMMAND_HELP; } else if (memcmp (cmd_buf, "VRFY", 4) == 0) { pcmd->command = SMTP_COMMAND_VRFY; } else { msg_info ("invalid command: %*s", 4, cmd_buf); return FALSE; } } else { /* Invalid command */ msg_info ("invalid command: %*s", 4, c); return FALSE; } /* Now check what we have */ if (ch == ' ' || ch == ':') { state = SMTP_PARSE_SPACES; } else if (ch == CR) { state = SMTP_PARSE_DONE; } else if (ch == LF) { return TRUE; } } else if ((ch < 'A' || ch > 'Z') && (ch < 'a' || ch > 'z')) { msg_info ("invalid letter code in SMTP command: %d", (gint)ch); return FALSE; } break; case SMTP_PARSE_SPACES: if (ch == CR) { state = SMTP_PARSE_DONE; } else if (ch == LF) { goto end; } else if (ch != ' ' && ch != ':') { state = SMTP_PARSE_ARGUMENT; arg = rspamd_mempool_alloc (session->pool, sizeof (rspamd_fstring_t)); c = p; } break; case SMTP_PARSE_ARGUMENT: if (ch == ' ' || ch == ':' || ch == CR || ch == LF || i == line->len - 1) { if (i == line->len - 1 && (ch != ' ' && ch != CR && ch != LF)) { p++; } arg->len = p - c; arg->begin = rspamd_mempool_alloc (session->pool, arg->len); memcpy ((gchar *)arg->begin, c, arg->len); pcmd->args = g_list_prepend (pcmd->args, arg); if (ch == ' ' || ch == ':') { state = SMTP_PARSE_SPACES; } else if (ch == CR) { state = SMTP_PARSE_DONE; } else { goto end; } } break; case SMTP_PARSE_DONE: if (ch == LF) { goto end; } msg_info ("CR without LF in SMTP command"); return FALSE; } } end: if (pcmd->args) { pcmd->args = g_list_reverse (pcmd->args); rspamd_mempool_add_destructor (session->pool, (rspamd_mempool_destruct_t)g_list_free, pcmd->args); } return TRUE; }
gboolean rspamd_parse_bind_line (struct rspamd_config *cfg, struct rspamd_worker_conf *cf, const gchar *str) { struct rspamd_worker_bind_conf *cnf; gchar **tokens, *err; gboolean ret = TRUE; if (str == NULL) { return FALSE; } if (str[0] == '[') { /* This is an ipv6 address */ gsize len, ntok; const gchar *start, *ip_pos; start = str + 1; len = strcspn (start, "]"); if (start[len] != ']') { return FALSE; } ip_pos = start; start += len + 1; ntok = 1; if (*start == ':') { ntok = 2; start ++; } tokens = g_malloc_n (ntok + 1, sizeof (gchar *)); tokens[ntok] = NULL; tokens[0] = g_malloc (len + 1); rspamd_strlcpy (tokens[0], ip_pos, len + 1); if (ntok > 1) { tokens[1] = g_strdup (start); } } else { tokens = g_strsplit_set (str, ":", 0); } if (!tokens || !tokens[0]) { return FALSE; } cnf = rspamd_mempool_alloc0 (cfg->cfg_pool, sizeof (struct rspamd_worker_bind_conf)); cnf->cnt = 1024; if (strcmp (tokens[0], "systemd") == 0) { /* The actual socket will be passed by systemd environment */ cnf->is_systemd = TRUE; cnf->cnt = strtoul (tokens[1], &err, 10); cnf->addrs = NULL; if (err == NULL || *err == '\0') { cnf->name = rspamd_mempool_strdup (cfg->cfg_pool, str); LL_PREPEND (cf->bind_conf, cnf); } else { msg_err_config ("cannot parse bind line: %s", str); ret = FALSE; } } else { if (!rspamd_parse_host_port_priority_strv (tokens, &cnf->addrs, NULL, &cnf->name, DEFAULT_BIND_PORT, cfg->cfg_pool)) { msg_err_config ("cannot parse bind line: %s", str); ret = FALSE; } else { cnf->cnt = cnf->addrs->len; LL_PREPEND (cf->bind_conf, cnf); } } g_strfreev (tokens); return ret; }
static void dkim_symbol_callback (struct rspamd_task *task, void *unused) { GList *hlist; rspamd_dkim_context_t *ctx; rspamd_dkim_key_t *key; GError *err = NULL; struct raw_header *rh; struct dkim_check_result *res = NULL, *cur; guint checked = 0; /* First check if plugin should be enabled */ if (task->user != NULL || rspamd_inet_address_is_local (task->from_addr)) { msg_info_task ("skip DKIM checks for local networks and authorized users"); return; } /* Check whitelist */ if (radix_find_compressed_addr (dkim_module_ctx->whitelist_ip, task->from_addr) != RADIX_NO_VALUE) { msg_info_task ("skip DKIM checks for whitelisted address"); return; } /* Now check if a message has its signature */ hlist = rspamd_message_get_header (task, DKIM_SIGNHEADER, FALSE); if (hlist != NULL) { msg_debug_task ("dkim signature found"); while (hlist != NULL) { rh = (struct raw_header *)hlist->data; if (rh->decoded == NULL || rh->decoded[0] == '\0') { msg_info_task ("<%s> cannot load empty DKIM context", task->message_id); hlist = g_list_next (hlist); continue; } if (res == NULL) { res = rspamd_mempool_alloc0 (task->task_pool, sizeof (*res)); res->prev = res; res->w = rspamd_session_get_watcher (task->s); cur = res; } else { cur = rspamd_mempool_alloc0 (task->task_pool, sizeof (*res)); } cur->first = res; cur->res = -1; cur->task = task; cur->mult_allow = 1.0; cur->mult_deny = 1.0; ctx = rspamd_create_dkim_context (rh->decoded, task->task_pool, dkim_module_ctx->time_jitter, &err); if (ctx == NULL) { if (err != NULL) { msg_info_task ("<%s> cannot parse DKIM context: %e", task->message_id, err); g_error_free (err); err = NULL; } else { msg_info_task ("<%s> cannot parse DKIM context: " "unknown error", task->message_id); } hlist = g_list_next (hlist); continue; } else { /* Get key */ cur->ctx = ctx; if (dkim_module_ctx->trusted_only && (dkim_module_ctx->dkim_domains == NULL || g_hash_table_lookup (dkim_module_ctx->dkim_domains, rspamd_dkim_get_domain (ctx)) == NULL)) { msg_debug_task ("skip dkim check for %s domain", rspamd_dkim_get_domain (ctx)); hlist = g_list_next (hlist); continue; } key = rspamd_lru_hash_lookup (dkim_module_ctx->dkim_hash, rspamd_dkim_get_dns_key (ctx), task->tv.tv_sec); if (key != NULL) { cur->key = rspamd_dkim_key_ref (key); /* Release key when task is processed */ rspamd_mempool_add_destructor (task->task_pool, dkim_module_key_dtor, cur->key); } else { rspamd_get_dkim_key (ctx, task, dkim_module_key_handler, cur); } } if (res != cur) { DL_APPEND (res, cur); } if (dkim_module_ctx->skip_multi) { if (hlist->next) { msg_info_task ("message has multiple signatures but we" " check only one as 'skip_multi' is set"); } break; } checked ++; if (checked > dkim_module_ctx->max_sigs) { msg_info_task ("message has multiple signatures but we" " stopped after %d checked signatures as limit" " is reached", checked); break; } hlist = g_list_next (hlist); } } else { rspamd_task_insert_result (task, dkim_module_ctx->symbol_na, 1.0, NULL); } if (res != NULL) { rspamd_session_watcher_push (task->s); dkim_module_check (res); } }
gint regexp_module_config (struct rspamd_config *cfg) { struct regexp_module_item *cur_item; const ucl_object_t *sec, *value, *elt; ucl_object_iter_t it = NULL; gint res = TRUE, id; if (!rspamd_config_is_module_enabled (cfg, "regexp")) { return TRUE; } sec = ucl_object_find_key (cfg->rcl_obj, "regexp"); if (sec == NULL) { msg_err_config ("regexp module enabled, but no rules are defined"); return TRUE; } regexp_module_ctx->max_size = 0; while ((value = ucl_iterate_object (sec, &it, true)) != NULL) { if (g_ascii_strncasecmp (ucl_object_key (value), "max_size", sizeof ("max_size") - 1) == 0) { regexp_module_ctx->max_size = ucl_obj_toint (value); rspamd_mime_expression_set_re_limit (regexp_module_ctx->max_size); } else if (g_ascii_strncasecmp (ucl_object_key (value), "max_threads", sizeof ("max_threads") - 1) == 0) { msg_warn_config ("regexp module is now single threaded, max_threads is ignored"); } else if (value->type == UCL_STRING) { cur_item = rspamd_mempool_alloc0 (regexp_module_ctx->regexp_pool, sizeof (struct regexp_module_item)); cur_item->symbol = ucl_object_key (value); if (!read_regexp_expression (regexp_module_ctx->regexp_pool, cur_item, ucl_object_key (value), ucl_obj_tostring (value), cfg)) { res = FALSE; } else { rspamd_symbols_cache_add_symbol (cfg->cache, cur_item->symbol, 0, process_regexp_item, cur_item, SYMBOL_TYPE_NORMAL, -1); } } else if (value->type == UCL_USERDATA) { /* Just a lua function */ cur_item = rspamd_mempool_alloc0 (regexp_module_ctx->regexp_pool, sizeof (struct regexp_module_item)); cur_item->symbol = ucl_object_key (value); cur_item->lua_function = ucl_object_toclosure (value); rspamd_symbols_cache_add_symbol (cfg->cache, cur_item->symbol, 0, process_regexp_item, cur_item, SYMBOL_TYPE_NORMAL, -1); } else if (value->type == UCL_OBJECT) { const gchar *description = NULL, *group = NULL, *metric = DEFAULT_METRIC; gdouble score = 0.0; gboolean one_shot = FALSE, is_lua = FALSE, valid_expression = TRUE; /* We have some lua table, extract its arguments */ elt = ucl_object_find_key (value, "callback"); if (elt == NULL || elt->type != UCL_USERDATA) { /* Try plain regexp expression */ elt = ucl_object_find_any_key (value, "regexp", "re", NULL); if (elt != NULL && ucl_object_type (elt) == UCL_STRING) { cur_item = rspamd_mempool_alloc0 (regexp_module_ctx->regexp_pool, sizeof (struct regexp_module_item)); cur_item->symbol = ucl_object_key (value); if (!read_regexp_expression (regexp_module_ctx->regexp_pool, cur_item, ucl_object_key (value), ucl_obj_tostring (elt), cfg)) { res = FALSE; } else { valid_expression = TRUE; } } else { msg_err_config ( "no callback/expression defined for regexp symbol: " "%s", ucl_object_key (value)); } } else { is_lua = TRUE; cur_item = rspamd_mempool_alloc0 ( regexp_module_ctx->regexp_pool, sizeof (struct regexp_module_item)); cur_item->symbol = ucl_object_key (value); cur_item->lua_function = ucl_object_toclosure (value); } if (cur_item && (is_lua || valid_expression)) { id = rspamd_symbols_cache_add_symbol (cfg->cache, cur_item->symbol, 0, process_regexp_item, cur_item, SYMBOL_TYPE_NORMAL, -1); elt = ucl_object_find_key (value, "condition"); if (elt != NULL && ucl_object_type (elt) == UCL_USERDATA) { struct ucl_lua_funcdata *conddata; conddata = ucl_object_toclosure (elt); rspamd_symbols_cache_add_condition (cfg->cache, id, conddata->L, conddata->idx); } elt = ucl_object_find_key (value, "metric"); if (elt) { metric = ucl_object_tostring (elt); } elt = ucl_object_find_key (value, "description"); if (elt) { description = ucl_object_tostring (elt); } elt = ucl_object_find_key (value, "group"); if (elt) { group = ucl_object_tostring (elt); } elt = ucl_object_find_key (value, "score"); if (elt) { score = ucl_object_todouble (elt); } elt = ucl_object_find_key (value, "one_shot"); if (elt) { one_shot = ucl_object_toboolean (elt); } rspamd_config_add_metric_symbol (cfg, metric, cur_item->symbol, score, description, group, one_shot, FALSE); } } else { msg_warn_config ("unknown type of attribute %s for regexp module", ucl_object_key (value)); } } return res; }