static gboolean rspamd_fuzzy_backend_redis_try_ucl (struct rspamd_fuzzy_backend_redis *backend, const ucl_object_t *obj, struct rspamd_config *cfg) { const ucl_object_t *elt, *relt; elt = ucl_object_lookup_any (obj, "read_servers", "servers", NULL); if (elt == NULL) { return FALSE; } backend->read_servers = rspamd_upstreams_create (cfg->ups_ctx); if (!rspamd_upstreams_from_ucl (backend->read_servers, elt, REDIS_DEFAULT_PORT, NULL)) { msg_err_config ("cannot get read servers configuration"); return FALSE; } relt = elt; elt = ucl_object_lookup (obj, "write_servers"); if (elt == NULL) { /* Use read servers as write ones */ g_assert (relt != NULL); backend->write_servers = rspamd_upstreams_create (cfg->ups_ctx); if (!rspamd_upstreams_from_ucl (backend->write_servers, relt, REDIS_DEFAULT_PORT, NULL)) { msg_err_config ("cannot get write servers configuration"); return FALSE; } } else { backend->write_servers = rspamd_upstreams_create (cfg->ups_ctx); if (!rspamd_upstreams_from_ucl (backend->write_servers, elt, REDIS_DEFAULT_PORT, NULL)) { msg_err_config ("cannot get write servers configuration"); rspamd_upstreams_destroy (backend->write_servers); backend->write_servers = NULL; } } elt = ucl_object_lookup (obj, "prefix"); if (elt == NULL || ucl_object_type (elt) != UCL_STRING) { backend->redis_object = REDIS_DEFAULT_OBJECT; } else { backend->redis_object = ucl_object_tostring (elt); } elt = ucl_object_lookup (obj, "timeout"); if (elt) { backend->timeout = ucl_object_todouble (elt); } else { backend->timeout = REDIS_DEFAULT_TIMEOUT; } elt = ucl_object_lookup (obj, "password"); if (elt) { backend->password = ucl_object_tostring (elt); } else { backend->password = NULL; } elt = ucl_object_lookup_any (obj, "db", "database", "dbname", NULL); if (elt) { backend->dbname = ucl_object_tostring (elt); } else { backend->dbname = NULL; } return TRUE; }
static gboolean rspamd_redis_cache_try_ucl (struct rspamd_redis_cache_ctx *cache_ctx, const ucl_object_t *obj, struct rspamd_config *cfg, const gchar *symbol) { const ucl_object_t *elt, *relt; elt = ucl_object_lookup_any (obj, "read_servers", "servers", NULL); if (elt == NULL) { return FALSE; } cache_ctx->read_servers = rspamd_upstreams_create (cfg->ups_ctx); if (!rspamd_upstreams_from_ucl (cache_ctx->read_servers, elt, REDIS_DEFAULT_PORT, NULL)) { msg_err ("statfile %s cannot get read servers configuration", symbol); return FALSE; } relt = elt; elt = ucl_object_lookup (obj, "write_servers"); if (elt == NULL) { /* Use read servers as write ones */ g_assert (relt != NULL); cache_ctx->write_servers = rspamd_upstreams_create (cfg->ups_ctx); if (!rspamd_upstreams_from_ucl (cache_ctx->write_servers, relt, REDIS_DEFAULT_PORT, NULL)) { msg_err ("statfile %s cannot get write servers configuration", symbol); return FALSE; } } else { cache_ctx->write_servers = rspamd_upstreams_create (cfg->ups_ctx); if (!rspamd_upstreams_from_ucl (cache_ctx->write_servers, elt, REDIS_DEFAULT_PORT, NULL)) { msg_err ("statfile %s cannot get write servers configuration", symbol); rspamd_upstreams_destroy (cache_ctx->write_servers); cache_ctx->write_servers = NULL; } } elt = ucl_object_lookup (obj, "timeout"); if (elt) { cache_ctx->timeout = ucl_object_todouble (elt); } else { cache_ctx->timeout = REDIS_DEFAULT_TIMEOUT; } elt = ucl_object_lookup (obj, "password"); if (elt) { cache_ctx->password = ucl_object_tostring (elt); } else { cache_ctx->password = NULL; } elt = ucl_object_lookup_any (obj, "db", "database", "dbname", NULL); if (elt) { cache_ctx->dbname = ucl_object_tostring (elt); } else { cache_ctx->dbname = NULL; } elt = ucl_object_lookup_any (obj, "cache_key", "key", NULL); if (elt == NULL || ucl_object_type (elt) != UCL_STRING) { cache_ctx->redis_object = DEFAULT_REDIS_KEY; } else { cache_ctx->redis_object = ucl_object_tostring (elt); } return TRUE; }
static gboolean rspamd_redis_try_ucl (struct redis_stat_ctx *backend, const ucl_object_t *obj, struct rspamd_config *cfg, const gchar *symbol) { const ucl_object_t *elt, *relt, *users_enabled; const gchar *lua_script; elt = ucl_object_lookup_any (obj, "read_servers", "servers", NULL); if (elt == NULL) { return FALSE; } backend->read_servers = rspamd_upstreams_create (cfg->ups_ctx); if (!rspamd_upstreams_from_ucl (backend->read_servers, elt, REDIS_DEFAULT_PORT, NULL)) { msg_err ("statfile %s cannot get read servers configuration", symbol); return FALSE; } relt = elt; elt = ucl_object_lookup (obj, "write_servers"); if (elt == NULL) { /* Use read servers as write ones */ g_assert (relt != NULL); backend->write_servers = rspamd_upstreams_create (cfg->ups_ctx); if (!rspamd_upstreams_from_ucl (backend->write_servers, relt, REDIS_DEFAULT_PORT, NULL)) { msg_err ("statfile %s cannot get write servers configuration", symbol); return FALSE; } } else { backend->write_servers = rspamd_upstreams_create (cfg->ups_ctx); if (!rspamd_upstreams_from_ucl (backend->write_servers, elt, REDIS_DEFAULT_PORT, NULL)) { msg_err ("statfile %s cannot get write servers configuration", symbol); rspamd_upstreams_destroy (backend->write_servers); backend->write_servers = NULL; } } elt = ucl_object_lookup (obj, "prefix"); if (elt == NULL || ucl_object_type (elt) != UCL_STRING) { /* Default non-users statistics */ backend->redis_object = REDIS_DEFAULT_OBJECT; /* * Make redis backend compatible with sqlite3 backend in users settings */ users_enabled = ucl_object_lookup_any (obj, "per_user", "users_enabled", NULL); if (users_enabled != NULL) { if (ucl_object_type (users_enabled) == UCL_BOOLEAN) { backend->enable_users = ucl_object_toboolean (users_enabled); backend->cbref_user = -1; if (backend->enable_users) { backend->redis_object = REDIS_DEFAULT_USERS_OBJECT; } } else if (ucl_object_type (users_enabled) == UCL_STRING) { lua_script = ucl_object_tostring (users_enabled); if (luaL_dostring (cfg->lua_state, lua_script) != 0) { msg_err_config ("cannot execute lua script for users " "extraction: %s", lua_tostring (cfg->lua_state, -1)); } else { if (lua_type (cfg->lua_state, -1) == LUA_TFUNCTION) { backend->enable_users = TRUE; backend->cbref_user = luaL_ref (cfg->lua_state, LUA_REGISTRYINDEX); } else { msg_err_config ("lua script must return " "function(task) and not %s", lua_typename (cfg->lua_state, lua_type ( cfg->lua_state, -1))); } } } } else { backend->enable_users = FALSE; } } else { /* XXX: sanity check */ backend->redis_object = ucl_object_tostring (elt); } elt = ucl_object_lookup (obj, "timeout"); if (elt) { backend->timeout = ucl_object_todouble (elt); } else { backend->timeout = REDIS_DEFAULT_TIMEOUT; } elt = ucl_object_lookup (obj, "password"); if (elt) { backend->password = ucl_object_tostring (elt); } else { backend->password = NULL; } elt = ucl_object_lookup_any (obj, "db", "database", "dbname", NULL); if (elt) { backend->dbname = ucl_object_tostring (elt); } else { backend->dbname = NULL; } return TRUE; }
void rspamd_upstream_test_func (void) { struct upstream_list *ls, *nls; struct upstream *up, *upn; struct event_base *ev_base = event_init (); struct rspamd_dns_resolver *resolver; struct rspamd_config *cfg; gint i, success = 0; const gint assumptions = 100500; gdouble p; struct event ev; struct timeval tv; rspamd_inet_addr_t *addr, *next_addr, *paddr; cfg = rspamd_config_new (); cfg->dns_retransmits = 2; cfg->dns_timeout = 0.5; cfg->upstream_max_errors = 1; cfg->upstream_revive_time = 0.5; cfg->upstream_error_time = 2; resolver = dns_resolver_init (NULL, ev_base, cfg); rspamd_upstreams_library_config (cfg, cfg->ups_ctx, ev_base, resolver->r); /* * Test v4/v6 priorities */ nls = rspamd_upstreams_create (cfg->ups_ctx); g_assert (rspamd_upstreams_add_upstream (nls, "127.0.0.1", 0, NULL)); up = rspamd_upstream_get (nls, RSPAMD_UPSTREAM_RANDOM, NULL, 0); rspamd_parse_inet_address (&paddr, "127.0.0.2", 0); g_assert (rspamd_upstream_add_addr (up, paddr)); rspamd_parse_inet_address (&paddr, "::1", 0); g_assert (rspamd_upstream_add_addr (up, paddr)); /* Rewind to start */ addr = rspamd_upstream_addr (up); addr = rspamd_upstream_addr (up); /* cur should be zero here */ addr = rspamd_upstream_addr (up); next_addr = rspamd_upstream_addr (up); g_assert (rspamd_inet_address_get_af (addr) == AF_INET); g_assert (rspamd_inet_address_get_af (next_addr) == AF_INET); next_addr = rspamd_upstream_addr (up); g_assert (rspamd_inet_address_get_af (next_addr) == AF_INET6); next_addr = rspamd_upstream_addr (up); g_assert (rspamd_inet_address_get_af (next_addr) == AF_INET); next_addr = rspamd_upstream_addr (up); g_assert (rspamd_inet_address_get_af (next_addr) == AF_INET); next_addr = rspamd_upstream_addr (up); g_assert (rspamd_inet_address_get_af (next_addr) == AF_INET6); /* Test errors with IPv6 */ rspamd_upstream_fail (up); /* Now we should have merely IPv4 addresses in rotation */ addr = rspamd_upstream_addr (up); for (i = 0; i < 256; i++) { next_addr = rspamd_upstream_addr (up); g_assert (rspamd_inet_address_get_af (addr) == AF_INET); g_assert (rspamd_inet_address_get_af (next_addr) == AF_INET); g_assert (rspamd_inet_address_compare (addr, next_addr) != 0); addr = next_addr; } rspamd_upstreams_destroy (nls); ls = rspamd_upstreams_create (cfg->ups_ctx); g_assert (rspamd_upstreams_parse_line (ls, test_upstream_list, 443, NULL)); g_assert (rspamd_upstreams_count (ls) == 3); /* Test master-slave rotation */ rspamd_upstream_test_method (ls, RSPAMD_UPSTREAM_MASTER_SLAVE, "kernel.org"); rspamd_upstream_test_method (ls, RSPAMD_UPSTREAM_MASTER_SLAVE, "kernel.org"); /* Test round-robin rotation */ rspamd_upstream_test_method (ls, RSPAMD_UPSTREAM_ROUND_ROBIN, "kernel.org"); rspamd_upstream_test_method (ls, RSPAMD_UPSTREAM_ROUND_ROBIN, "kernel.org"); rspamd_upstream_test_method (ls, RSPAMD_UPSTREAM_ROUND_ROBIN, "google.com"); rspamd_upstream_test_method (ls, RSPAMD_UPSTREAM_ROUND_ROBIN, "kernel.org"); rspamd_upstream_test_method (ls, RSPAMD_UPSTREAM_ROUND_ROBIN, "google.com"); rspamd_upstream_test_method (ls, RSPAMD_UPSTREAM_ROUND_ROBIN, "microsoft.com"); /* Test stable hashing */ nls = rspamd_upstreams_create (cfg->ups_ctx); g_assert (rspamd_upstreams_parse_line (nls, test_upstream_list, 443, NULL)); g_assert (rspamd_upstreams_parse_line (nls, new_upstream_list, 443, NULL)); for (i = 0; i < assumptions; i ++) { ottery_rand_bytes (test_key, sizeof (test_key)); up = rspamd_upstream_get (ls, RSPAMD_UPSTREAM_HASHED, test_key, sizeof (test_key)); upn = rspamd_upstream_get (nls, RSPAMD_UPSTREAM_HASHED, test_key, sizeof (test_key)); if (strcmp (rspamd_upstream_name (up), rspamd_upstream_name (upn)) == 0) { success ++; } } p = 1.0 - fabs (3.0 / 4.0 - (gdouble)success / (gdouble)assumptions); /* * P value is calculated as following: * when we add/remove M upstreams from the list, the probability of hash * miss should be close to the relation N / (N + M), where N is the size of * the previous upstreams list. */ msg_debug ("p value for hash consistency: %.6f", p); g_assert (p > 0.9); rspamd_upstreams_destroy (nls); /* Upstream fail test */ evtimer_set (&ev, rspamd_upstream_timeout_handler, resolver); event_base_set (ev_base, &ev); up = rspamd_upstream_get (ls, RSPAMD_UPSTREAM_MASTER_SLAVE, NULL, 0); for (i = 0; i < 100; i ++) { rspamd_upstream_fail (up); } g_assert (rspamd_upstreams_alive (ls) == 2); tv.tv_sec = 2; tv.tv_usec = 0; event_add (&ev, &tv); event_base_loop (ev_base, 0); g_assert (rspamd_upstreams_alive (ls) == 3); rspamd_upstreams_destroy (ls); REF_RELEASE (cfg); }
static gint fuzzy_parse_rule (struct rspamd_config *cfg, const ucl_object_t *obj, gint cb_id) { const ucl_object_t *value, *cur; struct fuzzy_rule *rule; ucl_object_iter_t it = NULL; const char *k = NULL; if (obj->type != UCL_OBJECT) { msg_err_config ("invalid rule definition"); return -1; } rule = fuzzy_rule_new (fuzzy_module_ctx->default_symbol, fuzzy_module_ctx->fuzzy_pool); if ((value = ucl_object_find_key (obj, "mime_types")) != NULL) { it = NULL; while ((cur = ucl_iterate_object (value, &it, value->type == UCL_ARRAY)) != NULL) { rule->mime_types = g_list_concat (rule->mime_types, parse_mime_types (ucl_obj_tostring (cur))); } } if (rule->mime_types != NULL) { rspamd_mempool_add_destructor (fuzzy_module_ctx->fuzzy_pool, (rspamd_mempool_destruct_t)g_list_free, rule->mime_types); } if ((value = ucl_object_find_key (obj, "max_score")) != NULL) { rule->max_score = ucl_obj_todouble (value); } if ((value = ucl_object_find_key (obj, "symbol")) != NULL) { rule->symbol = ucl_obj_tostring (value); } if ((value = ucl_object_find_key (obj, "read_only")) != NULL) { rule->read_only = ucl_obj_toboolean (value); } if ((value = ucl_object_find_key (obj, "skip_unknown")) != NULL) { rule->skip_unknown = ucl_obj_toboolean (value); } if ((value = ucl_object_find_key (obj, "servers")) != NULL) { rule->servers = rspamd_upstreams_create (); rspamd_mempool_add_destructor (fuzzy_module_ctx->fuzzy_pool, (rspamd_mempool_destruct_t)rspamd_upstreams_destroy, rule->servers); rspamd_upstreams_from_ucl (rule->servers, value, DEFAULT_PORT, NULL); } if ((value = ucl_object_find_key (obj, "fuzzy_map")) != NULL) { it = NULL; while ((cur = ucl_iterate_object (value, &it, true)) != NULL) { parse_flags (rule, cfg, cur, cb_id); } } if ((value = ucl_object_find_key (obj, "encryption_key")) != NULL) { /* Create key from user's input */ k = ucl_object_tostring (value); if (k == NULL || (rule->peer_key = rspamd_http_connection_make_peer_key (k)) == NULL) { msg_err_config ("bad encryption key value: %s", k); return -1; } rule->local_key = rspamd_http_connection_gen_key (); } if ((value = ucl_object_find_key (obj, "fuzzy_key")) != NULL) { /* Create key from user's input */ k = ucl_object_tostring (value); } /* Setup keys */ if (k == NULL) { /* Use some default key for all ops */ k = "rspamd"; } rule->hash_key = g_string_sized_new (BLAKE2B_KEYBYTES); blake2 (rule->hash_key->str, k, NULL, BLAKE2B_KEYBYTES, strlen (k), 0); rule->hash_key->len = BLAKE2B_KEYBYTES; if ((value = ucl_object_find_key (obj, "fuzzy_shingles_key")) != NULL) { k = ucl_object_tostring (value); } if (k == NULL) { k = "rspamd"; } rule->shingles_key = g_string_sized_new (16); blake2 (rule->shingles_key->str, k, NULL, 16, strlen (k), 0); rule->shingles_key->len = 16; if (rspamd_upstreams_count (rule->servers) == 0) { msg_err_config ("no servers defined for fuzzy rule with symbol: %s", rule->symbol); return -1; } else { fuzzy_module_ctx->fuzzy_rules = g_list_prepend ( fuzzy_module_ctx->fuzzy_rules, rule); if (rule->symbol != fuzzy_module_ctx->default_symbol) { rspamd_symbols_cache_add_symbol (cfg->cache, rule->symbol, 0, NULL, NULL, SYMBOL_TYPE_VIRTUAL|SYMBOL_TYPE_FINE, cb_id); } } rspamd_mempool_add_destructor (fuzzy_module_ctx->fuzzy_pool, fuzzy_free_rule, rule); return 0; }