static gboolean
rspamd_fuzzy_backend_redis_try_ucl (struct rspamd_fuzzy_backend_redis *backend,
const ucl_object_t *obj,
struct rspamd_config *cfg)
{
const ucl_object_t *elt, *relt;
elt = ucl_object_lookup_any (obj, "read_servers", "servers", NULL);
if (elt == NULL) {
return FALSE;
}
backend->read_servers = rspamd_upstreams_create (cfg->ups_ctx);
if (!rspamd_upstreams_from_ucl (backend->read_servers, elt,
REDIS_DEFAULT_PORT, NULL)) {
msg_err_config ("cannot get read servers configuration");
return FALSE;
}
relt = elt;
elt = ucl_object_lookup (obj, "write_servers");
if (elt == NULL) {
/* Use read servers as write ones */
g_assert (relt != NULL);
backend->write_servers = rspamd_upstreams_create (cfg->ups_ctx);
if (!rspamd_upstreams_from_ucl (backend->write_servers, relt,
REDIS_DEFAULT_PORT, NULL)) {
msg_err_config ("cannot get write servers configuration");
return FALSE;
}
}
else {
backend->write_servers = rspamd_upstreams_create (cfg->ups_ctx);
if (!rspamd_upstreams_from_ucl (backend->write_servers, elt,
REDIS_DEFAULT_PORT, NULL)) {
msg_err_config ("cannot get write servers configuration");
rspamd_upstreams_destroy (backend->write_servers);
backend->write_servers = NULL;
}
}
elt = ucl_object_lookup (obj, "prefix");
if (elt == NULL || ucl_object_type (elt) != UCL_STRING) {
backend->redis_object = REDIS_DEFAULT_OBJECT;
}
else {
backend->redis_object = ucl_object_tostring (elt);
}
elt = ucl_object_lookup (obj, "timeout");
if (elt) {
backend->timeout = ucl_object_todouble (elt);
}
else {
backend->timeout = REDIS_DEFAULT_TIMEOUT;
}
elt = ucl_object_lookup (obj, "password");
if (elt) {
backend->password = ucl_object_tostring (elt);
}
else {
backend->password = NULL;
}
elt = ucl_object_lookup_any (obj, "db", "database", "dbname", NULL);
if (elt) {
backend->dbname = ucl_object_tostring (elt);
}
else {
backend->dbname = NULL;
}
return TRUE;
}
static gboolean
rspamd_redis_cache_try_ucl (struct rspamd_redis_cache_ctx *cache_ctx,
const ucl_object_t *obj,
struct rspamd_config *cfg,
const gchar *symbol)
{
const ucl_object_t *elt, *relt;
elt = ucl_object_lookup_any (obj, "read_servers", "servers", NULL);
if (elt == NULL) {
return FALSE;
}
cache_ctx->read_servers = rspamd_upstreams_create (cfg->ups_ctx);
if (!rspamd_upstreams_from_ucl (cache_ctx->read_servers, elt,
REDIS_DEFAULT_PORT, NULL)) {
msg_err ("statfile %s cannot get read servers configuration",
symbol);
return FALSE;
}
relt = elt;
elt = ucl_object_lookup (obj, "write_servers");
if (elt == NULL) {
/* Use read servers as write ones */
g_assert (relt != NULL);
cache_ctx->write_servers = rspamd_upstreams_create (cfg->ups_ctx);
if (!rspamd_upstreams_from_ucl (cache_ctx->write_servers, relt,
REDIS_DEFAULT_PORT, NULL)) {
msg_err ("statfile %s cannot get write servers configuration",
symbol);
return FALSE;
}
}
else {
cache_ctx->write_servers = rspamd_upstreams_create (cfg->ups_ctx);
if (!rspamd_upstreams_from_ucl (cache_ctx->write_servers, elt,
REDIS_DEFAULT_PORT, NULL)) {
msg_err ("statfile %s cannot get write servers configuration",
symbol);
rspamd_upstreams_destroy (cache_ctx->write_servers);
cache_ctx->write_servers = NULL;
}
}
elt = ucl_object_lookup (obj, "timeout");
if (elt) {
cache_ctx->timeout = ucl_object_todouble (elt);
}
else {
cache_ctx->timeout = REDIS_DEFAULT_TIMEOUT;
}
elt = ucl_object_lookup (obj, "password");
if (elt) {
cache_ctx->password = ucl_object_tostring (elt);
}
else {
cache_ctx->password = NULL;
}
elt = ucl_object_lookup_any (obj, "db", "database", "dbname", NULL);
if (elt) {
cache_ctx->dbname = ucl_object_tostring (elt);
}
else {
cache_ctx->dbname = NULL;
}
elt = ucl_object_lookup_any (obj, "cache_key", "key", NULL);
if (elt == NULL || ucl_object_type (elt) != UCL_STRING) {
cache_ctx->redis_object = DEFAULT_REDIS_KEY;
}
else {
cache_ctx->redis_object = ucl_object_tostring (elt);
}
return TRUE;
}
static gboolean
rspamd_redis_try_ucl (struct redis_stat_ctx *backend,
const ucl_object_t *obj,
struct rspamd_config *cfg,
const gchar *symbol)
{
const ucl_object_t *elt, *relt, *users_enabled;
const gchar *lua_script;
elt = ucl_object_lookup_any (obj, "read_servers", "servers", NULL);
if (elt == NULL) {
return FALSE;
}
backend->read_servers = rspamd_upstreams_create (cfg->ups_ctx);
if (!rspamd_upstreams_from_ucl (backend->read_servers, elt,
REDIS_DEFAULT_PORT, NULL)) {
msg_err ("statfile %s cannot get read servers configuration",
symbol);
return FALSE;
}
relt = elt;
elt = ucl_object_lookup (obj, "write_servers");
if (elt == NULL) {
/* Use read servers as write ones */
g_assert (relt != NULL);
backend->write_servers = rspamd_upstreams_create (cfg->ups_ctx);
if (!rspamd_upstreams_from_ucl (backend->write_servers, relt,
REDIS_DEFAULT_PORT, NULL)) {
msg_err ("statfile %s cannot get write servers configuration",
symbol);
return FALSE;
}
}
else {
backend->write_servers = rspamd_upstreams_create (cfg->ups_ctx);
if (!rspamd_upstreams_from_ucl (backend->write_servers, elt,
REDIS_DEFAULT_PORT, NULL)) {
msg_err ("statfile %s cannot get write servers configuration",
symbol);
rspamd_upstreams_destroy (backend->write_servers);
backend->write_servers = NULL;
}
}
elt = ucl_object_lookup (obj, "prefix");
if (elt == NULL || ucl_object_type (elt) != UCL_STRING) {
/* Default non-users statistics */
backend->redis_object = REDIS_DEFAULT_OBJECT;
/*
* Make redis backend compatible with sqlite3 backend in users settings
*/
users_enabled = ucl_object_lookup_any (obj, "per_user",
"users_enabled", NULL);
if (users_enabled != NULL) {
if (ucl_object_type (users_enabled) == UCL_BOOLEAN) {
backend->enable_users = ucl_object_toboolean (users_enabled);
backend->cbref_user = -1;
if (backend->enable_users) {
backend->redis_object = REDIS_DEFAULT_USERS_OBJECT;
}
}
else if (ucl_object_type (users_enabled) == UCL_STRING) {
lua_script = ucl_object_tostring (users_enabled);
if (luaL_dostring (cfg->lua_state, lua_script) != 0) {
msg_err_config ("cannot execute lua script for users "
"extraction: %s", lua_tostring (cfg->lua_state, -1));
}
else {
if (lua_type (cfg->lua_state, -1) == LUA_TFUNCTION) {
backend->enable_users = TRUE;
backend->cbref_user = luaL_ref (cfg->lua_state,
LUA_REGISTRYINDEX);
}
else {
msg_err_config ("lua script must return "
"function(task) and not %s",
lua_typename (cfg->lua_state, lua_type (
cfg->lua_state, -1)));
}
}
}
}
else {
backend->enable_users = FALSE;
}
}
else {
/* XXX: sanity check */
backend->redis_object = ucl_object_tostring (elt);
}
elt = ucl_object_lookup (obj, "timeout");
if (elt) {
backend->timeout = ucl_object_todouble (elt);
}
else {
backend->timeout = REDIS_DEFAULT_TIMEOUT;
}
elt = ucl_object_lookup (obj, "password");
if (elt) {
backend->password = ucl_object_tostring (elt);
}
else {
backend->password = NULL;
}
elt = ucl_object_lookup_any (obj, "db", "database", "dbname", NULL);
if (elt) {
backend->dbname = ucl_object_tostring (elt);
}
else {
backend->dbname = NULL;
}
return TRUE;
}
void
rspamd_upstream_test_func (void)
{
struct upstream_list *ls, *nls;
struct upstream *up, *upn;
struct event_base *ev_base = event_init ();
struct rspamd_dns_resolver *resolver;
struct rspamd_config *cfg;
gint i, success = 0;
const gint assumptions = 100500;
gdouble p;
struct event ev;
struct timeval tv;
rspamd_inet_addr_t *addr, *next_addr, *paddr;
cfg = rspamd_config_new ();
cfg->dns_retransmits = 2;
cfg->dns_timeout = 0.5;
cfg->upstream_max_errors = 1;
cfg->upstream_revive_time = 0.5;
cfg->upstream_error_time = 2;
resolver = dns_resolver_init (NULL, ev_base, cfg);
rspamd_upstreams_library_config (cfg, cfg->ups_ctx, ev_base, resolver->r);
/*
* Test v4/v6 priorities
*/
nls = rspamd_upstreams_create (cfg->ups_ctx);
g_assert (rspamd_upstreams_add_upstream (nls, "127.0.0.1", 0, NULL));
up = rspamd_upstream_get (nls, RSPAMD_UPSTREAM_RANDOM, NULL, 0);
rspamd_parse_inet_address (&paddr, "127.0.0.2", 0);
g_assert (rspamd_upstream_add_addr (up, paddr));
rspamd_parse_inet_address (&paddr, "::1", 0);
g_assert (rspamd_upstream_add_addr (up, paddr));
/* Rewind to start */
addr = rspamd_upstream_addr (up);
addr = rspamd_upstream_addr (up);
/* cur should be zero here */
addr = rspamd_upstream_addr (up);
next_addr = rspamd_upstream_addr (up);
g_assert (rspamd_inet_address_get_af (addr) == AF_INET);
g_assert (rspamd_inet_address_get_af (next_addr) == AF_INET);
next_addr = rspamd_upstream_addr (up);
g_assert (rspamd_inet_address_get_af (next_addr) == AF_INET6);
next_addr = rspamd_upstream_addr (up);
g_assert (rspamd_inet_address_get_af (next_addr) == AF_INET);
next_addr = rspamd_upstream_addr (up);
g_assert (rspamd_inet_address_get_af (next_addr) == AF_INET);
next_addr = rspamd_upstream_addr (up);
g_assert (rspamd_inet_address_get_af (next_addr) == AF_INET6);
/* Test errors with IPv6 */
rspamd_upstream_fail (up);
/* Now we should have merely IPv4 addresses in rotation */
addr = rspamd_upstream_addr (up);
for (i = 0; i < 256; i++) {
next_addr = rspamd_upstream_addr (up);
g_assert (rspamd_inet_address_get_af (addr) == AF_INET);
g_assert (rspamd_inet_address_get_af (next_addr) == AF_INET);
g_assert (rspamd_inet_address_compare (addr, next_addr) != 0);
addr = next_addr;
}
rspamd_upstreams_destroy (nls);
ls = rspamd_upstreams_create (cfg->ups_ctx);
g_assert (rspamd_upstreams_parse_line (ls, test_upstream_list, 443, NULL));
g_assert (rspamd_upstreams_count (ls) == 3);
/* Test master-slave rotation */
rspamd_upstream_test_method (ls, RSPAMD_UPSTREAM_MASTER_SLAVE, "kernel.org");
rspamd_upstream_test_method (ls, RSPAMD_UPSTREAM_MASTER_SLAVE, "kernel.org");
/* Test round-robin rotation */
rspamd_upstream_test_method (ls, RSPAMD_UPSTREAM_ROUND_ROBIN, "kernel.org");
rspamd_upstream_test_method (ls, RSPAMD_UPSTREAM_ROUND_ROBIN, "kernel.org");
rspamd_upstream_test_method (ls, RSPAMD_UPSTREAM_ROUND_ROBIN, "google.com");
rspamd_upstream_test_method (ls, RSPAMD_UPSTREAM_ROUND_ROBIN, "kernel.org");
rspamd_upstream_test_method (ls, RSPAMD_UPSTREAM_ROUND_ROBIN, "google.com");
rspamd_upstream_test_method (ls, RSPAMD_UPSTREAM_ROUND_ROBIN, "microsoft.com");
/* Test stable hashing */
nls = rspamd_upstreams_create (cfg->ups_ctx);
g_assert (rspamd_upstreams_parse_line (nls, test_upstream_list, 443, NULL));
g_assert (rspamd_upstreams_parse_line (nls, new_upstream_list, 443, NULL));
for (i = 0; i < assumptions; i ++) {
ottery_rand_bytes (test_key, sizeof (test_key));
up = rspamd_upstream_get (ls, RSPAMD_UPSTREAM_HASHED, test_key,
sizeof (test_key));
upn = rspamd_upstream_get (nls, RSPAMD_UPSTREAM_HASHED, test_key,
sizeof (test_key));
if (strcmp (rspamd_upstream_name (up), rspamd_upstream_name (upn)) == 0) {
success ++;
}
}
p = 1.0 - fabs (3.0 / 4.0 - (gdouble)success / (gdouble)assumptions);
/*
* P value is calculated as following:
* when we add/remove M upstreams from the list, the probability of hash
* miss should be close to the relation N / (N + M), where N is the size of
* the previous upstreams list.
*/
msg_debug ("p value for hash consistency: %.6f", p);
g_assert (p > 0.9);
rspamd_upstreams_destroy (nls);
/* Upstream fail test */
evtimer_set (&ev, rspamd_upstream_timeout_handler, resolver);
event_base_set (ev_base, &ev);
up = rspamd_upstream_get (ls, RSPAMD_UPSTREAM_MASTER_SLAVE, NULL, 0);
for (i = 0; i < 100; i ++) {
rspamd_upstream_fail (up);
}
g_assert (rspamd_upstreams_alive (ls) == 2);
tv.tv_sec = 2;
tv.tv_usec = 0;
event_add (&ev, &tv);
event_base_loop (ev_base, 0);
g_assert (rspamd_upstreams_alive (ls) == 3);
rspamd_upstreams_destroy (ls);
REF_RELEASE (cfg);
}
static gint
fuzzy_parse_rule (struct rspamd_config *cfg, const ucl_object_t *obj, gint cb_id)
{
const ucl_object_t *value, *cur;
struct fuzzy_rule *rule;
ucl_object_iter_t it = NULL;
const char *k = NULL;
if (obj->type != UCL_OBJECT) {
msg_err_config ("invalid rule definition");
return -1;
}
rule = fuzzy_rule_new (fuzzy_module_ctx->default_symbol,
fuzzy_module_ctx->fuzzy_pool);
if ((value = ucl_object_find_key (obj, "mime_types")) != NULL) {
it = NULL;
while ((cur = ucl_iterate_object (value, &it, value->type == UCL_ARRAY))
!= NULL) {
rule->mime_types = g_list_concat (rule->mime_types,
parse_mime_types (ucl_obj_tostring (cur)));
}
}
if (rule->mime_types != NULL) {
rspamd_mempool_add_destructor (fuzzy_module_ctx->fuzzy_pool,
(rspamd_mempool_destruct_t)g_list_free, rule->mime_types);
}
if ((value = ucl_object_find_key (obj, "max_score")) != NULL) {
rule->max_score = ucl_obj_todouble (value);
}
if ((value = ucl_object_find_key (obj, "symbol")) != NULL) {
rule->symbol = ucl_obj_tostring (value);
}
if ((value = ucl_object_find_key (obj, "read_only")) != NULL) {
rule->read_only = ucl_obj_toboolean (value);
}
if ((value = ucl_object_find_key (obj, "skip_unknown")) != NULL) {
rule->skip_unknown = ucl_obj_toboolean (value);
}
if ((value = ucl_object_find_key (obj, "servers")) != NULL) {
rule->servers = rspamd_upstreams_create ();
rspamd_mempool_add_destructor (fuzzy_module_ctx->fuzzy_pool,
(rspamd_mempool_destruct_t)rspamd_upstreams_destroy,
rule->servers);
rspamd_upstreams_from_ucl (rule->servers, value, DEFAULT_PORT, NULL);
}
if ((value = ucl_object_find_key (obj, "fuzzy_map")) != NULL) {
it = NULL;
while ((cur = ucl_iterate_object (value, &it, true)) != NULL) {
parse_flags (rule, cfg, cur, cb_id);
}
}
if ((value = ucl_object_find_key (obj, "encryption_key")) != NULL) {
/* Create key from user's input */
k = ucl_object_tostring (value);
if (k == NULL || (rule->peer_key =
rspamd_http_connection_make_peer_key (k)) == NULL) {
msg_err_config ("bad encryption key value: %s",
k);
return -1;
}
rule->local_key = rspamd_http_connection_gen_key ();
}
if ((value = ucl_object_find_key (obj, "fuzzy_key")) != NULL) {
/* Create key from user's input */
k = ucl_object_tostring (value);
}
/* Setup keys */
if (k == NULL) {
/* Use some default key for all ops */
k = "rspamd";
}
rule->hash_key = g_string_sized_new (BLAKE2B_KEYBYTES);
blake2 (rule->hash_key->str, k, NULL, BLAKE2B_KEYBYTES, strlen (k), 0);
rule->hash_key->len = BLAKE2B_KEYBYTES;
if ((value = ucl_object_find_key (obj, "fuzzy_shingles_key")) != NULL) {
k = ucl_object_tostring (value);
}
if (k == NULL) {
k = "rspamd";
}
rule->shingles_key = g_string_sized_new (16);
blake2 (rule->shingles_key->str, k, NULL, 16, strlen (k), 0);
rule->shingles_key->len = 16;
if (rspamd_upstreams_count (rule->servers) == 0) {
msg_err_config ("no servers defined for fuzzy rule with symbol: %s",
rule->symbol);
return -1;
}
else {
fuzzy_module_ctx->fuzzy_rules = g_list_prepend (
fuzzy_module_ctx->fuzzy_rules,
rule);
if (rule->symbol != fuzzy_module_ctx->default_symbol) {
rspamd_symbols_cache_add_symbol (cfg->cache, rule->symbol,
0,
NULL, NULL,
SYMBOL_TYPE_VIRTUAL|SYMBOL_TYPE_FINE,
cb_id);
}
}
rspamd_mempool_add_destructor (fuzzy_module_ctx->fuzzy_pool, fuzzy_free_rule,
rule);
return 0;
}