gpointer rspamd_redis_runtime (struct rspamd_task *task, struct rspamd_statfile_config *stcf, gboolean learn, gpointer c) { struct redis_stat_ctx *ctx = REDIS_CTX (c); struct redis_stat_runtime *rt; struct upstream *up; rspamd_inet_addr_t *addr; g_assert (ctx != NULL); g_assert (stcf != NULL); if (learn && ctx->write_servers == NULL) { msg_err_task ("no write servers defined for %s, cannot learn", stcf->symbol); return NULL; } if (learn) { up = rspamd_upstream_get (ctx->write_servers, RSPAMD_UPSTREAM_MASTER_SLAVE, NULL, 0); } else { up = rspamd_upstream_get (ctx->read_servers, RSPAMD_UPSTREAM_ROUND_ROBIN, NULL, 0); } if (up == NULL) { msg_err_task ("no upstreams reachable"); return NULL; } rt = rspamd_mempool_alloc0 (task->task_pool, sizeof (*rt)); rspamd_redis_expand_object (ctx->redis_object, ctx, task, &rt->redis_object_expanded); rt->selected = up; rt->task = task; rt->ctx = ctx; rt->stcf = stcf; addr = rspamd_upstream_addr (up); g_assert (addr != NULL); rt->redis = redisAsyncConnect (rspamd_inet_address_to_string (addr), rspamd_inet_address_get_port (addr)); if (rt->redis == NULL) { msg_err_task ("cannot connect redis"); return NULL; } redisLibeventAttach (rt->redis, task->ev_base); rspamd_redis_maybe_auth (ctx, rt->redis); return rt; }
static void rspamd_upstream_test_method (struct upstream_list *ls, enum rspamd_upstream_rotation rot, const gchar *expected) { struct upstream *up; if (rot != RSPAMD_UPSTREAM_HASHED) { up = rspamd_upstream_get (ls, rot, NULL, 0); g_assert (up != NULL); g_assert (strcmp (rspamd_upstream_name (up), expected) == 0); } else { up = rspamd_upstream_get (ls, RSPAMD_UPSTREAM_HASHED, test_key, sizeof (test_key)); g_assert (up != NULL); g_assert (strcmp (rspamd_upstream_name (up), expected) == 0); } }
static void rspamd_redis_async_stat_cb (struct rspamd_stat_async_elt *elt, gpointer d) { struct redis_stat_ctx *ctx; struct rspamd_redis_stat_elt *redis_elt = elt->ud; struct rspamd_redis_stat_cbdata *cbdata; rspamd_inet_addr_t *addr; g_assert (redis_elt != NULL); ctx = redis_elt->ctx; if (redis_elt->cbdata) { /* We have some other process pending */ rspamd_redis_async_cbdata_cleanup (redis_elt->cbdata); } /* Disable further events unless needed */ elt->enabled = FALSE; cbdata = g_slice_alloc0 (sizeof (*cbdata)); cbdata->selected = rspamd_upstream_get (ctx->read_servers, RSPAMD_UPSTREAM_ROUND_ROBIN, NULL, 0); g_assert (cbdata->selected != NULL); addr = rspamd_upstream_addr (cbdata->selected); g_assert (addr != NULL); cbdata->redis = redisAsyncConnect (rspamd_inet_address_to_string (addr), rspamd_inet_address_get_port (addr)); g_assert (cbdata->redis != NULL); redisLibeventAttach (cbdata->redis, redis_elt->ev_base); cbdata->inflight = 1; cbdata->cur = ucl_object_typed_new (UCL_OBJECT); cbdata->elt = redis_elt; cbdata->cur_keys = g_ptr_array_new (); redis_elt->cbdata = cbdata; /* XXX: deal with timeouts maybe */ /* Get keys in redis that match our symbol */ rspamd_redis_maybe_auth (ctx, cbdata->redis); redisAsyncCommand (cbdata->redis, rspamd_redis_stat_keys, cbdata, "SMEMBERS %s_keys", ctx->stcf->symbol); }
gboolean create_smtp_upstream_connection (struct smtp_session *session) { struct upstream *selected; /* Try to select upstream */ selected = rspamd_upstream_get (session->ctx->upstreams, RSPAMD_UPSTREAM_ROUND_ROBIN); if (selected == NULL) { msg_err ("no upstreams suitable found"); return FALSE; } session->upstream = selected; /* Now try to create socket */ session->upstream_sock = rspamd_inet_address_connect ( rspamd_upstream_addr (selected), SOCK_STREAM, TRUE); if (session->upstream_sock == -1) { msg_err ("cannot make a connection to %s", rspamd_upstream_name (selected)); rspamd_upstream_fail (selected); return FALSE; } /* Create a dispatcher for upstream connection */ session->upstream_dispatcher = rspamd_create_dispatcher (session->ev_base, session->upstream_sock, BUFFER_LINE, smtp_upstream_read_socket, smtp_upstream_write_socket, smtp_upstream_err_socket, &session->ctx->smtp_timeout, session); session->state = SMTP_STATE_WAIT_UPSTREAM; session->upstream_state = SMTP_STATE_GREETING; register_async_event (session->s, (event_finalizer_t)smtp_upstream_finalize_connection, session, g_quark_from_static_string ("smtp proxy")); return TRUE; }
void rspamd_fuzzy_backend_count_redis (struct rspamd_fuzzy_backend *bk, rspamd_fuzzy_count_cb cb, void *ud, void *subr_ud) { struct rspamd_fuzzy_backend_redis *backend = subr_ud; struct rspamd_fuzzy_redis_session *session; struct upstream *up; struct timeval tv; rspamd_inet_addr_t *addr; g_assert (backend != NULL); session = g_slice_alloc0 (sizeof (*session)); session->backend = backend; REF_RETAIN (session->backend); session->callback.cb_count = cb; session->cbdata = ud; session->command = RSPAMD_FUZZY_REDIS_COMMAND_COUNT; session->nargs = 2; session->argv = g_malloc (sizeof (gchar *) * 2); session->argv[0] = g_strdup ("HLEN"); session->argv[1] = g_strdup (backend->redis_object); up = rspamd_upstream_get (backend->read_servers, RSPAMD_UPSTREAM_ROUND_ROBIN, NULL, 0); session->up = up; addr = rspamd_upstream_addr (up); g_assert (addr != NULL); session->ctx = rspamd_redis_pool_connect (backend->pool, backend->dbname, backend->password, rspamd_inet_address_to_string (addr), rspamd_inet_address_get_port (addr)); if (session->ctx == NULL) { rspamd_fuzzy_redis_session_dtor (session); if (cb) { cb (0, subr_ud); } } else { if (redisAsyncCommandArgv (session->ctx, rspamd_fuzzy_redis_count_callback, session, session->nargs, (const gchar **)session->argv, NULL) != REDIS_OK) { rspamd_fuzzy_redis_session_dtor (session); if (cb) { cb (0, subr_ud); } } else { /* Add timeout */ event_set (&session->timeout, -1, EV_TIMEOUT, rspamd_fuzzy_redis_timeout, session); event_base_set (rspamd_fuzzy_backend_event_base (bk), &session->timeout); double_to_tv (backend->timeout, &tv); event_add (&session->timeout, &tv); } } }
gpointer rspamd_stat_cache_redis_runtime (struct rspamd_task *task, gpointer c, gboolean learn) { struct rspamd_redis_cache_ctx *ctx = c; struct rspamd_redis_cache_runtime *rt; struct upstream *up; rspamd_inet_addr_t *addr; g_assert (ctx != NULL); if (learn && ctx->write_servers == NULL) { msg_err_task ("no write servers defined for %s, cannot learn", ctx->stcf->symbol); return NULL; } if (task->tokens == NULL || task->tokens->len == 0) { return NULL; } if (learn) { up = rspamd_upstream_get (ctx->write_servers, RSPAMD_UPSTREAM_MASTER_SLAVE, NULL, 0); } else { up = rspamd_upstream_get (ctx->read_servers, RSPAMD_UPSTREAM_ROUND_ROBIN, NULL, 0); } if (up == NULL) { msg_err_task ("no upstreams reachable"); return NULL; } rt = rspamd_mempool_alloc0 (task->task_pool, sizeof (*rt)); rt->selected = up; rt->task = task; rt->ctx = ctx; addr = rspamd_upstream_addr (up); g_assert (addr != NULL); rt->redis = redisAsyncConnect (rspamd_inet_address_to_string (addr), rspamd_inet_address_get_port (addr)); g_assert (rt->redis != NULL); redisLibeventAttach (rt->redis, task->ev_base); /* Now check stats */ event_set (&rt->timeout_event, -1, EV_TIMEOUT, rspamd_redis_cache_timeout, rt); event_base_set (task->ev_base, &rt->timeout_event); rspamd_redis_cache_maybe_auth (ctx, rt->redis); if (!learn) { rspamd_stat_cache_redis_generate_id (task); } return rt; }
void rspamd_upstream_test_func (void) { struct upstream_list *ls, *nls; struct upstream *up, *upn; struct event_base *ev_base = event_init (); struct rspamd_dns_resolver *resolver; struct rspamd_config *cfg; gint i, success = 0; const gint assumptions = 100500; gdouble p; struct event ev; struct timeval tv; rspamd_inet_addr_t *addr, *next_addr, *paddr; cfg = rspamd_config_new (); cfg->dns_retransmits = 2; cfg->dns_timeout = 0.5; cfg->upstream_max_errors = 1; cfg->upstream_revive_time = 0.5; cfg->upstream_error_time = 2; resolver = dns_resolver_init (NULL, ev_base, cfg); rspamd_upstreams_library_config (cfg, cfg->ups_ctx, ev_base, resolver->r); /* * Test v4/v6 priorities */ nls = rspamd_upstreams_create (cfg->ups_ctx); g_assert (rspamd_upstreams_add_upstream (nls, "127.0.0.1", 0, NULL)); up = rspamd_upstream_get (nls, RSPAMD_UPSTREAM_RANDOM, NULL, 0); rspamd_parse_inet_address (&paddr, "127.0.0.2", 0); g_assert (rspamd_upstream_add_addr (up, paddr)); rspamd_parse_inet_address (&paddr, "::1", 0); g_assert (rspamd_upstream_add_addr (up, paddr)); /* Rewind to start */ addr = rspamd_upstream_addr (up); addr = rspamd_upstream_addr (up); /* cur should be zero here */ addr = rspamd_upstream_addr (up); next_addr = rspamd_upstream_addr (up); g_assert (rspamd_inet_address_get_af (addr) == AF_INET); g_assert (rspamd_inet_address_get_af (next_addr) == AF_INET); next_addr = rspamd_upstream_addr (up); g_assert (rspamd_inet_address_get_af (next_addr) == AF_INET6); next_addr = rspamd_upstream_addr (up); g_assert (rspamd_inet_address_get_af (next_addr) == AF_INET); next_addr = rspamd_upstream_addr (up); g_assert (rspamd_inet_address_get_af (next_addr) == AF_INET); next_addr = rspamd_upstream_addr (up); g_assert (rspamd_inet_address_get_af (next_addr) == AF_INET6); /* Test errors with IPv6 */ rspamd_upstream_fail (up); /* Now we should have merely IPv4 addresses in rotation */ addr = rspamd_upstream_addr (up); for (i = 0; i < 256; i++) { next_addr = rspamd_upstream_addr (up); g_assert (rspamd_inet_address_get_af (addr) == AF_INET); g_assert (rspamd_inet_address_get_af (next_addr) == AF_INET); g_assert (rspamd_inet_address_compare (addr, next_addr) != 0); addr = next_addr; } rspamd_upstreams_destroy (nls); ls = rspamd_upstreams_create (cfg->ups_ctx); g_assert (rspamd_upstreams_parse_line (ls, test_upstream_list, 443, NULL)); g_assert (rspamd_upstreams_count (ls) == 3); /* Test master-slave rotation */ rspamd_upstream_test_method (ls, RSPAMD_UPSTREAM_MASTER_SLAVE, "kernel.org"); rspamd_upstream_test_method (ls, RSPAMD_UPSTREAM_MASTER_SLAVE, "kernel.org"); /* Test round-robin rotation */ rspamd_upstream_test_method (ls, RSPAMD_UPSTREAM_ROUND_ROBIN, "kernel.org"); rspamd_upstream_test_method (ls, RSPAMD_UPSTREAM_ROUND_ROBIN, "kernel.org"); rspamd_upstream_test_method (ls, RSPAMD_UPSTREAM_ROUND_ROBIN, "google.com"); rspamd_upstream_test_method (ls, RSPAMD_UPSTREAM_ROUND_ROBIN, "kernel.org"); rspamd_upstream_test_method (ls, RSPAMD_UPSTREAM_ROUND_ROBIN, "google.com"); rspamd_upstream_test_method (ls, RSPAMD_UPSTREAM_ROUND_ROBIN, "microsoft.com"); /* Test stable hashing */ nls = rspamd_upstreams_create (cfg->ups_ctx); g_assert (rspamd_upstreams_parse_line (nls, test_upstream_list, 443, NULL)); g_assert (rspamd_upstreams_parse_line (nls, new_upstream_list, 443, NULL)); for (i = 0; i < assumptions; i ++) { ottery_rand_bytes (test_key, sizeof (test_key)); up = rspamd_upstream_get (ls, RSPAMD_UPSTREAM_HASHED, test_key, sizeof (test_key)); upn = rspamd_upstream_get (nls, RSPAMD_UPSTREAM_HASHED, test_key, sizeof (test_key)); if (strcmp (rspamd_upstream_name (up), rspamd_upstream_name (upn)) == 0) { success ++; } } p = 1.0 - fabs (3.0 / 4.0 - (gdouble)success / (gdouble)assumptions); /* * P value is calculated as following: * when we add/remove M upstreams from the list, the probability of hash * miss should be close to the relation N / (N + M), where N is the size of * the previous upstreams list. */ msg_debug ("p value for hash consistency: %.6f", p); g_assert (p > 0.9); rspamd_upstreams_destroy (nls); /* Upstream fail test */ evtimer_set (&ev, rspamd_upstream_timeout_handler, resolver); event_base_set (ev_base, &ev); up = rspamd_upstream_get (ls, RSPAMD_UPSTREAM_MASTER_SLAVE, NULL, 0); for (i = 0; i < 100; i ++) { rspamd_upstream_fail (up); } g_assert (rspamd_upstreams_alive (ls) == 2); tv.tv_sec = 2; tv.tv_usec = 0; event_add (&ev, &tv); event_base_loop (ev_base, 0); g_assert (rspamd_upstreams_alive (ls) == 3); rspamd_upstreams_destroy (ls); REF_RELEASE (cfg); }
gboolean rspamd_redis_learn_tokens (struct rspamd_task *task, GPtrArray *tokens, gint id, gpointer p) { struct redis_stat_runtime *rt = REDIS_RUNTIME (p); struct upstream *up; rspamd_inet_addr_t *addr; struct timeval tv; rspamd_fstring_t *query; const gchar *redis_cmd; rspamd_token_t *tok; gint ret; up = rspamd_upstream_get (rt->ctx->write_servers, RSPAMD_UPSTREAM_MASTER_SLAVE, NULL, 0); if (up == NULL) { msg_err_task ("no upstreams reachable"); return FALSE; } rt->selected = up; addr = rspamd_upstream_addr (up); g_assert (addr != NULL); rt->redis = redisAsyncConnect (rspamd_inet_address_to_string (addr), rspamd_inet_address_get_port (addr)); g_assert (rt->redis != NULL); redisLibeventAttach (rt->redis, task->ev_base); rspamd_redis_maybe_auth (rt->ctx, rt->redis); /* * Add the current key to the set of learned keys */ redisAsyncCommand (rt->redis, NULL, NULL, "SADD %s_keys %s", rt->stcf->symbol, rt->redis_object_expanded); if (rt->stcf->clcf->flags & RSPAMD_FLAG_CLASSIFIER_INTEGER) { redis_cmd = "HINCRBY"; } else { redis_cmd = "HINCRBYFLOAT"; } rt->id = id; query = rspamd_redis_tokens_to_query (task, tokens, redis_cmd, rt->redis_object_expanded, TRUE, id, rt->stcf->clcf->flags & RSPAMD_FLAG_CLASSIFIER_INTEGER); g_assert (query != NULL); /* * XXX: * Dirty hack: we get a token and check if it's value is -1 or 1, so * we could understand that we are learning or unlearning */ tok = g_ptr_array_index (task->tokens, 0); if (tok->values[id] > 0) { rspamd_printf_fstring (&query, "" "*4\r\n" "$7\r\n" "HINCRBY\r\n" "$%d\r\n" "%s\r\n" "$6\r\n" "learns\r\n" "$1\r\n" "1\r\n", (gint)strlen (rt->redis_object_expanded), rt->redis_object_expanded); } else { rspamd_printf_fstring (&query, "" "*4\r\n" "$7\r\n" "HINCRBY\r\n" "$%d\r\n" "%s\r\n" "$6\r\n" "learns\r\n" "$2\r\n" "-1\r\n", (gint)strlen (rt->redis_object_expanded), rt->redis_object_expanded); } rspamd_mempool_add_destructor (task->task_pool, (rspamd_mempool_destruct_t)rspamd_fstring_free, query); ret = redisAsyncFormattedCommand (rt->redis, rspamd_redis_learned, rt, query->str, query->len); if (ret == REDIS_OK) { rspamd_session_add_event (task->s, rspamd_redis_fin_learn, rt, rspamd_redis_stat_quark ()); rt->has_event = TRUE; /* Set timeout */ if (event_get_base (&rt->timeout_event)) { event_del (&rt->timeout_event); } event_set (&rt->timeout_event, -1, EV_TIMEOUT, rspamd_redis_timeout, rt); event_base_set (task->ev_base, &rt->timeout_event); double_to_tv (rt->ctx->timeout, &tv); event_add (&rt->timeout_event, &tv); return TRUE; } else { msg_err_task ("call to redis failed: %s", rt->redis->errstr); } return FALSE; }