gpointer
rspamd_redis_runtime (struct rspamd_task *task,
struct rspamd_statfile_config *stcf,
gboolean learn, gpointer c)
{
struct redis_stat_ctx *ctx = REDIS_CTX (c);
struct redis_stat_runtime *rt;
struct upstream *up;
rspamd_inet_addr_t *addr;
g_assert (ctx != NULL);
g_assert (stcf != NULL);
if (learn && ctx->write_servers == NULL) {
msg_err_task ("no write servers defined for %s, cannot learn", stcf->symbol);
return NULL;
}
if (learn) {
up = rspamd_upstream_get (ctx->write_servers,
RSPAMD_UPSTREAM_MASTER_SLAVE,
NULL,
0);
}
else {
up = rspamd_upstream_get (ctx->read_servers,
RSPAMD_UPSTREAM_ROUND_ROBIN,
NULL,
0);
}
if (up == NULL) {
msg_err_task ("no upstreams reachable");
return NULL;
}
rt = rspamd_mempool_alloc0 (task->task_pool, sizeof (*rt));
rspamd_redis_expand_object (ctx->redis_object, ctx, task,
&rt->redis_object_expanded);
rt->selected = up;
rt->task = task;
rt->ctx = ctx;
rt->stcf = stcf;
addr = rspamd_upstream_addr (up);
g_assert (addr != NULL);
rt->redis = redisAsyncConnect (rspamd_inet_address_to_string (addr),
rspamd_inet_address_get_port (addr));
if (rt->redis == NULL) {
msg_err_task ("cannot connect redis");
return NULL;
}
redisLibeventAttach (rt->redis, task->ev_base);
rspamd_redis_maybe_auth (ctx, rt->redis);
return rt;
}
static void
rspamd_upstream_test_method (struct upstream_list *ls,
enum rspamd_upstream_rotation rot, const gchar *expected)
{
struct upstream *up;
if (rot != RSPAMD_UPSTREAM_HASHED) {
up = rspamd_upstream_get (ls, rot, NULL, 0);
g_assert (up != NULL);
g_assert (strcmp (rspamd_upstream_name (up), expected) == 0);
}
else {
up = rspamd_upstream_get (ls, RSPAMD_UPSTREAM_HASHED, test_key,
sizeof (test_key));
g_assert (up != NULL);
g_assert (strcmp (rspamd_upstream_name (up), expected) == 0);
}
}
static void
rspamd_redis_async_stat_cb (struct rspamd_stat_async_elt *elt, gpointer d)
{
struct redis_stat_ctx *ctx;
struct rspamd_redis_stat_elt *redis_elt = elt->ud;
struct rspamd_redis_stat_cbdata *cbdata;
rspamd_inet_addr_t *addr;
g_assert (redis_elt != NULL);
ctx = redis_elt->ctx;
if (redis_elt->cbdata) {
/* We have some other process pending */
rspamd_redis_async_cbdata_cleanup (redis_elt->cbdata);
}
/* Disable further events unless needed */
elt->enabled = FALSE;
cbdata = g_slice_alloc0 (sizeof (*cbdata));
cbdata->selected = rspamd_upstream_get (ctx->read_servers,
RSPAMD_UPSTREAM_ROUND_ROBIN,
NULL,
0);
g_assert (cbdata->selected != NULL);
addr = rspamd_upstream_addr (cbdata->selected);
g_assert (addr != NULL);
cbdata->redis = redisAsyncConnect (rspamd_inet_address_to_string (addr),
rspamd_inet_address_get_port (addr));
g_assert (cbdata->redis != NULL);
redisLibeventAttach (cbdata->redis, redis_elt->ev_base);
cbdata->inflight = 1;
cbdata->cur = ucl_object_typed_new (UCL_OBJECT);
cbdata->elt = redis_elt;
cbdata->cur_keys = g_ptr_array_new ();
redis_elt->cbdata = cbdata;
/* XXX: deal with timeouts maybe */
/* Get keys in redis that match our symbol */
rspamd_redis_maybe_auth (ctx, cbdata->redis);
redisAsyncCommand (cbdata->redis, rspamd_redis_stat_keys, cbdata,
"SMEMBERS %s_keys",
ctx->stcf->symbol);
}
gboolean
create_smtp_upstream_connection (struct smtp_session *session)
{
struct upstream *selected;
/* Try to select upstream */
selected = rspamd_upstream_get (session->ctx->upstreams,
RSPAMD_UPSTREAM_ROUND_ROBIN);
if (selected == NULL) {
msg_err ("no upstreams suitable found");
return FALSE;
}
session->upstream = selected;
/* Now try to create socket */
session->upstream_sock = rspamd_inet_address_connect (
rspamd_upstream_addr (selected), SOCK_STREAM, TRUE);
if (session->upstream_sock == -1) {
msg_err ("cannot make a connection to %s", rspamd_upstream_name (selected));
rspamd_upstream_fail (selected);
return FALSE;
}
/* Create a dispatcher for upstream connection */
session->upstream_dispatcher = rspamd_create_dispatcher (session->ev_base,
session->upstream_sock,
BUFFER_LINE,
smtp_upstream_read_socket,
smtp_upstream_write_socket,
smtp_upstream_err_socket,
&session->ctx->smtp_timeout,
session);
session->state = SMTP_STATE_WAIT_UPSTREAM;
session->upstream_state = SMTP_STATE_GREETING;
register_async_event (session->s,
(event_finalizer_t)smtp_upstream_finalize_connection,
session,
g_quark_from_static_string ("smtp proxy"));
return TRUE;
}
void
rspamd_fuzzy_backend_count_redis (struct rspamd_fuzzy_backend *bk,
rspamd_fuzzy_count_cb cb, void *ud,
void *subr_ud)
{
struct rspamd_fuzzy_backend_redis *backend = subr_ud;
struct rspamd_fuzzy_redis_session *session;
struct upstream *up;
struct timeval tv;
rspamd_inet_addr_t *addr;
g_assert (backend != NULL);
session = g_slice_alloc0 (sizeof (*session));
session->backend = backend;
REF_RETAIN (session->backend);
session->callback.cb_count = cb;
session->cbdata = ud;
session->command = RSPAMD_FUZZY_REDIS_COMMAND_COUNT;
session->nargs = 2;
session->argv = g_malloc (sizeof (gchar *) * 2);
session->argv[0] = g_strdup ("HLEN");
session->argv[1] = g_strdup (backend->redis_object);
up = rspamd_upstream_get (backend->read_servers,
RSPAMD_UPSTREAM_ROUND_ROBIN,
NULL,
0);
session->up = up;
addr = rspamd_upstream_addr (up);
g_assert (addr != NULL);
session->ctx = rspamd_redis_pool_connect (backend->pool,
backend->dbname, backend->password,
rspamd_inet_address_to_string (addr),
rspamd_inet_address_get_port (addr));
if (session->ctx == NULL) {
rspamd_fuzzy_redis_session_dtor (session);
if (cb) {
cb (0, subr_ud);
}
}
else {
if (redisAsyncCommandArgv (session->ctx, rspamd_fuzzy_redis_count_callback,
session, session->nargs,
(const gchar **)session->argv, NULL) != REDIS_OK) {
rspamd_fuzzy_redis_session_dtor (session);
if (cb) {
cb (0, subr_ud);
}
}
else {
/* Add timeout */
event_set (&session->timeout, -1, EV_TIMEOUT, rspamd_fuzzy_redis_timeout,
session);
event_base_set (rspamd_fuzzy_backend_event_base (bk),
&session->timeout);
double_to_tv (backend->timeout, &tv);
event_add (&session->timeout, &tv);
}
}
}
gpointer
rspamd_stat_cache_redis_runtime (struct rspamd_task *task,
gpointer c, gboolean learn)
{
struct rspamd_redis_cache_ctx *ctx = c;
struct rspamd_redis_cache_runtime *rt;
struct upstream *up;
rspamd_inet_addr_t *addr;
g_assert (ctx != NULL);
if (learn && ctx->write_servers == NULL) {
msg_err_task ("no write servers defined for %s, cannot learn",
ctx->stcf->symbol);
return NULL;
}
if (task->tokens == NULL || task->tokens->len == 0) {
return NULL;
}
if (learn) {
up = rspamd_upstream_get (ctx->write_servers,
RSPAMD_UPSTREAM_MASTER_SLAVE,
NULL,
0);
}
else {
up = rspamd_upstream_get (ctx->read_servers,
RSPAMD_UPSTREAM_ROUND_ROBIN,
NULL,
0);
}
if (up == NULL) {
msg_err_task ("no upstreams reachable");
return NULL;
}
rt = rspamd_mempool_alloc0 (task->task_pool, sizeof (*rt));
rt->selected = up;
rt->task = task;
rt->ctx = ctx;
addr = rspamd_upstream_addr (up);
g_assert (addr != NULL);
rt->redis = redisAsyncConnect (rspamd_inet_address_to_string (addr),
rspamd_inet_address_get_port (addr));
g_assert (rt->redis != NULL);
redisLibeventAttach (rt->redis, task->ev_base);
/* Now check stats */
event_set (&rt->timeout_event, -1, EV_TIMEOUT, rspamd_redis_cache_timeout, rt);
event_base_set (task->ev_base, &rt->timeout_event);
rspamd_redis_cache_maybe_auth (ctx, rt->redis);
if (!learn) {
rspamd_stat_cache_redis_generate_id (task);
}
return rt;
}
void
rspamd_upstream_test_func (void)
{
struct upstream_list *ls, *nls;
struct upstream *up, *upn;
struct event_base *ev_base = event_init ();
struct rspamd_dns_resolver *resolver;
struct rspamd_config *cfg;
gint i, success = 0;
const gint assumptions = 100500;
gdouble p;
struct event ev;
struct timeval tv;
rspamd_inet_addr_t *addr, *next_addr, *paddr;
cfg = rspamd_config_new ();
cfg->dns_retransmits = 2;
cfg->dns_timeout = 0.5;
cfg->upstream_max_errors = 1;
cfg->upstream_revive_time = 0.5;
cfg->upstream_error_time = 2;
resolver = dns_resolver_init (NULL, ev_base, cfg);
rspamd_upstreams_library_config (cfg, cfg->ups_ctx, ev_base, resolver->r);
/*
* Test v4/v6 priorities
*/
nls = rspamd_upstreams_create (cfg->ups_ctx);
g_assert (rspamd_upstreams_add_upstream (nls, "127.0.0.1", 0, NULL));
up = rspamd_upstream_get (nls, RSPAMD_UPSTREAM_RANDOM, NULL, 0);
rspamd_parse_inet_address (&paddr, "127.0.0.2", 0);
g_assert (rspamd_upstream_add_addr (up, paddr));
rspamd_parse_inet_address (&paddr, "::1", 0);
g_assert (rspamd_upstream_add_addr (up, paddr));
/* Rewind to start */
addr = rspamd_upstream_addr (up);
addr = rspamd_upstream_addr (up);
/* cur should be zero here */
addr = rspamd_upstream_addr (up);
next_addr = rspamd_upstream_addr (up);
g_assert (rspamd_inet_address_get_af (addr) == AF_INET);
g_assert (rspamd_inet_address_get_af (next_addr) == AF_INET);
next_addr = rspamd_upstream_addr (up);
g_assert (rspamd_inet_address_get_af (next_addr) == AF_INET6);
next_addr = rspamd_upstream_addr (up);
g_assert (rspamd_inet_address_get_af (next_addr) == AF_INET);
next_addr = rspamd_upstream_addr (up);
g_assert (rspamd_inet_address_get_af (next_addr) == AF_INET);
next_addr = rspamd_upstream_addr (up);
g_assert (rspamd_inet_address_get_af (next_addr) == AF_INET6);
/* Test errors with IPv6 */
rspamd_upstream_fail (up);
/* Now we should have merely IPv4 addresses in rotation */
addr = rspamd_upstream_addr (up);
for (i = 0; i < 256; i++) {
next_addr = rspamd_upstream_addr (up);
g_assert (rspamd_inet_address_get_af (addr) == AF_INET);
g_assert (rspamd_inet_address_get_af (next_addr) == AF_INET);
g_assert (rspamd_inet_address_compare (addr, next_addr) != 0);
addr = next_addr;
}
rspamd_upstreams_destroy (nls);
ls = rspamd_upstreams_create (cfg->ups_ctx);
g_assert (rspamd_upstreams_parse_line (ls, test_upstream_list, 443, NULL));
g_assert (rspamd_upstreams_count (ls) == 3);
/* Test master-slave rotation */
rspamd_upstream_test_method (ls, RSPAMD_UPSTREAM_MASTER_SLAVE, "kernel.org");
rspamd_upstream_test_method (ls, RSPAMD_UPSTREAM_MASTER_SLAVE, "kernel.org");
/* Test round-robin rotation */
rspamd_upstream_test_method (ls, RSPAMD_UPSTREAM_ROUND_ROBIN, "kernel.org");
rspamd_upstream_test_method (ls, RSPAMD_UPSTREAM_ROUND_ROBIN, "kernel.org");
rspamd_upstream_test_method (ls, RSPAMD_UPSTREAM_ROUND_ROBIN, "google.com");
rspamd_upstream_test_method (ls, RSPAMD_UPSTREAM_ROUND_ROBIN, "kernel.org");
rspamd_upstream_test_method (ls, RSPAMD_UPSTREAM_ROUND_ROBIN, "google.com");
rspamd_upstream_test_method (ls, RSPAMD_UPSTREAM_ROUND_ROBIN, "microsoft.com");
/* Test stable hashing */
nls = rspamd_upstreams_create (cfg->ups_ctx);
g_assert (rspamd_upstreams_parse_line (nls, test_upstream_list, 443, NULL));
g_assert (rspamd_upstreams_parse_line (nls, new_upstream_list, 443, NULL));
for (i = 0; i < assumptions; i ++) {
ottery_rand_bytes (test_key, sizeof (test_key));
up = rspamd_upstream_get (ls, RSPAMD_UPSTREAM_HASHED, test_key,
sizeof (test_key));
upn = rspamd_upstream_get (nls, RSPAMD_UPSTREAM_HASHED, test_key,
sizeof (test_key));
if (strcmp (rspamd_upstream_name (up), rspamd_upstream_name (upn)) == 0) {
success ++;
}
}
p = 1.0 - fabs (3.0 / 4.0 - (gdouble)success / (gdouble)assumptions);
/*
* P value is calculated as following:
* when we add/remove M upstreams from the list, the probability of hash
* miss should be close to the relation N / (N + M), where N is the size of
* the previous upstreams list.
*/
msg_debug ("p value for hash consistency: %.6f", p);
g_assert (p > 0.9);
rspamd_upstreams_destroy (nls);
/* Upstream fail test */
evtimer_set (&ev, rspamd_upstream_timeout_handler, resolver);
event_base_set (ev_base, &ev);
up = rspamd_upstream_get (ls, RSPAMD_UPSTREAM_MASTER_SLAVE, NULL, 0);
for (i = 0; i < 100; i ++) {
rspamd_upstream_fail (up);
}
g_assert (rspamd_upstreams_alive (ls) == 2);
tv.tv_sec = 2;
tv.tv_usec = 0;
event_add (&ev, &tv);
event_base_loop (ev_base, 0);
g_assert (rspamd_upstreams_alive (ls) == 3);
rspamd_upstreams_destroy (ls);
REF_RELEASE (cfg);
}
gboolean
rspamd_redis_learn_tokens (struct rspamd_task *task, GPtrArray *tokens,
gint id, gpointer p)
{
struct redis_stat_runtime *rt = REDIS_RUNTIME (p);
struct upstream *up;
rspamd_inet_addr_t *addr;
struct timeval tv;
rspamd_fstring_t *query;
const gchar *redis_cmd;
rspamd_token_t *tok;
gint ret;
up = rspamd_upstream_get (rt->ctx->write_servers,
RSPAMD_UPSTREAM_MASTER_SLAVE,
NULL,
0);
if (up == NULL) {
msg_err_task ("no upstreams reachable");
return FALSE;
}
rt->selected = up;
addr = rspamd_upstream_addr (up);
g_assert (addr != NULL);
rt->redis = redisAsyncConnect (rspamd_inet_address_to_string (addr),
rspamd_inet_address_get_port (addr));
g_assert (rt->redis != NULL);
redisLibeventAttach (rt->redis, task->ev_base);
rspamd_redis_maybe_auth (rt->ctx, rt->redis);
/*
* Add the current key to the set of learned keys
*/
redisAsyncCommand (rt->redis, NULL, NULL, "SADD %s_keys %s",
rt->stcf->symbol, rt->redis_object_expanded);
if (rt->stcf->clcf->flags & RSPAMD_FLAG_CLASSIFIER_INTEGER) {
redis_cmd = "HINCRBY";
}
else {
redis_cmd = "HINCRBYFLOAT";
}
rt->id = id;
query = rspamd_redis_tokens_to_query (task, tokens,
redis_cmd, rt->redis_object_expanded, TRUE, id,
rt->stcf->clcf->flags & RSPAMD_FLAG_CLASSIFIER_INTEGER);
g_assert (query != NULL);
/*
* XXX:
* Dirty hack: we get a token and check if it's value is -1 or 1, so
* we could understand that we are learning or unlearning
*/
tok = g_ptr_array_index (task->tokens, 0);
if (tok->values[id] > 0) {
rspamd_printf_fstring (&query, ""
"*4\r\n"
"$7\r\n"
"HINCRBY\r\n"
"$%d\r\n"
"%s\r\n"
"$6\r\n"
"learns\r\n"
"$1\r\n"
"1\r\n",
(gint)strlen (rt->redis_object_expanded),
rt->redis_object_expanded);
}
else {
rspamd_printf_fstring (&query, ""
"*4\r\n"
"$7\r\n"
"HINCRBY\r\n"
"$%d\r\n"
"%s\r\n"
"$6\r\n"
"learns\r\n"
"$2\r\n"
"-1\r\n",
(gint)strlen (rt->redis_object_expanded),
rt->redis_object_expanded);
}
rspamd_mempool_add_destructor (task->task_pool,
(rspamd_mempool_destruct_t)rspamd_fstring_free, query);
ret = redisAsyncFormattedCommand (rt->redis, rspamd_redis_learned, rt,
query->str, query->len);
if (ret == REDIS_OK) {
rspamd_session_add_event (task->s, rspamd_redis_fin_learn, rt,
rspamd_redis_stat_quark ());
rt->has_event = TRUE;
/* Set timeout */
if (event_get_base (&rt->timeout_event)) {
event_del (&rt->timeout_event);
}
event_set (&rt->timeout_event, -1, EV_TIMEOUT, rspamd_redis_timeout, rt);
event_base_set (task->ev_base, &rt->timeout_event);
double_to_tv (rt->ctx->timeout, &tv);
event_add (&rt->timeout_event, &tv);
return TRUE;
}
else {
msg_err_task ("call to redis failed: %s", rt->redis->errstr);
}
return FALSE;
}
