static void
rspamd_fuzzy_redis_count_callback (redisAsyncContext *c, gpointer r,
gpointer priv)
{
struct rspamd_fuzzy_redis_session *session = priv;
redisReply *reply = r;
event_del (&session->timeout);
if (c->err == 0) {
rspamd_upstream_ok (session->up);
if (reply->type == REDIS_REPLY_INTEGER) {
if (session->callback.cb_count) {
session->callback.cb_count (reply->integer, session->cbdata);
}
}
else {
if (session->callback.cb_count) {
session->callback.cb_count (0, session->cbdata);
}
}
}
else {
if (session->callback.cb_count) {
session->callback.cb_count (0, session->cbdata);
}
rspamd_upstream_fail (session->up);
}
rspamd_fuzzy_redis_session_dtor (session);
}
/* Called when we have checked the specified message id */
static void
rspamd_stat_cache_redis_get (redisAsyncContext *c, gpointer r, gpointer priv)
{
struct rspamd_redis_cache_runtime *rt = priv;
redisReply *reply = r;
struct rspamd_task *task;
glong val = 0;
task = rt->task;
if (c->err == 0) {
if (reply) {
if (G_LIKELY (reply->type == REDIS_REPLY_INTEGER)) {
val = reply->integer;
}
else if (reply->type == REDIS_REPLY_STRING) {
rspamd_strtol (reply->str, reply->len, &val);
}
else {
if (reply->type != REDIS_REPLY_NIL) {
msg_err_task ("bad learned type for %s: %d",
rt->ctx->stcf->symbol, reply->type);
}
val = 0;
}
}
if ((val > 0 && (task->flags & RSPAMD_TASK_FLAG_LEARN_SPAM)) ||
(val < 0 && (task->flags & RSPAMD_TASK_FLAG_LEARN_HAM))) {
/* Already learned */
g_set_error (&task->err, rspamd_stat_quark (), 404,
"<%s> has been already "
"learned as %s, ignore it", task->message_id,
(task->flags & RSPAMD_TASK_FLAG_LEARN_SPAM) ? "spam" : "ham");
task->flags |= RSPAMD_TASK_FLAG_ALREADY_LEARNED;
}
else if (val != 0) {
/* Unlearn flag */
task->flags |= RSPAMD_TASK_FLAG_UNLEARN;
}
rspamd_upstream_ok (rt->selected);
}
else {
rspamd_upstream_fail (rt->selected);
}
if (rt->has_event) {
rspamd_session_remove_event (task->s, rspamd_redis_cache_fin, rt);
}
}
static void
rspamd_redis_cache_timeout (gint fd, short what, gpointer d)
{
struct rspamd_redis_cache_runtime *rt = d;
struct rspamd_task *task;
task = rt->task;
msg_err_task ("connection to redis server %s timed out",
rspamd_upstream_name (rt->selected));
rspamd_upstream_fail (rt->selected);
if (rt->has_event) {
rspamd_session_remove_event (task->s, rspamd_redis_cache_fin, d);
}
}
/* Called when we have connected to the redis server and got stats */
static void
rspamd_redis_connected (redisAsyncContext *c, gpointer r, gpointer priv)
{
struct redis_stat_runtime *rt = REDIS_RUNTIME (priv);
redisReply *reply = r;
struct rspamd_task *task;
glong val = 0;
task = rt->task;
if (c->err == 0) {
if (r != NULL) {
if (G_UNLIKELY (reply->type == REDIS_REPLY_INTEGER)) {
val = reply->integer;
}
else if (reply->type == REDIS_REPLY_STRING) {
rspamd_strtol (reply->str, reply->len, &val);
}
else {
if (reply->type != REDIS_REPLY_NIL) {
msg_err_task ("bad learned type for %s: %d",
rt->stcf->symbol, reply->type);
}
val = 0;
}
if (val < 0) {
msg_warn_task ("invalid number of learns for %s: %L",
rt->stcf->symbol, val);
val = 0;
}
rt->learned = val;
msg_debug_task ("connected to redis server, tokens learned for %s: %uL",
rt->redis_object_expanded, rt->learned);
rspamd_upstream_ok (rt->selected);
}
}
else {
msg_err_task ("error getting reply from redis server %s: %s",
rspamd_upstream_name (rt->selected), c->errstr);
rspamd_upstream_fail (rt->selected);
}
}
/* Called when we have learned the specified message id */
static void
rspamd_stat_cache_redis_set (redisAsyncContext *c, gpointer r, gpointer priv)
{
struct rspamd_redis_cache_runtime *rt = priv;
struct rspamd_task *task;
task = rt->task;
if (c->err == 0) {
/* XXX: we ignore results here */
rspamd_upstream_ok (rt->selected);
}
else {
rspamd_upstream_fail (rt->selected);
}
if (rt->has_event) {
rspamd_session_remove_event (task->s, rspamd_redis_cache_fin, rt);
}
}
static void
rspamd_redis_timeout (gint fd, short what, gpointer d)
{
struct redis_stat_runtime *rt = REDIS_RUNTIME (d);
struct rspamd_task *task;
redisAsyncContext *redis;
task = rt->task;
msg_err_task_check ("connection to redis server %s timed out",
rspamd_upstream_name (rt->selected));
rspamd_upstream_fail (rt->selected);
if (rt->redis) {
redis = rt->redis;
rt->redis = NULL;
/* This calls for all callbacks pending */
redisAsyncFree (redis);
}
}
gboolean
create_smtp_upstream_connection (struct smtp_session *session)
{
struct upstream *selected;
/* Try to select upstream */
selected = rspamd_upstream_get (session->ctx->upstreams,
RSPAMD_UPSTREAM_ROUND_ROBIN);
if (selected == NULL) {
msg_err ("no upstreams suitable found");
return FALSE;
}
session->upstream = selected;
/* Now try to create socket */
session->upstream_sock = rspamd_inet_address_connect (
rspamd_upstream_addr (selected), SOCK_STREAM, TRUE);
if (session->upstream_sock == -1) {
msg_err ("cannot make a connection to %s", rspamd_upstream_name (selected));
rspamd_upstream_fail (selected);
return FALSE;
}
/* Create a dispatcher for upstream connection */
session->upstream_dispatcher = rspamd_create_dispatcher (session->ev_base,
session->upstream_sock,
BUFFER_LINE,
smtp_upstream_read_socket,
smtp_upstream_write_socket,
smtp_upstream_err_socket,
&session->ctx->smtp_timeout,
session);
session->state = SMTP_STATE_WAIT_UPSTREAM;
session->upstream_state = SMTP_STATE_GREETING;
register_async_event (session->s,
(event_finalizer_t)smtp_upstream_finalize_connection,
session,
g_quark_from_static_string ("smtp proxy"));
return TRUE;
}
void
smtp_upstream_err_socket (GError *err, void *arg)
{
struct smtp_session *session = arg;
msg_info ("abnormally closing connection with upstream %s, error: %s",
rspamd_upstream_name (session->upstream),
err->message);
session->error = SMTP_ERROR_UPSTREAM;
session->state = SMTP_STATE_CRITICAL_ERROR;
/* XXX: assume upstream errors as critical errors */
rspamd_dispatcher_restore (session->dispatcher);
if (!rspamd_dispatcher_write (session->dispatcher, session->error, 0, FALSE,
TRUE)) {
return;
}
if (!rspamd_dispatcher_write (session->dispatcher, CRLF, sizeof (CRLF) - 1,
FALSE, TRUE)) {
return;
}
rspamd_upstream_fail (session->upstream);
rspamd_session_destroy (session->s);
}
/* Called when we have set tokens during learning */
static void
rspamd_redis_learned (redisAsyncContext *c, gpointer r, gpointer priv)
{
struct redis_stat_runtime *rt = REDIS_RUNTIME (priv);
struct rspamd_task *task;
task = rt->task;
if (c->err == 0) {
rspamd_upstream_ok (rt->selected);
}
else {
msg_err_task_check ("error getting reply from redis server %s: %s",
rspamd_upstream_name (rt->selected), c->errstr);
if (rt->redis) {
rspamd_upstream_fail (rt->selected);
}
}
if (rt->has_event) {
rspamd_session_remove_event (task->s, rspamd_redis_fin_learn, rt);
}
}
void
rspamd_upstream_test_func (void)
{
struct upstream_list *ls, *nls;
struct upstream *up, *upn;
struct event_base *ev_base = event_init ();
struct rspamd_dns_resolver *resolver;
struct rspamd_config *cfg;
gint i, success = 0;
const gint assumptions = 100500;
gdouble p;
struct event ev;
struct timeval tv;
rspamd_inet_addr_t *addr, *next_addr, *paddr;
cfg = rspamd_config_new ();
cfg->dns_retransmits = 2;
cfg->dns_timeout = 0.5;
cfg->upstream_max_errors = 1;
cfg->upstream_revive_time = 0.5;
cfg->upstream_error_time = 2;
resolver = dns_resolver_init (NULL, ev_base, cfg);
rspamd_upstreams_library_config (cfg, cfg->ups_ctx, ev_base, resolver->r);
/*
* Test v4/v6 priorities
*/
nls = rspamd_upstreams_create (cfg->ups_ctx);
g_assert (rspamd_upstreams_add_upstream (nls, "127.0.0.1", 0, NULL));
up = rspamd_upstream_get (nls, RSPAMD_UPSTREAM_RANDOM, NULL, 0);
rspamd_parse_inet_address (&paddr, "127.0.0.2", 0);
g_assert (rspamd_upstream_add_addr (up, paddr));
rspamd_parse_inet_address (&paddr, "::1", 0);
g_assert (rspamd_upstream_add_addr (up, paddr));
/* Rewind to start */
addr = rspamd_upstream_addr (up);
addr = rspamd_upstream_addr (up);
/* cur should be zero here */
addr = rspamd_upstream_addr (up);
next_addr = rspamd_upstream_addr (up);
g_assert (rspamd_inet_address_get_af (addr) == AF_INET);
g_assert (rspamd_inet_address_get_af (next_addr) == AF_INET);
next_addr = rspamd_upstream_addr (up);
g_assert (rspamd_inet_address_get_af (next_addr) == AF_INET6);
next_addr = rspamd_upstream_addr (up);
g_assert (rspamd_inet_address_get_af (next_addr) == AF_INET);
next_addr = rspamd_upstream_addr (up);
g_assert (rspamd_inet_address_get_af (next_addr) == AF_INET);
next_addr = rspamd_upstream_addr (up);
g_assert (rspamd_inet_address_get_af (next_addr) == AF_INET6);
/* Test errors with IPv6 */
rspamd_upstream_fail (up);
/* Now we should have merely IPv4 addresses in rotation */
addr = rspamd_upstream_addr (up);
for (i = 0; i < 256; i++) {
next_addr = rspamd_upstream_addr (up);
g_assert (rspamd_inet_address_get_af (addr) == AF_INET);
g_assert (rspamd_inet_address_get_af (next_addr) == AF_INET);
g_assert (rspamd_inet_address_compare (addr, next_addr) != 0);
addr = next_addr;
}
rspamd_upstreams_destroy (nls);
ls = rspamd_upstreams_create (cfg->ups_ctx);
g_assert (rspamd_upstreams_parse_line (ls, test_upstream_list, 443, NULL));
g_assert (rspamd_upstreams_count (ls) == 3);
/* Test master-slave rotation */
rspamd_upstream_test_method (ls, RSPAMD_UPSTREAM_MASTER_SLAVE, "kernel.org");
rspamd_upstream_test_method (ls, RSPAMD_UPSTREAM_MASTER_SLAVE, "kernel.org");
/* Test round-robin rotation */
rspamd_upstream_test_method (ls, RSPAMD_UPSTREAM_ROUND_ROBIN, "kernel.org");
rspamd_upstream_test_method (ls, RSPAMD_UPSTREAM_ROUND_ROBIN, "kernel.org");
rspamd_upstream_test_method (ls, RSPAMD_UPSTREAM_ROUND_ROBIN, "google.com");
rspamd_upstream_test_method (ls, RSPAMD_UPSTREAM_ROUND_ROBIN, "kernel.org");
rspamd_upstream_test_method (ls, RSPAMD_UPSTREAM_ROUND_ROBIN, "google.com");
rspamd_upstream_test_method (ls, RSPAMD_UPSTREAM_ROUND_ROBIN, "microsoft.com");
/* Test stable hashing */
nls = rspamd_upstreams_create (cfg->ups_ctx);
g_assert (rspamd_upstreams_parse_line (nls, test_upstream_list, 443, NULL));
g_assert (rspamd_upstreams_parse_line (nls, new_upstream_list, 443, NULL));
for (i = 0; i < assumptions; i ++) {
ottery_rand_bytes (test_key, sizeof (test_key));
up = rspamd_upstream_get (ls, RSPAMD_UPSTREAM_HASHED, test_key,
sizeof (test_key));
upn = rspamd_upstream_get (nls, RSPAMD_UPSTREAM_HASHED, test_key,
sizeof (test_key));
if (strcmp (rspamd_upstream_name (up), rspamd_upstream_name (upn)) == 0) {
success ++;
}
}
p = 1.0 - fabs (3.0 / 4.0 - (gdouble)success / (gdouble)assumptions);
/*
* P value is calculated as following:
* when we add/remove M upstreams from the list, the probability of hash
* miss should be close to the relation N / (N + M), where N is the size of
* the previous upstreams list.
*/
msg_debug ("p value for hash consistency: %.6f", p);
g_assert (p > 0.9);
rspamd_upstreams_destroy (nls);
/* Upstream fail test */
evtimer_set (&ev, rspamd_upstream_timeout_handler, resolver);
event_base_set (ev_base, &ev);
up = rspamd_upstream_get (ls, RSPAMD_UPSTREAM_MASTER_SLAVE, NULL, 0);
for (i = 0; i < 100; i ++) {
rspamd_upstream_fail (up);
}
g_assert (rspamd_upstreams_alive (ls) == 2);
tv.tv_sec = 2;
tv.tv_usec = 0;
event_add (&ev, &tv);
event_base_loop (ev_base, 0);
g_assert (rspamd_upstreams_alive (ls) == 3);
rspamd_upstreams_destroy (ls);
REF_RELEASE (cfg);
}
/* Called when we have received tokens values from redis */
static void
rspamd_redis_processed (redisAsyncContext *c, gpointer r, gpointer priv)
{
struct redis_stat_runtime *rt = REDIS_RUNTIME (priv);
redisReply *reply = r, *elt;
struct rspamd_task *task;
rspamd_token_t *tok;
guint i, processed = 0, found = 0;
gulong val;
gdouble float_val;
task = rt->task;
if (c->err == 0) {
if (r != NULL) {
if (reply->type == REDIS_REPLY_ARRAY) {
if (reply->elements == task->tokens->len) {
for (i = 0; i < reply->elements; i ++) {
tok = g_ptr_array_index (task->tokens, i);
elt = reply->element[i];
if (G_UNLIKELY (elt->type == REDIS_REPLY_INTEGER)) {
tok->values[rt->id] = elt->integer;
found ++;
}
else if (elt->type == REDIS_REPLY_STRING) {
if (rt->stcf->clcf->flags &
RSPAMD_FLAG_CLASSIFIER_INTEGER) {
rspamd_strtoul (elt->str, elt->len, &val);
tok->values[rt->id] = val;
}
else {
float_val = strtod (elt->str, NULL);
tok->values[rt->id] = float_val;
}
found ++;
}
else {
tok->values[rt->id] = 0;
}
processed ++;
}
if (rt->stcf->is_spam) {
task->flags |= RSPAMD_TASK_FLAG_HAS_SPAM_TOKENS;
}
else {
task->flags |= RSPAMD_TASK_FLAG_HAS_HAM_TOKENS;
}
}
else {
msg_err_task_check ("got invalid length of reply vector from redis: "
"%d, expected: %d",
(gint)reply->elements,
(gint)task->tokens->len);
}
}
else {
msg_err_task_check ("got invalid reply from redis: %d",
reply->type);
}
msg_debug_task_check ("received tokens for %s: %d processed, %d found",
rt->redis_object_expanded, processed, found);
rspamd_upstream_ok (rt->selected);
}
}
else {
msg_err_task ("error getting reply from redis server %s: %s",
rspamd_upstream_name (rt->selected), c->errstr);
if (rt->redis) {
rspamd_upstream_fail (rt->selected);
}
}
if (rt->has_event) {
rspamd_session_remove_event (task->s, rspamd_redis_fin, rt);
}
}
/* Called when we have connected to the redis server and got keys to check */
static void
rspamd_redis_stat_keys (redisAsyncContext *c, gpointer r, gpointer priv)
{
struct rspamd_redis_stat_cbdata *cbdata = priv;
redisReply *reply = r, *elt;
gchar **pk, *k;
guint i, processed = 0;
if (cbdata->wanna_die) {
return;
}
cbdata->inflight --;
if (c->err == 0 && r != NULL) {
if (reply->type == REDIS_REPLY_ARRAY) {
g_ptr_array_set_size (cbdata->cur_keys, reply->elements);
for (i = 0; i < reply->elements; i ++) {
elt = reply->element[i];
if (elt->type == REDIS_REPLY_STRING) {
pk = (gchar **)&g_ptr_array_index (cbdata->cur_keys, i);
*pk = g_malloc (elt->len + 1);
rspamd_strlcpy (*pk, elt->str, elt->len + 1);
processed ++;
}
}
if (processed) {
for (i = 0; i < cbdata->cur_keys->len; i ++) {
k = (gchar *)g_ptr_array_index (cbdata->cur_keys, i);
if (k) {
redisAsyncCommand (cbdata->redis, rspamd_redis_stat_key,
cbdata,
"HLEN %s",
k);
redisAsyncCommand (cbdata->redis, rspamd_redis_stat_learns,
cbdata,
"HGET %s learns",
k);
cbdata->inflight += 2;
}
}
}
}
/* Set up the required keys */
ucl_object_insert_key (cbdata->cur,
ucl_object_typed_new (UCL_INT), "revision", 0, false);
ucl_object_insert_key (cbdata->cur,
ucl_object_typed_new (UCL_INT), "used", 0, false);
ucl_object_insert_key (cbdata->cur,
ucl_object_typed_new (UCL_INT), "total", 0, false);
ucl_object_insert_key (cbdata->cur,
ucl_object_typed_new (UCL_INT), "size", 0, false);
ucl_object_insert_key (cbdata->cur,
ucl_object_fromstring (cbdata->elt->ctx->stcf->symbol),
"symbol", 0, false);
ucl_object_insert_key (cbdata->cur, ucl_object_fromstring ("redis"),
"type", 0, false);
ucl_object_insert_key (cbdata->cur, ucl_object_fromint (0),
"languages", 0, false);
ucl_object_insert_key (cbdata->cur, ucl_object_fromint (processed),
"users", 0, false);
rspamd_upstream_ok (cbdata->selected);
}
else {
if (c->errstr) {
msg_err ("cannot get keys to gather stat: %s", c->errstr);
}
else {
msg_err ("cannot get keys to gather stat: unknown error");
}
rspamd_upstream_fail (cbdata->selected);
rspamd_redis_async_cbdata_cleanup (cbdata);
}
if (cbdata->inflight == 0) {
rspamd_redis_async_cbdata_cleanup (cbdata);
}
}