gpointer
rspamd_redis_runtime (struct rspamd_task *task,
struct rspamd_statfile_config *stcf,
gboolean learn, gpointer c)
{
struct redis_stat_ctx *ctx = REDIS_CTX (c);
struct redis_stat_runtime *rt;
struct upstream *up;
rspamd_inet_addr_t *addr;
g_assert (ctx != NULL);
g_assert (stcf != NULL);
if (learn && ctx->write_servers == NULL) {
msg_err_task ("no write servers defined for %s, cannot learn", stcf->symbol);
return NULL;
}
if (learn) {
up = rspamd_upstream_get (ctx->write_servers,
RSPAMD_UPSTREAM_MASTER_SLAVE,
NULL,
0);
}
else {
up = rspamd_upstream_get (ctx->read_servers,
RSPAMD_UPSTREAM_ROUND_ROBIN,
NULL,
0);
}
if (up == NULL) {
msg_err_task ("no upstreams reachable");
return NULL;
}
rt = rspamd_mempool_alloc0 (task->task_pool, sizeof (*rt));
rspamd_redis_expand_object (ctx->redis_object, ctx, task,
&rt->redis_object_expanded);
rt->selected = up;
rt->task = task;
rt->ctx = ctx;
rt->stcf = stcf;
addr = rspamd_upstream_addr (up);
g_assert (addr != NULL);
rt->redis = redisAsyncConnect (rspamd_inet_address_to_string (addr),
rspamd_inet_address_get_port (addr));
if (rt->redis == NULL) {
msg_err_task ("cannot connect redis");
return NULL;
}
redisLibeventAttach (rt->redis, task->ev_base);
rspamd_redis_maybe_auth (ctx, rt->redis);
return rt;
}
static void
process_regexp_item (struct rspamd_task *task, void *user_data)
{
struct regexp_module_item *item = user_data;
gint res = FALSE;
/* Non-threaded version */
if (item->lua_function) {
/* Just call function */
res = FALSE;
if (!rspamd_lua_call_expression_func (item->lua_function, task, NULL,
&res)) {
msg_err_task ("error occurred when checking symbol %s",
item->symbol);
}
}
else {
/* Process expression */
if (item->expr) {
res = rspamd_process_expression (item->expr, 0, task);
}
else {
msg_warn_task ("FIXME: %s symbol is broken with new expressions",
item->symbol);
}
}
if (res) {
rspamd_task_insert_result (task, item->symbol, res, NULL);
}
}
gint
rspamd_re_cache_process (struct rspamd_task *task,
struct rspamd_re_runtime *rt,
rspamd_regexp_t *re,
enum rspamd_re_type type,
gpointer type_data,
gsize datalen,
gboolean is_strong)
{
guint64 re_id;
struct rspamd_re_class *re_class;
struct rspamd_re_cache *cache;
g_assert (rt != NULL);
g_assert (task != NULL);
g_assert (re != NULL);
cache = rt->cache;
re_id = rspamd_regexp_get_cache_id (re);
if (re_id == RSPAMD_INVALID_ID || re_id > cache->nre) {
msg_err_task ("re '%s' has no valid id for the cache",
rspamd_regexp_get_pattern (re));
return 0;
}
if (isset (rt->checked, re_id)) {
/* Fast path */
rt->stat.regexp_fast_cached ++;
return rt->results[re_id];
}
else {
/* Slow path */
re_class = rspamd_regexp_get_class (re);
if (re_class == NULL) {
msg_err_task ("cannot find re class for regexp '%s'",
rspamd_regexp_get_pattern (re));
return 0;
}
return rspamd_re_cache_exec_re (task, rt, re, re_class,
is_strong);
}
return 0;
}
/* Called when we have connected to the redis server and got stats */
static void
rspamd_redis_connected (redisAsyncContext *c, gpointer r, gpointer priv)
{
struct redis_stat_runtime *rt = REDIS_RUNTIME (priv);
redisReply *reply = r;
struct rspamd_task *task;
glong val = 0;
task = rt->task;
if (c->err == 0) {
if (r != NULL) {
if (G_UNLIKELY (reply->type == REDIS_REPLY_INTEGER)) {
val = reply->integer;
}
else if (reply->type == REDIS_REPLY_STRING) {
rspamd_strtol (reply->str, reply->len, &val);
}
else {
if (reply->type != REDIS_REPLY_NIL) {
msg_err_task ("bad learned type for %s: %d",
rt->stcf->symbol, reply->type);
}
val = 0;
}
if (val < 0) {
msg_warn_task ("invalid number of learns for %s: %L",
rt->stcf->symbol, val);
val = 0;
}
rt->learned = val;
msg_debug_task ("connected to redis server, tokens learned for %s: %uL",
rt->redis_object_expanded, rt->learned);
rspamd_upstream_ok (rt->selected);
}
}
else {
msg_err_task ("error getting reply from redis server %s: %s",
rspamd_upstream_name (rt->selected), c->errstr);
rspamd_upstream_fail (rt->selected);
}
}
static gboolean rspamd_lua_call_expression_func(
struct ucl_lua_funcdata *lua_data, struct rspamd_task *task,
GArray *args, gint *res)
{
lua_State *L = lua_data->L;
struct rspamd_task **ptask;
struct expression_argument *arg;
gint pop = 0, i, nargs = 0;
lua_rawgeti (L, LUA_REGISTRYINDEX, lua_data->idx);
/* Now we got function in top of stack */
ptask = lua_newuserdata (L, sizeof(struct rspamd_task *));
rspamd_lua_setclass (L, "rspamd{task}", -1);
*ptask = task;
/* Now push all arguments */
if (args) {
for (i = 0; i < (gint)args->len; i ++) {
arg = &g_array_index (args, struct expression_argument, i);
if (arg) {
switch (arg->type) {
case EXPRESSION_ARGUMENT_NORMAL:
lua_pushstring (L, (const gchar *) arg->data);
break;
case EXPRESSION_ARGUMENT_BOOL:
lua_pushboolean (L, (gboolean) GPOINTER_TO_SIZE(arg->data));
break;
default:
msg_err_task ("cannot pass custom params to lua function");
return FALSE;
}
}
}
nargs = args->len;
}
if (lua_pcall (L, nargs + 1, 1, 0) != 0) {
msg_info_task ("call to lua function failed: %s", lua_tostring (L, -1));
return FALSE;
}
pop++;
if (lua_type (L, -1) == LUA_TNUMBER) {
*res = lua_tonumber (L, -1);
}
else if (lua_type (L, -1) == LUA_TBOOLEAN) {
*res = lua_toboolean (L, -1);
}
else {
msg_info_task ("lua function must return a boolean");
}
lua_pop (L, pop);
return TRUE;
}
/* Called when we have checked the specified message id */
static void
rspamd_stat_cache_redis_get (redisAsyncContext *c, gpointer r, gpointer priv)
{
struct rspamd_redis_cache_runtime *rt = priv;
redisReply *reply = r;
struct rspamd_task *task;
glong val = 0;
task = rt->task;
if (c->err == 0) {
if (reply) {
if (G_LIKELY (reply->type == REDIS_REPLY_INTEGER)) {
val = reply->integer;
}
else if (reply->type == REDIS_REPLY_STRING) {
rspamd_strtol (reply->str, reply->len, &val);
}
else {
if (reply->type != REDIS_REPLY_NIL) {
msg_err_task ("bad learned type for %s: %d",
rt->ctx->stcf->symbol, reply->type);
}
val = 0;
}
}
if ((val > 0 && (task->flags & RSPAMD_TASK_FLAG_LEARN_SPAM)) ||
(val < 0 && (task->flags & RSPAMD_TASK_FLAG_LEARN_HAM))) {
/* Already learned */
g_set_error (&task->err, rspamd_stat_quark (), 404,
"<%s> has been already "
"learned as %s, ignore it", task->message_id,
(task->flags & RSPAMD_TASK_FLAG_LEARN_SPAM) ? "spam" : "ham");
task->flags |= RSPAMD_TASK_FLAG_ALREADY_LEARNED;
}
else if (val != 0) {
/* Unlearn flag */
task->flags |= RSPAMD_TASK_FLAG_UNLEARN;
}
rspamd_upstream_ok (rt->selected);
}
else {
rspamd_upstream_fail (rt->selected);
}
if (rt->has_event) {
rspamd_session_remove_event (task->s, rspamd_redis_cache_fin, rt);
}
}
gboolean
rspamd_redis_process_tokens (struct rspamd_task *task,
GPtrArray *tokens,
gint id, gpointer p)
{
struct redis_stat_runtime *rt = REDIS_RUNTIME (p);
rspamd_fstring_t *query;
struct timeval tv;
gint ret;
if (tokens == NULL || tokens->len == 0 || rt->redis == NULL) {
return FALSE;
}
rt->id = id;
if (redisAsyncCommand (rt->redis, rspamd_redis_connected, rt, "HGET %s %s",
rt->redis_object_expanded, "learns") == REDIS_OK) {
rspamd_session_add_event (task->s, rspamd_redis_fin, rt,
rspamd_redis_stat_quark ());
rt->has_event = TRUE;
if (event_get_base (&rt->timeout_event)) {
event_del (&rt->timeout_event);
}
event_set (&rt->timeout_event, -1, EV_TIMEOUT, rspamd_redis_timeout, rt);
event_base_set (task->ev_base, &rt->timeout_event);
double_to_tv (rt->ctx->timeout, &tv);
event_add (&rt->timeout_event, &tv);
query = rspamd_redis_tokens_to_query (task, tokens,
"HMGET", rt->redis_object_expanded, FALSE, -1,
rt->stcf->clcf->flags & RSPAMD_FLAG_CLASSIFIER_INTEGER);
g_assert (query != NULL);
rspamd_mempool_add_destructor (task->task_pool,
(rspamd_mempool_destruct_t)rspamd_fstring_free, query);
ret = redisAsyncFormattedCommand (rt->redis, rspamd_redis_processed, rt,
query->str, query->len);
if (ret == REDIS_OK) {
return TRUE;
}
else {
msg_err_task ("call to redis failed: %s", rt->redis->errstr);
}
}
return FALSE;
}
static void
rspamd_redis_cache_timeout (gint fd, short what, gpointer d)
{
struct rspamd_redis_cache_runtime *rt = d;
struct rspamd_task *task;
task = rt->task;
msg_err_task ("connection to redis server %s timed out",
rspamd_upstream_name (rt->selected));
rspamd_upstream_fail (rt->selected);
if (rt->has_event) {
rspamd_session_remove_event (task->s, rspamd_redis_cache_fin, d);
}
}
/*
* Calculates the specified regexp for the specified class if it's not calculated
*/
static guint
rspamd_re_cache_exec_re (struct rspamd_task *task,
struct rspamd_re_runtime *rt,
rspamd_regexp_t *re,
struct rspamd_re_class *re_class,
gboolean is_strong)
{
guint ret = 0, i, re_id;
GPtrArray *headerlist;
GHashTableIter it;
struct raw_header *rh;
const gchar *in, *end;
const guchar **scvec;
guint *lenvec;
gboolean raw = FALSE;
struct mime_text_part *part;
struct rspamd_url *url;
struct rspamd_re_cache *cache = rt->cache;
gpointer k, v;
guint len, cnt;
msg_debug_re_cache ("get to the slow path for re type: %s: %s",
rspamd_re_cache_type_to_string (re_class->type),
rspamd_regexp_get_pattern (re));
re_id = rspamd_regexp_get_cache_id (re);
switch (re_class->type) {
case RSPAMD_RE_HEADER:
case RSPAMD_RE_RAWHEADER:
/* Get list of specified headers */
headerlist = rspamd_message_get_header_array (task,
re_class->type_data,
is_strong);
if (headerlist) {
scvec = g_malloc (sizeof (*scvec) * headerlist->len);
lenvec = g_malloc (sizeof (*lenvec) * headerlist->len);
for (i = 0; i < headerlist->len; i ++) {
rh = g_ptr_array_index (headerlist, i);
if (re_class->type == RSPAMD_RE_RAWHEADER) {
in = rh->value;
raw = TRUE;
lenvec[i] = strlen (rh->value);
}
else {
in = rh->decoded;
/* Validate input */
if (!in || !g_utf8_validate (in, -1, &end)) {
lenvec[i] = 0;
scvec[i] = (guchar *)"";
continue;
}
lenvec[i] = end - in;
}
scvec[i] = (guchar *)in;
}
ret = rspamd_re_cache_process_regexp_data (rt, re,
task->task_pool, scvec, lenvec, headerlist->len, raw);
debug_task ("checking header %s regexp: %s -> %d",
re_class->type_data,
rspamd_regexp_get_pattern (re), ret);
g_free (scvec);
g_free (lenvec);
}
break;
case RSPAMD_RE_ALLHEADER:
raw = TRUE;
in = task->raw_headers_content.begin;
len = task->raw_headers_content.len;
ret = rspamd_re_cache_process_regexp_data (rt, re,
task->task_pool, (const guchar **)&in, &len, 1, raw);
debug_task ("checking allheader regexp: %s -> %d",
rspamd_regexp_get_pattern (re), ret);
break;
case RSPAMD_RE_MIME:
case RSPAMD_RE_RAWMIME:
/* Iterate through text parts */
if (task->text_parts->len > 0) {
scvec = g_malloc (sizeof (*scvec) * task->text_parts->len);
lenvec = g_malloc (sizeof (*lenvec) * task->text_parts->len);
for (i = 0; i < task->text_parts->len; i++) {
part = g_ptr_array_index (task->text_parts, i);
/* Skip empty parts */
if (IS_PART_EMPTY (part)) {
lenvec[i] = 0;
scvec[i] = (guchar *) "";
continue;
}
/* Check raw flags */
if (!IS_PART_UTF (part)) {
raw = TRUE;
}
/* Select data for regexp */
if (re_class->type == RSPAMD_RE_RAWMIME) {
in = part->orig->data;
len = part->orig->len;
raw = TRUE;
}
else {
in = part->content->data;
len = part->content->len;
}
scvec[i] = (guchar *) in;
lenvec[i] = len;
}
ret = rspamd_re_cache_process_regexp_data (rt, re,
task->task_pool, scvec, lenvec, task->text_parts->len, raw);
debug_task ("checking mime regexp: %s -> %d",
rspamd_regexp_get_pattern (re), ret);
g_free (scvec);
g_free (lenvec);
}
break;
case RSPAMD_RE_URL:
cnt = g_hash_table_size (task->urls) + g_hash_table_size (task->emails);
if (cnt > 0) {
scvec = g_malloc (sizeof (*scvec) * cnt);
lenvec = g_malloc (sizeof (*lenvec) * cnt);
g_hash_table_iter_init (&it, task->urls);
i = 0;
while (g_hash_table_iter_next (&it, &k, &v)) {
url = v;
in = url->string;
len = url->urllen;
raw = FALSE;
scvec[i] = (guchar *)in;
lenvec[i++] = len;
}
g_hash_table_iter_init (&it, task->emails);
while (g_hash_table_iter_next (&it, &k, &v)) {
url = v;
in = url->string;
len = url->urllen;
raw = FALSE;
scvec[i] = (guchar *) in;
lenvec[i++] = len;
}
g_assert (i == cnt);
ret = rspamd_re_cache_process_regexp_data (rt, re,
task->task_pool, scvec, lenvec, i, raw);
debug_task ("checking url regexp: %s -> %d",
rspamd_regexp_get_pattern (re), ret);
g_free (scvec);
g_free (lenvec);
}
break;
case RSPAMD_RE_BODY:
raw = TRUE;
in = task->msg.begin;
len = task->msg.len;
ret = rspamd_re_cache_process_regexp_data (rt, re, task->task_pool,
(const guchar **)&in, &len, 1, raw);
debug_task ("checking rawbody regexp: %s -> %d",
rspamd_regexp_get_pattern (re), ret);
break;
case RSPAMD_RE_MAX:
msg_err_task ("regexp of class invalid has been called: %s",
rspamd_regexp_get_pattern (re));
break;
}
#if WITH_HYPERSCAN
if (!rt->cache->disable_hyperscan) {
rspamd_re_cache_finish_class (rt, re_class);
}
#endif
setbit (rt->checked, re_id);
return rt->results[re_id];
}
gint
lua_dkim_sign_handler (lua_State *L)
{
struct rspamd_task *task = lua_check_task (L, 1);
luaL_argcheck (L, lua_type (L, 2) == LUA_TTABLE, 2, "'table' expected");
GError *err = NULL;
GString *hdr;
const gchar *selector = NULL, *domain = NULL, *key = NULL;
rspamd_dkim_sign_context_t *ctx;
rspamd_dkim_sign_key_t *dkim_key;
/*
* Get the following elements:
* - selector
* - domain
* - key
*/
if (!rspamd_lua_parse_table_arguments (L, 2, &err,
"*key=S;*domain=S;*selector=S",
&key, &domain, &selector)) {
msg_err_task ("invalid return value from sign condition: %e",
err);
g_error_free (err);
lua_pushboolean (L, FALSE);
return 1;
}
if (dkim_module_ctx->dkim_sign_hash == NULL) {
dkim_module_ctx->dkim_sign_hash = rspamd_lru_hash_new (
128,
g_free, /* Keys are just C-strings */
(GDestroyNotify)rspamd_dkim_sign_key_unref);
}
dkim_key = rspamd_lru_hash_lookup (dkim_module_ctx->dkim_sign_hash,
key, time (NULL));
if (dkim_key == NULL) {
dkim_key = rspamd_dkim_sign_key_load (key, &err);
if (dkim_key == NULL) {
msg_err_task ("cannot load dkim key %s: %e",
key, err);
g_error_free (err);
lua_pushboolean (L, FALSE);
return 1;
}
rspamd_lru_hash_insert (dkim_module_ctx->dkim_sign_hash,
g_strdup (key), dkim_key,
time (NULL), 0);
}
ctx = rspamd_create_dkim_sign_context (task, dkim_key,
DKIM_CANON_RELAXED, DKIM_CANON_RELAXED,
dkim_module_ctx->sign_headers, &err);
if (ctx == NULL) {
msg_err_task ("cannot create sign context: %e",
err);
g_error_free (err);
lua_pushboolean (L, FALSE);
return 1;
}
hdr = rspamd_dkim_sign (task, selector, domain, 0, 0, ctx);
if (hdr) {
rspamd_mempool_set_variable (task->task_pool, "dkim-signature",
hdr, rspamd_gstring_free_hard);
lua_pushboolean (L, TRUE);
return 1;
}
lua_pushboolean (L, FALSE);
return 1;
}
gpointer
rspamd_stat_cache_redis_runtime (struct rspamd_task *task,
gpointer c, gboolean learn)
{
struct rspamd_redis_cache_ctx *ctx = c;
struct rspamd_redis_cache_runtime *rt;
struct upstream *up;
rspamd_inet_addr_t *addr;
g_assert (ctx != NULL);
if (learn && ctx->write_servers == NULL) {
msg_err_task ("no write servers defined for %s, cannot learn",
ctx->stcf->symbol);
return NULL;
}
if (task->tokens == NULL || task->tokens->len == 0) {
return NULL;
}
if (learn) {
up = rspamd_upstream_get (ctx->write_servers,
RSPAMD_UPSTREAM_MASTER_SLAVE,
NULL,
0);
}
else {
up = rspamd_upstream_get (ctx->read_servers,
RSPAMD_UPSTREAM_ROUND_ROBIN,
NULL,
0);
}
if (up == NULL) {
msg_err_task ("no upstreams reachable");
return NULL;
}
rt = rspamd_mempool_alloc0 (task->task_pool, sizeof (*rt));
rt->selected = up;
rt->task = task;
rt->ctx = ctx;
addr = rspamd_upstream_addr (up);
g_assert (addr != NULL);
rt->redis = redisAsyncConnect (rspamd_inet_address_to_string (addr),
rspamd_inet_address_get_port (addr));
g_assert (rt->redis != NULL);
redisLibeventAttach (rt->redis, task->ev_base);
/* Now check stats */
event_set (&rt->timeout_event, -1, EV_TIMEOUT, rspamd_redis_cache_timeout, rt);
event_base_set (task->ev_base, &rt->timeout_event);
rspamd_redis_cache_maybe_auth (ctx, rt->redis);
if (!learn) {
rspamd_stat_cache_redis_generate_id (task);
}
return rt;
}
/* Called when we have received tokens values from redis */
static void
rspamd_redis_processed (redisAsyncContext *c, gpointer r, gpointer priv)
{
struct redis_stat_runtime *rt = REDIS_RUNTIME (priv);
redisReply *reply = r, *elt;
struct rspamd_task *task;
rspamd_token_t *tok;
guint i, processed = 0, found = 0;
gulong val;
gdouble float_val;
task = rt->task;
if (c->err == 0) {
if (r != NULL) {
if (reply->type == REDIS_REPLY_ARRAY) {
if (reply->elements == task->tokens->len) {
for (i = 0; i < reply->elements; i ++) {
tok = g_ptr_array_index (task->tokens, i);
elt = reply->element[i];
if (G_UNLIKELY (elt->type == REDIS_REPLY_INTEGER)) {
tok->values[rt->id] = elt->integer;
found ++;
}
else if (elt->type == REDIS_REPLY_STRING) {
if (rt->stcf->clcf->flags &
RSPAMD_FLAG_CLASSIFIER_INTEGER) {
rspamd_strtoul (elt->str, elt->len, &val);
tok->values[rt->id] = val;
}
else {
float_val = strtod (elt->str, NULL);
tok->values[rt->id] = float_val;
}
found ++;
}
else {
tok->values[rt->id] = 0;
}
processed ++;
}
if (rt->stcf->is_spam) {
task->flags |= RSPAMD_TASK_FLAG_HAS_SPAM_TOKENS;
}
else {
task->flags |= RSPAMD_TASK_FLAG_HAS_HAM_TOKENS;
}
}
else {
msg_err_task_check ("got invalid length of reply vector from redis: "
"%d, expected: %d",
(gint)reply->elements,
(gint)task->tokens->len);
}
}
else {
msg_err_task_check ("got invalid reply from redis: %d",
reply->type);
}
msg_debug_task_check ("received tokens for %s: %d processed, %d found",
rt->redis_object_expanded, processed, found);
rspamd_upstream_ok (rt->selected);
}
}
else {
msg_err_task ("error getting reply from redis server %s: %s",
rspamd_upstream_name (rt->selected), c->errstr);
if (rt->redis) {
rspamd_upstream_fail (rt->selected);
}
}
if (rt->has_event) {
rspamd_session_remove_event (task->s, rspamd_redis_fin, rt);
}
}
gboolean
rspamd_task_process (struct rspamd_task *task, guint stages)
{
gint st;
gboolean ret = TRUE;
GError *stat_error = NULL;
/* Avoid nested calls */
if (task->flags & RSPAMD_TASK_FLAG_PROCESSING) {
return TRUE;
}
if (RSPAMD_TASK_IS_PROCESSED (task)) {
return TRUE;
}
task->flags |= RSPAMD_TASK_FLAG_PROCESSING;
st = rspamd_task_select_processing_stage (task, stages);
switch (st) {
case RSPAMD_TASK_STAGE_READ_MESSAGE:
if (!rspamd_message_parse (task)) {
ret = FALSE;
}
break;
case RSPAMD_TASK_STAGE_PRE_FILTERS:
rspamd_lua_call_pre_filters (task);
break;
case RSPAMD_TASK_STAGE_FILTERS:
if (!rspamd_process_filters (task)) {
ret = FALSE;
}
break;
case RSPAMD_TASK_STAGE_CLASSIFIERS:
if (rspamd_stat_classify (task, task->cfg->lua_state, &stat_error) ==
RSPAMD_STAT_PROCESS_ERROR) {
msg_err_task ("classify error: %e", stat_error);
g_error_free (stat_error);
}
break;
case RSPAMD_TASK_STAGE_COMPOSITES:
rspamd_make_composites (task);
break;
case RSPAMD_TASK_STAGE_POST_FILTERS:
rspamd_lua_call_post_filters (task);
break;
case RSPAMD_TASK_STAGE_DONE:
task->processed_stages |= RSPAMD_TASK_STAGE_DONE;
break;
default:
/* TODO: not implemented stage */
break;
}
if (RSPAMD_TASK_IS_SKIPPED (task)) {
task->processed_stages |= RSPAMD_TASK_STAGE_DONE;
}
task->flags &= ~RSPAMD_TASK_FLAG_PROCESSING;
if (!ret || RSPAMD_TASK_IS_PROCESSED (task)) {
if (!ret) {
/* Set processed flags */
task->processed_stages |= RSPAMD_TASK_STAGE_DONE;
}
msg_debug_task ("task is processed", st);
return ret;
}
if (rspamd_session_events_pending (task->s) != 0) {
/* We have events pending, so we consider this stage as incomplete */
msg_debug_task ("need more work on stage %d", st);
}
else {
/* Mark the current stage as done and go to the next stage */
msg_debug_task ("completed stage %d", st);
task->processed_stages |= st;
/* Reset checkpoint */
task->checkpoint = NULL;
/* Tail recursion */
return rspamd_task_process (task, stages);
}
return ret;
}
gboolean
rspamd_redis_learn_tokens (struct rspamd_task *task, GPtrArray *tokens,
gint id, gpointer p)
{
struct redis_stat_runtime *rt = REDIS_RUNTIME (p);
struct upstream *up;
rspamd_inet_addr_t *addr;
struct timeval tv;
rspamd_fstring_t *query;
const gchar *redis_cmd;
rspamd_token_t *tok;
gint ret;
up = rspamd_upstream_get (rt->ctx->write_servers,
RSPAMD_UPSTREAM_MASTER_SLAVE,
NULL,
0);
if (up == NULL) {
msg_err_task ("no upstreams reachable");
return FALSE;
}
rt->selected = up;
addr = rspamd_upstream_addr (up);
g_assert (addr != NULL);
rt->redis = redisAsyncConnect (rspamd_inet_address_to_string (addr),
rspamd_inet_address_get_port (addr));
g_assert (rt->redis != NULL);
redisLibeventAttach (rt->redis, task->ev_base);
rspamd_redis_maybe_auth (rt->ctx, rt->redis);
/*
* Add the current key to the set of learned keys
*/
redisAsyncCommand (rt->redis, NULL, NULL, "SADD %s_keys %s",
rt->stcf->symbol, rt->redis_object_expanded);
if (rt->stcf->clcf->flags & RSPAMD_FLAG_CLASSIFIER_INTEGER) {
redis_cmd = "HINCRBY";
}
else {
redis_cmd = "HINCRBYFLOAT";
}
rt->id = id;
query = rspamd_redis_tokens_to_query (task, tokens,
redis_cmd, rt->redis_object_expanded, TRUE, id,
rt->stcf->clcf->flags & RSPAMD_FLAG_CLASSIFIER_INTEGER);
g_assert (query != NULL);
/*
* XXX:
* Dirty hack: we get a token and check if it's value is -1 or 1, so
* we could understand that we are learning or unlearning
*/
tok = g_ptr_array_index (task->tokens, 0);
if (tok->values[id] > 0) {
rspamd_printf_fstring (&query, ""
"*4\r\n"
"$7\r\n"
"HINCRBY\r\n"
"$%d\r\n"
"%s\r\n"
"$6\r\n"
"learns\r\n"
"$1\r\n"
"1\r\n",
(gint)strlen (rt->redis_object_expanded),
rt->redis_object_expanded);
}
else {
rspamd_printf_fstring (&query, ""
"*4\r\n"
"$7\r\n"
"HINCRBY\r\n"
"$%d\r\n"
"%s\r\n"
"$6\r\n"
"learns\r\n"
"$2\r\n"
"-1\r\n",
(gint)strlen (rt->redis_object_expanded),
rt->redis_object_expanded);
}
rspamd_mempool_add_destructor (task->task_pool,
(rspamd_mempool_destruct_t)rspamd_fstring_free, query);
ret = redisAsyncFormattedCommand (rt->redis, rspamd_redis_learned, rt,
query->str, query->len);
if (ret == REDIS_OK) {
rspamd_session_add_event (task->s, rspamd_redis_fin_learn, rt,
rspamd_redis_stat_quark ());
rt->has_event = TRUE;
/* Set timeout */
if (event_get_base (&rt->timeout_event)) {
event_del (&rt->timeout_event);
}
event_set (&rt->timeout_event, -1, EV_TIMEOUT, rspamd_redis_timeout, rt);
event_base_set (task->ev_base, &rt->timeout_event);
double_to_tv (rt->ctx->timeout, &tv);
event_add (&rt->timeout_event, &tv);
return TRUE;
}
else {
msg_err_task ("call to redis failed: %s", rt->redis->errstr);
}
return FALSE;
}
/*
* Non-static for lua unit testing
*/
gsize
rspamd_redis_expand_object (const gchar *pattern,
struct redis_stat_ctx *ctx,
struct rspamd_task *task,
gchar **target)
{
gsize tlen = 0;
const gchar *p = pattern, *elt;
gchar *d, *end;
enum {
just_char,
percent_char,
mod_char
} state = just_char;
struct rspamd_statfile_config *stcf;
lua_State *L = NULL;
struct rspamd_task **ptask;
GString *tb;
const gchar *rcpt = NULL;
gint err_idx;
g_assert (ctx != NULL);
stcf = ctx->stcf;
if (task) {
L = task->cfg->lua_state;
}
if (ctx->enable_users) {
if (ctx->cbref_user == -1) {
rcpt = rspamd_task_get_principal_recipient (task);
}
else if (L) {
/* Execute lua function to get userdata */
lua_pushcfunction (L, &rspamd_lua_traceback);
err_idx = lua_gettop (L);
lua_rawgeti (L, LUA_REGISTRYINDEX, ctx->cbref_user);
ptask = lua_newuserdata (L, sizeof (struct rspamd_task *));
*ptask = task;
rspamd_lua_setclass (L, "rspamd{task}", -1);
if (lua_pcall (L, 1, 1, err_idx) != 0) {
tb = lua_touserdata (L, -1);
msg_err_task ("call to user extraction script failed: %v", tb);
g_string_free (tb, TRUE);
}
else {
rcpt = rspamd_mempool_strdup (task->task_pool, lua_tostring (L, -1));
}
/* Result + error function */
lua_pop (L, 2);
}
if (rcpt) {
rspamd_mempool_set_variable (task->task_pool, "stat_user",
(gpointer)rcpt, NULL);
}
}
/* Length calculation */
while (*p) {
switch (state) {
case just_char:
if (*p == '%') {
state = percent_char;
}
else {
tlen ++;
}
p ++;
break;
case percent_char:
switch (*p) {
case '%':
tlen ++;
state = just_char;
break;
case 'u':
elt = GET_TASK_ELT (task, user);
if (elt) {
tlen += strlen (elt);
}
break;
case 'r':
if (rcpt == NULL) {
elt = rspamd_task_get_principal_recipient (task);
}
else {
elt = rcpt;
}
if (elt) {
tlen += strlen (elt);
}
break;
case 'l':
if (stcf->label) {
tlen += strlen (stcf->label);
}
break;
case 's':
if (stcf->symbol) {
tlen += strlen (stcf->symbol);
}
break;
default:
state = just_char;
tlen ++;
break;
}
if (state == percent_char) {
state = mod_char;
}
p ++;
break;
case mod_char:
switch (*p) {
case 'd':
p ++;
state = just_char;
break;
default:
state = just_char;
break;
}
break;
}
}
if (target == NULL || task == NULL) {
return tlen;
}
*target = rspamd_mempool_alloc (task->task_pool, tlen + 1);
d = *target;
end = d + tlen + 1;
d[tlen] = '\0';
p = pattern;
state = just_char;
/* Expand string */
while (*p && d < end) {
switch (state) {
case just_char:
if (*p == '%') {
state = percent_char;
}
else {
*d++ = *p;
}
p ++;
break;
case percent_char:
switch (*p) {
case '%':
*d++ = *p;
state = just_char;
break;
case 'u':
elt = GET_TASK_ELT (task, user);
if (elt) {
d += rspamd_strlcpy (d, elt, end - d);
}
break;
case 'r':
if (rcpt == NULL) {
elt = rspamd_task_get_principal_recipient (task);
}
else {
elt = rcpt;
}
if (elt) {
d += rspamd_strlcpy (d, elt, end - d);
}
break;
case 'l':
if (stcf->label) {
d += rspamd_strlcpy (d, stcf->label, end - d);
}
break;
case 's':
if (stcf->symbol) {
d += rspamd_strlcpy (d, stcf->symbol, end - d);
}
break;
default:
state = just_char;
*d++ = *p;
break;
}
if (state == percent_char) {
state = mod_char;
}
p ++;
break;
case mod_char:
switch (*p) {
case 'd':
/* TODO: not supported yet */
p ++;
state = just_char;
break;
default:
state = just_char;
break;
}
break;
}
}
return tlen;
}
static void
dkim_sign_callback (struct rspamd_task *task, void *unused)
{
lua_State *L;
struct rspamd_task **ptask;
gboolean sign = FALSE;
gint err_idx;
GString *tb, *hdr;
GError *err = NULL;
const gchar *selector = NULL, *domain = NULL, *key = NULL;
rspamd_dkim_sign_context_t *ctx;
rspamd_dkim_sign_key_t *dkim_key;
if (dkim_module_ctx->sign_condition_ref != -1) {
sign = FALSE;
L = task->cfg->lua_state;
lua_pushcfunction (L, &rspamd_lua_traceback);
err_idx = lua_gettop (L);
lua_rawgeti (L, LUA_REGISTRYINDEX, dkim_module_ctx->sign_condition_ref);
ptask = lua_newuserdata (L, sizeof (struct rspamd_task *));
*ptask = task;
rspamd_lua_setclass (L, "rspamd{task}", -1);
if (lua_pcall (L, 1, 1, err_idx) != 0) {
tb = lua_touserdata (L, -1);
msg_err_task ("call to user extraction script failed: %v", tb);
g_string_free (tb, TRUE);
}
else {
if (lua_istable (L, -1)) {
/*
* Get the following elements:
* - selector
* - domain
* - key
*/
if (!rspamd_lua_parse_table_arguments (L, -1, &err,
"*key=S;*domain=S;*selector=S",
&key, &domain, &selector)) {
msg_err_task ("invalid return value from sign condition: %e",
err);
g_error_free (err);
return;
}
dkim_key = rspamd_lru_hash_lookup (dkim_module_ctx->dkim_sign_hash,
key, time (NULL));
if (dkim_key == NULL) {
dkim_key = rspamd_dkim_sign_key_load (key, &err);
if (dkim_key == NULL) {
msg_err_task ("cannot load dkim key %s: %e",
key, err);
g_error_free (err);
return;
}
rspamd_lru_hash_insert (dkim_module_ctx->dkim_sign_hash,
g_strdup (key), dkim_key,
time (NULL), 0);
}
ctx = rspamd_create_dkim_sign_context (task, dkim_key,
DKIM_CANON_RELAXED, DKIM_CANON_RELAXED,
dkim_module_ctx->sign_headers, &err);
if (ctx == NULL) {
msg_err_task ("cannot create sign context: %e",
key, err);
g_error_free (err);
return;
}
hdr = rspamd_dkim_sign (task, selector, domain, 0, 0, ctx);
if (hdr) {
rspamd_mempool_set_variable (task->task_pool, "dkim-signature",
hdr, rspamd_gstring_free_hard);
}
sign = TRUE;
}
else {
sign = FALSE;
}
}
/* Result + error function */
lua_settop (L, 0);
if (!sign) {
msg_debug_task ("skip signing as dkim condition callback returned"
" false");
return;
}
}
}
/*
* Calculates the specified regexp for the specified class if it's not calculated
*/
static guint
rspamd_re_cache_exec_re (struct rspamd_task *task,
struct rspamd_re_runtime *rt,
rspamd_regexp_t *re,
struct rspamd_re_class *re_class,
gboolean is_strong)
{
guint ret = 0, i, re_id;
GList *cur, *headerlist;
GHashTableIter it;
struct raw_header *rh;
const gchar *in;
gboolean raw = FALSE;
struct mime_text_part *part;
struct rspamd_url *url;
struct rspamd_re_cache *cache = rt->cache;
gpointer k, v;
gsize len;
msg_debug_re_cache ("get to the slow path for re type: %s: %s",
rspamd_re_cache_type_to_string (re_class->type),
rspamd_regexp_get_pattern (re));
re_id = rspamd_regexp_get_cache_id (re);
switch (re_class->type) {
case RSPAMD_RE_HEADER:
case RSPAMD_RE_RAWHEADER:
/* Get list of specified headers */
headerlist = rspamd_message_get_header (task,
re_class->type_data,
is_strong);
if (headerlist) {
cur = headerlist;
while (cur) {
rh = cur->data;
if (re_class->type == RSPAMD_RE_RAWHEADER) {
in = rh->value;
raw = TRUE;
}
else {
in = rh->decoded;
/* Validate input */
if (!in || !g_utf8_validate (in, -1, NULL)) {
cur = g_list_next (cur);
continue;
}
}
/* Match re */
if (in) {
ret = rspamd_re_cache_process_regexp_data (rt, re,
task->task_pool, in, strlen (in), raw);
debug_task ("checking header %s regexp: %s -> %d",
re_class->type_data,
rspamd_regexp_get_pattern (re), ret);
}
cur = g_list_next (cur);
}
}
break;
case RSPAMD_RE_ALLHEADER:
raw = TRUE;
in = task->raw_headers_content.begin;
len = task->raw_headers_content.len;
ret = rspamd_re_cache_process_regexp_data (rt, re,
task->task_pool, in, len, raw);
debug_task ("checking allheader regexp: %s -> %d",
rspamd_regexp_get_pattern (re), ret);
break;
case RSPAMD_RE_MIME:
case RSPAMD_RE_RAWMIME:
/* Iterate throught text parts */
for (i = 0; i < task->text_parts->len; i++) {
part = g_ptr_array_index (task->text_parts, i);
/* Skip empty parts */
if (IS_PART_EMPTY (part)) {
continue;
}
/* Check raw flags */
if (!IS_PART_UTF (part)) {
raw = TRUE;
}
/* Select data for regexp */
if (re_class->type == RSPAMD_RE_RAWMIME) {
in = part->orig->data;
len = part->orig->len;
raw = TRUE;
}
else {
in = part->content->data;
len = part->content->len;
}
if (len > 0) {
ret = rspamd_re_cache_process_regexp_data (rt, re,
task->task_pool, in, len, raw);
debug_task ("checking mime regexp: %s -> %d",
rspamd_regexp_get_pattern (re), ret);
}
}
break;
case RSPAMD_RE_URL:
g_hash_table_iter_init (&it, task->urls);
while (g_hash_table_iter_next (&it, &k, &v)) {
url = v;
in = url->string;
len = url->urllen;
raw = FALSE;
ret = rspamd_re_cache_process_regexp_data (rt, re,
task->task_pool, in, len, raw);
}
g_hash_table_iter_init (&it, task->emails);
while (g_hash_table_iter_next (&it, &k, &v)) {
url = v;
in = url->string;
len = url->urllen;
raw = FALSE;
ret = rspamd_re_cache_process_regexp_data (rt, re,
task->task_pool, in, len, raw);
}
debug_task ("checking url regexp: %s -> %d",
rspamd_regexp_get_pattern (re), ret);
break;
case RSPAMD_RE_BODY:
raw = TRUE;
in = task->msg.begin;
len = task->msg.len;
ret = rspamd_re_cache_process_regexp_data (rt, re, task->task_pool, in,
len, raw);
debug_task ("checking rawbody regexp: %s -> %d",
rspamd_regexp_get_pattern (re), ret);
break;
case RSPAMD_RE_MAX:
msg_err_task ("regexp of class invalid has been called: %s",
rspamd_regexp_get_pattern (re));
break;
}
#if WITH_HYPERSCAN
if (!rt->cache->disable_hyperscan) {
rspamd_re_cache_finish_class (rt, re_class);
}
#endif
setbit (rt->checked, re_id);
return rt->results[re_id];
}
