void
rspamd_symbols_cache_disable_symbol (struct rspamd_task *task,
struct symbols_cache *cache, const gchar *symbol)
{
struct cache_savepoint *checkpoint;
struct cache_item *item;
gint id;
if (task->checkpoint == NULL) {
checkpoint = rspamd_symbols_cache_make_checkpoint (task, cache);
task->checkpoint = checkpoint;
}
else {
checkpoint = task->checkpoint;
}
id = rspamd_symbols_cache_find_symbol_parent (cache, symbol);
if (id > 0) {
/* Set executed and finished flags */
item = g_ptr_array_index (cache->items_by_id, id);
setbit (checkpoint->processed_bits, item->id * 2);
setbit (checkpoint->processed_bits, item->id * 2 + 1);
msg_debug_task ("disable execution of %s", symbol);
}
else {
msg_info_task ("cannot disable %s: not found", symbol);
}
}
static void
rspamd_symbols_cache_watcher_cb (gpointer sessiond, gpointer ud)
{
struct rspamd_task *task = sessiond;
struct cache_item *item = ud, *it;
struct cache_savepoint *checkpoint;
struct symbols_cache *cache;
gint i, remain = 0;
checkpoint = task->checkpoint;
cache = task->cfg->cache;
/* Specify that we are done with this item */
setbit (checkpoint->processed_bits, item->id * 2 + 1);
if (checkpoint->pass > 0) {
for (i = 0; i < (gint)checkpoint->waitq->len; i ++) {
it = g_ptr_array_index (checkpoint->waitq, i);
if (!isset (checkpoint->processed_bits, it->id * 2)) {
if (!rspamd_symbols_cache_check_deps (task, cache, it,
checkpoint)) {
remain ++;
break;
}
rspamd_symbols_cache_check_symbol (task, cache, it, checkpoint,
NULL);
}
}
}
msg_debug_task ("finished watcher, %ud symbols waiting", remain);
}
/* Called when we have connected to the redis server and got stats */
static void
rspamd_redis_connected (redisAsyncContext *c, gpointer r, gpointer priv)
{
struct redis_stat_runtime *rt = REDIS_RUNTIME (priv);
redisReply *reply = r;
struct rspamd_task *task;
glong val = 0;
task = rt->task;
if (c->err == 0) {
if (r != NULL) {
if (G_UNLIKELY (reply->type == REDIS_REPLY_INTEGER)) {
val = reply->integer;
}
else if (reply->type == REDIS_REPLY_STRING) {
rspamd_strtol (reply->str, reply->len, &val);
}
else {
if (reply->type != REDIS_REPLY_NIL) {
msg_err_task ("bad learned type for %s: %d",
rt->stcf->symbol, reply->type);
}
val = 0;
}
if (val < 0) {
msg_warn_task ("invalid number of learns for %s: %L",
rt->stcf->symbol, val);
val = 0;
}
rt->learned = val;
msg_debug_task ("connected to redis server, tokens learned for %s: %uL",
rt->redis_object_expanded, rt->learned);
rspamd_upstream_ok (rt->selected);
}
}
else {
msg_err_task ("error getting reply from redis server %s: %s",
rspamd_upstream_name (rt->selected), c->errstr);
rspamd_upstream_fail (rt->selected);
}
}
static double
get_specific_action_score (struct rspamd_task *task,
const ucl_object_t *metric,
struct metric_action *action)
{
const ucl_object_t *act, *sact;
const gchar *act_name;
double score;
if (metric) {
act = ucl_object_find_key (metric, "actions");
if (act) {
act_name = rspamd_action_to_str (action->action);
sact = ucl_object_find_key (act, act_name);
if (sact != NULL && ucl_object_todouble_safe (sact, &score)) {
msg_debug_task ("found override score %.2f for action %s in settings",
score, act_name);
return score;
}
}
}
return action->score;
}
static void
dkim_sign_callback (struct rspamd_task *task, void *unused)
{
lua_State *L;
struct rspamd_task **ptask;
gboolean sign = FALSE;
gint err_idx;
GString *tb, *hdr;
GError *err = NULL;
const gchar *selector = NULL, *domain = NULL, *key = NULL;
rspamd_dkim_sign_context_t *ctx;
rspamd_dkim_sign_key_t *dkim_key;
if (dkim_module_ctx->sign_condition_ref != -1) {
sign = FALSE;
L = task->cfg->lua_state;
lua_pushcfunction (L, &rspamd_lua_traceback);
err_idx = lua_gettop (L);
lua_rawgeti (L, LUA_REGISTRYINDEX, dkim_module_ctx->sign_condition_ref);
ptask = lua_newuserdata (L, sizeof (struct rspamd_task *));
*ptask = task;
rspamd_lua_setclass (L, "rspamd{task}", -1);
if (lua_pcall (L, 1, 1, err_idx) != 0) {
tb = lua_touserdata (L, -1);
msg_err_task ("call to user extraction script failed: %v", tb);
g_string_free (tb, TRUE);
}
else {
if (lua_istable (L, -1)) {
/*
* Get the following elements:
* - selector
* - domain
* - key
*/
if (!rspamd_lua_parse_table_arguments (L, -1, &err,
"*key=S;*domain=S;*selector=S",
&key, &domain, &selector)) {
msg_err_task ("invalid return value from sign condition: %e",
err);
g_error_free (err);
return;
}
dkim_key = rspamd_lru_hash_lookup (dkim_module_ctx->dkim_sign_hash,
key, time (NULL));
if (dkim_key == NULL) {
dkim_key = rspamd_dkim_sign_key_load (key, &err);
if (dkim_key == NULL) {
msg_err_task ("cannot load dkim key %s: %e",
key, err);
g_error_free (err);
return;
}
rspamd_lru_hash_insert (dkim_module_ctx->dkim_sign_hash,
g_strdup (key), dkim_key,
time (NULL), 0);
}
ctx = rspamd_create_dkim_sign_context (task, dkim_key,
DKIM_CANON_RELAXED, DKIM_CANON_RELAXED,
dkim_module_ctx->sign_headers, &err);
if (ctx == NULL) {
msg_err_task ("cannot create sign context: %e",
key, err);
g_error_free (err);
return;
}
hdr = rspamd_dkim_sign (task, selector, domain, 0, 0, ctx);
if (hdr) {
rspamd_mempool_set_variable (task->task_pool, "dkim-signature",
hdr, rspamd_gstring_free_hard);
}
sign = TRUE;
}
else {
sign = FALSE;
}
}
/* Result + error function */
lua_settop (L, 0);
if (!sign) {
msg_debug_task ("skip signing as dkim condition callback returned"
" false");
return;
}
}
}
static void
dkim_symbol_callback (struct rspamd_task *task, void *unused)
{
GList *hlist;
rspamd_dkim_context_t *ctx;
rspamd_dkim_key_t *key;
GError *err = NULL;
struct raw_header *rh;
struct dkim_check_result *res = NULL, *cur;
guint checked = 0;
/* First check if plugin should be enabled */
if (task->user != NULL || rspamd_inet_address_is_local (task->from_addr)) {
msg_info_task ("skip DKIM checks for local networks and authorized users");
return;
}
/* Check whitelist */
if (radix_find_compressed_addr (dkim_module_ctx->whitelist_ip,
task->from_addr) != RADIX_NO_VALUE) {
msg_info_task ("skip DKIM checks for whitelisted address");
return;
}
/* Now check if a message has its signature */
hlist = rspamd_message_get_header (task,
DKIM_SIGNHEADER,
FALSE);
if (hlist != NULL) {
msg_debug_task ("dkim signature found");
while (hlist != NULL) {
rh = (struct raw_header *)hlist->data;
if (rh->decoded == NULL || rh->decoded[0] == '\0') {
msg_info_task ("<%s> cannot load empty DKIM context",
task->message_id);
hlist = g_list_next (hlist);
continue;
}
if (res == NULL) {
res = rspamd_mempool_alloc0 (task->task_pool, sizeof (*res));
res->prev = res;
res->w = rspamd_session_get_watcher (task->s);
cur = res;
}
else {
cur = rspamd_mempool_alloc0 (task->task_pool, sizeof (*res));
}
cur->first = res;
cur->res = -1;
cur->task = task;
cur->mult_allow = 1.0;
cur->mult_deny = 1.0;
ctx = rspamd_create_dkim_context (rh->decoded,
task->task_pool,
dkim_module_ctx->time_jitter,
&err);
if (ctx == NULL) {
if (err != NULL) {
msg_info_task ("<%s> cannot parse DKIM context: %e",
task->message_id, err);
g_error_free (err);
err = NULL;
}
else {
msg_info_task ("<%s> cannot parse DKIM context: "
"unknown error",
task->message_id);
}
hlist = g_list_next (hlist);
continue;
}
else {
/* Get key */
cur->ctx = ctx;
if (dkim_module_ctx->trusted_only &&
(dkim_module_ctx->dkim_domains == NULL ||
g_hash_table_lookup (dkim_module_ctx->dkim_domains,
rspamd_dkim_get_domain (ctx)) == NULL)) {
msg_debug_task ("skip dkim check for %s domain",
rspamd_dkim_get_domain (ctx));
hlist = g_list_next (hlist);
continue;
}
key = rspamd_lru_hash_lookup (dkim_module_ctx->dkim_hash,
rspamd_dkim_get_dns_key (ctx),
task->tv.tv_sec);
if (key != NULL) {
cur->key = rspamd_dkim_key_ref (key);
/* Release key when task is processed */
rspamd_mempool_add_destructor (task->task_pool,
dkim_module_key_dtor, cur->key);
}
else {
rspamd_get_dkim_key (ctx,
task,
dkim_module_key_handler,
cur);
}
}
if (res != cur) {
DL_APPEND (res, cur);
}
if (dkim_module_ctx->skip_multi) {
if (hlist->next) {
msg_info_task ("message has multiple signatures but we"
" check only one as 'skip_multi' is set");
}
break;
}
checked ++;
if (checked > dkim_module_ctx->max_sigs) {
msg_info_task ("message has multiple signatures but we"
" stopped after %d checked signatures as limit"
" is reached", checked);
break;
}
hlist = g_list_next (hlist);
}
}
else {
rspamd_task_insert_result (task,
dkim_module_ctx->symbol_na,
1.0,
NULL);
}
if (res != NULL) {
rspamd_session_watcher_push (task->s);
dkim_module_check (res);
}
}
static void
rspamd_archive_process_zip (struct rspamd_task *task,
struct rspamd_mime_part *part)
{
const guchar *p, *start, *end, *eocd = NULL, *cd;
const guint32 eocd_magic = 0x06054b50, cd_basic_len = 46;
const guchar cd_magic[] = {0x50, 0x4b, 0x01, 0x02};
guint32 cd_offset, cd_size, comp_size, uncomp_size;
guint16 extra_len, fname_len, comment_len;
struct rspamd_archive *arch;
struct rspamd_archive_file *f;
/* Zip files have interesting data at the end of archive */
p = part->parsed_data.begin + part->parsed_data.len - 1;
start = part->parsed_data.begin;
end = p;
/* Search for EOCD:
* 22 bytes is a typical size of eocd without a comment and
* end points one byte after the last character
*/
p -= 21;
while (p > start + sizeof (guint32)) {
guint32 t;
/* XXX: not an efficient approach */
memcpy (&t, p, sizeof (t));
if (GUINT32_FROM_LE (t) == eocd_magic) {
eocd = p;
break;
}
p --;
}
if (eocd == NULL) {
/* Not a zip file */
msg_debug_task ("zip archive is invalid (no EOCD)");
return;
}
if (end - eocd < 21) {
msg_debug_task ("zip archive is invalid (short EOCD)");
return;
}
memcpy (&cd_size, eocd + 12, sizeof (cd_size));
cd_size = GUINT32_FROM_LE (cd_size);
memcpy (&cd_offset, eocd + 16, sizeof (cd_offset));
cd_offset = GUINT32_FROM_LE (cd_offset);
/* We need to check sanity as well */
if (cd_offset + cd_size != (guint)(eocd - start)) {
msg_debug_task ("zip archive is invalid (bad size/offset for CD)");
return;
}
cd = start + cd_offset;
arch = rspamd_mempool_alloc0 (task->task_pool, sizeof (*arch));
arch->files = g_ptr_array_new ();
arch->type = RSPAMD_ARCHIVE_ZIP;
rspamd_mempool_add_destructor (task->task_pool, rspamd_archive_dtor,
arch);
while (cd < eocd) {
/* Read central directory record */
if (eocd - cd < cd_basic_len ||
memcmp (cd, cd_magic, sizeof (cd_magic)) != 0) {
msg_debug_task ("zip archive is invalid (bad cd record)");
return;
}
memcpy (&comp_size, cd + 20, sizeof (guint32));
comp_size = GUINT32_FROM_LE (comp_size);
memcpy (&uncomp_size, cd + 24, sizeof (guint32));
uncomp_size = GUINT32_FROM_LE (uncomp_size);
memcpy (&fname_len, cd + 28, sizeof (fname_len));
fname_len = GUINT16_FROM_LE (fname_len);
memcpy (&extra_len, cd + 30, sizeof (extra_len));
extra_len = GUINT16_FROM_LE (extra_len);
memcpy (&comment_len, cd + 32, sizeof (comment_len));
comment_len = GUINT16_FROM_LE (comment_len);
if (cd + fname_len + comment_len + extra_len + cd_basic_len > eocd) {
msg_debug_task ("zip archive is invalid (too large cd record)");
return;
}
f = g_slice_alloc0 (sizeof (*f));
f->fname = g_string_new_len (cd + cd_basic_len, fname_len);
f->compressed_size = comp_size;
f->uncompressed_size = uncomp_size;
g_ptr_array_add (arch->files, f);
msg_debug_task ("found file in zip archive: %v", f->fname);
cd += fname_len + comment_len + extra_len + cd_basic_len;
}
part->flags |= RSPAMD_MIME_PART_ARCHIVE;
part->specific.arch = arch;
if (part->cd) {
arch->archive_name = &part->cd->filename;
}
arch->size = part->parsed_data.len;
}
static void
rspamd_archive_process_rar (struct rspamd_task *task,
struct rspamd_mime_part *part)
{
const guchar *p, *end, *section_start;
const guchar rar_v5_magic[] = {0x52, 0x61, 0x72, 0x21, 0x1A, 0x07, 0x01, 0x00},
rar_v4_magic[] = {0x52, 0x61, 0x72, 0x21, 0x1A, 0x07, 0x00};
const guint rar_encrypted_header = 4, rar_main_header = 1,
rar_file_header = 2;
guint64 vint, sz, comp_sz = 0, uncomp_sz = 0, flags = 0, type = 0;
struct rspamd_archive *arch;
struct rspamd_archive_file *f;
gint r;
p = part->parsed_data.begin;
end = p + part->parsed_data.len;
if ((gsize)(end - p) <= sizeof (rar_v5_magic)) {
msg_debug_task ("rar archive is invalid (too small)");
return;
}
if (memcmp (p, rar_v5_magic, sizeof (rar_v5_magic)) == 0) {
p += sizeof (rar_v5_magic);
}
else if (memcmp (p, rar_v4_magic, sizeof (rar_v4_magic)) == 0) {
p += sizeof (rar_v4_magic);
rspamd_archive_process_rar_v4 (task, p, end, part);
return;
}
else {
msg_debug_task ("rar archive is invalid (no rar magic)");
return;
}
/* Rar v5 format */
arch = rspamd_mempool_alloc0 (task->task_pool, sizeof (*arch));
arch->files = g_ptr_array_new ();
arch->type = RSPAMD_ARCHIVE_RAR;
rspamd_mempool_add_destructor (task->task_pool, rspamd_archive_dtor,
arch);
/* Now we can have either encryption header or archive header */
/* Crc 32 */
RAR_SKIP_BYTES (sizeof (guint32));
/* Size */
RAR_READ_VINT_SKIP ();
sz = vint;
/* Type */
section_start = p;
RAR_READ_VINT_SKIP ();
type = vint;
/* Header flags */
RAR_READ_VINT_SKIP ();
flags = vint;
if (flags & 0x1) {
/* Have extra zone */
RAR_READ_VINT_SKIP ();
}
if (flags & 0x2) {
/* Data zone is presented */
RAR_READ_VINT_SKIP ();
sz += vint;
}
if (type == rar_encrypted_header) {
/* We can't read any further information as archive is encrypted */
arch->flags |= RSPAMD_ARCHIVE_ENCRYPTED;
goto end;
}
else if (type != rar_main_header) {
msg_debug_task ("rar archive is invalid (bad main header)");
return;
}
/* Nothing useful in main header */
p = section_start;
RAR_SKIP_BYTES (sz);
while (p < end) {
/* Read the next header */
/* Crc 32 */
RAR_SKIP_BYTES (sizeof (guint32));
/* Size */
RAR_READ_VINT_SKIP ();
sz = vint;
if (sz == 0) {
/* Zero sized block - error */
msg_debug_task ("rar archive is invalid (zero size block)");
return;
}
section_start = p;
/* Type */
RAR_READ_VINT_SKIP ();
type = vint;
/* Header flags */
RAR_READ_VINT_SKIP ();
flags = vint;
if (flags & 0x1) {
/* Have extra zone */
RAR_READ_VINT_SKIP ();
}
if (flags & 0x2) {
/* Data zone is presented */
RAR_READ_VINT_SKIP ();
sz += vint;
comp_sz = vint;
}
if (type != rar_file_header) {
p = section_start;
RAR_SKIP_BYTES (sz);
}
else {
/* We have a file header, go forward */
guint64 fname_len;
/* File header specific flags */
RAR_READ_VINT_SKIP ();
flags = vint;
/* Unpacked size */
RAR_READ_VINT_SKIP ();
uncomp_sz = vint;
/* Attributes */
RAR_READ_VINT_SKIP ();
if (flags & 0x2) {
/* Unix mtime */
RAR_SKIP_BYTES (sizeof (guint32));
}
if (flags & 0x4) {
/* Crc32 */
RAR_SKIP_BYTES (sizeof (guint32));
}
/* Compression */
RAR_READ_VINT_SKIP ();
/* Host OS */
RAR_READ_VINT_SKIP ();
/* Filename length (finally!) */
RAR_READ_VINT_SKIP ();
fname_len = vint;
if (fname_len == 0 || fname_len > (gsize)(end - p)) {
msg_debug_task ("rar archive is invalid (bad fileame size)");
return;
}
f = g_slice_alloc0 (sizeof (*f));
f->uncompressed_size = uncomp_sz;
f->compressed_size = comp_sz;
f->fname = g_string_new_len (p, fname_len);
g_ptr_array_add (arch->files, f);
/* Restore p to the beginning of the header */
p = section_start;
RAR_SKIP_BYTES (sz);
}
}
end:
part->flags |= RSPAMD_MIME_PART_ARCHIVE;
part->specific.arch = arch;
if (part->cd != NULL) {
arch->archive_name = &part->cd->filename;
}
arch->size = part->parsed_data.len;
}
static void
rspamd_archive_process_rar_v4 (struct rspamd_task *task, const guchar *start,
const guchar *end, struct rspamd_mime_part *part)
{
const guchar *p = start, *start_section;
guint8 type;
guint flags;
guint64 sz, comp_sz = 0, uncomp_sz = 0;
struct rspamd_archive *arch;
struct rspamd_archive_file *f;
arch = rspamd_mempool_alloc0 (task->task_pool, sizeof (*arch));
arch->files = g_ptr_array_new ();
arch->type = RSPAMD_ARCHIVE_RAR;
rspamd_mempool_add_destructor (task->task_pool, rspamd_archive_dtor,
arch);
while (p < end) {
/* Crc16 */
start_section = p;
RAR_SKIP_BYTES (sizeof (guint16));
type = *p;
p ++;
RAR_READ_UINT16 (flags);
if (type == 0x73) {
/* Main header, check for encryption */
if (flags & 0x80) {
arch->flags |= RSPAMD_ARCHIVE_ENCRYPTED;
goto end;
}
}
RAR_READ_UINT16 (sz);
if (flags & 0x8000) {
/* We also need to read ADD_SIZE element */
guint32 tmp;
RAR_READ_UINT32 (tmp);
sz += tmp;
/* This is also used as PACK_SIZE */
comp_sz = tmp;
}
if (sz == 0) {
/* Zero sized block - error */
msg_debug_task ("rar archive is invalid (zero size block)");
return;
}
if (type == 0x74) {
guint fname_len;
/* File header */
/* Uncompressed size */
RAR_READ_UINT32 (uncomp_sz);
/* Skip to NAME_SIZE element */
RAR_SKIP_BYTES (11);
RAR_READ_UINT16 (fname_len);
if (fname_len == 0 || fname_len > (gsize)(end - p)) {
msg_debug_task ("rar archive is invalid (bad fileame size)");
return;
}
/* Attrs */
RAR_SKIP_BYTES (4);
if (flags & 0x100) {
/* We also need to read HIGH_PACK_SIZE */
guint32 tmp;
RAR_READ_UINT32 (tmp);
sz += tmp;
comp_sz += tmp;
/* HIGH_UNP_SIZE */
RAR_READ_UINT32 (tmp);
uncomp_sz += tmp;
}
f = g_slice_alloc0 (sizeof (*f));
if (flags & 0x200) {
/* We have unicode + normal version */
guchar *tmp;
tmp = memchr (p, '\0', fname_len);
if (tmp != NULL) {
/* Just use ASCII version */
f->fname = g_string_new_len (p, tmp - p);
}
else {
/* We have UTF8 filename, use it as is */
f->fname = g_string_new_len (p, fname_len);
}
}
else {
f->fname = g_string_new_len (p, fname_len);
}
f->compressed_size = comp_sz;
f->uncompressed_size = uncomp_sz;
if (flags & 0x4) {
f->flags |= RSPAMD_ARCHIVE_FILE_ENCRYPTED;
}
g_ptr_array_add (arch->files, f);
}
p = start_section;
RAR_SKIP_BYTES (sz);
}
end:
part->flags |= RSPAMD_MIME_PART_ARCHIVE;
part->specific.arch = arch;
arch->archive_name = &part->cd->filename;
arch->size = part->parsed_data.len;
}
gboolean
rspamd_task_process (struct rspamd_task *task, guint stages)
{
gint st;
gboolean ret = TRUE;
GError *stat_error = NULL;
/* Avoid nested calls */
if (task->flags & RSPAMD_TASK_FLAG_PROCESSING) {
return TRUE;
}
if (RSPAMD_TASK_IS_PROCESSED (task)) {
return TRUE;
}
task->flags |= RSPAMD_TASK_FLAG_PROCESSING;
st = rspamd_task_select_processing_stage (task, stages);
switch (st) {
case RSPAMD_TASK_STAGE_READ_MESSAGE:
if (!rspamd_message_parse (task)) {
ret = FALSE;
}
break;
case RSPAMD_TASK_STAGE_PRE_FILTERS:
rspamd_lua_call_pre_filters (task);
break;
case RSPAMD_TASK_STAGE_FILTERS:
if (!rspamd_process_filters (task)) {
ret = FALSE;
}
break;
case RSPAMD_TASK_STAGE_CLASSIFIERS:
if (rspamd_stat_classify (task, task->cfg->lua_state, &stat_error) ==
RSPAMD_STAT_PROCESS_ERROR) {
msg_err_task ("classify error: %e", stat_error);
g_error_free (stat_error);
}
break;
case RSPAMD_TASK_STAGE_COMPOSITES:
rspamd_make_composites (task);
break;
case RSPAMD_TASK_STAGE_POST_FILTERS:
rspamd_lua_call_post_filters (task);
break;
case RSPAMD_TASK_STAGE_DONE:
task->processed_stages |= RSPAMD_TASK_STAGE_DONE;
break;
default:
/* TODO: not implemented stage */
break;
}
if (RSPAMD_TASK_IS_SKIPPED (task)) {
task->processed_stages |= RSPAMD_TASK_STAGE_DONE;
}
task->flags &= ~RSPAMD_TASK_FLAG_PROCESSING;
if (!ret || RSPAMD_TASK_IS_PROCESSED (task)) {
if (!ret) {
/* Set processed flags */
task->processed_stages |= RSPAMD_TASK_STAGE_DONE;
}
msg_debug_task ("task is processed", st);
return ret;
}
if (rspamd_session_events_pending (task->s) != 0) {
/* We have events pending, so we consider this stage as incomplete */
msg_debug_task ("need more work on stage %d", st);
}
else {
/* Mark the current stage as done and go to the next stage */
msg_debug_task ("completed stage %d", st);
task->processed_stages |= st;
/* Reset checkpoint */
task->checkpoint = NULL;
/* Tail recursion */
return rspamd_task_process (task, stages);
}
return ret;
}
gboolean
rspamd_symbols_cache_process_symbols (struct rspamd_task * task,
struct symbols_cache *cache)
{
struct cache_item *item = NULL;
struct cache_savepoint *checkpoint;
gint i;
gdouble total_microseconds = 0;
const gdouble max_microseconds = 3e5;
guint start_events_pending;
g_assert (cache != NULL);
if (task->checkpoint == NULL) {
checkpoint = rspamd_symbols_cache_make_checkpoint (task, cache);
task->checkpoint = checkpoint;
}
else {
checkpoint = task->checkpoint;
}
if (task->settings) {
if (rspamd_symbols_cache_process_settings (task, cache)) {
return TRUE;
}
}
msg_debug_task ("symbols processing stage at pass: %d", checkpoint->pass);
start_events_pending = rspamd_session_events_pending (task->s);
if (checkpoint->pass == 0) {
/*
* On the first pass we check symbols that do not have dependencies
* If we figure out symbol that has no dependencies satisfied, then
* we just save it for another pass
*/
for (i = 0; i < (gint)checkpoint->version; i ++) {
if (rspamd_symbols_cache_metric_limit (task, checkpoint)) {
msg_info_task ("<%s> has already scored more than %.2f, so do "
"not "
"plan any more checks", task->message_id,
checkpoint->rs->score);
return TRUE;
}
item = g_ptr_array_index (checkpoint->order->d, i);
if (!isset (checkpoint->processed_bits, item->id * 2)) {
if (!rspamd_symbols_cache_check_deps (task, cache, item,
checkpoint)) {
msg_debug_task ("blocked execution of %d unless deps are "
"resolved",
item->id);
g_ptr_array_add (checkpoint->waitq, item);
continue;
}
rspamd_symbols_cache_check_symbol (task, cache, item,
checkpoint, &total_microseconds);
}
if (total_microseconds > max_microseconds) {
/* Maybe we should stop and check pending events? */
if (rspamd_session_events_pending (task->s) > start_events_pending) {
/* Add some timeout event to avoid too long waiting */
#if 0
struct event *ev;
struct timeval tv;
rspamd_session_add_event (task->s,
rspamd_symbols_cache_continuation, task,
rspamd_symbols_cache_quark ());
ev = rspamd_mempool_alloc (task->task_pool, sizeof (*ev));
event_set (ev, -1, EV_TIMEOUT, rspamd_symbols_cache_tm, task);
event_base_set (task->ev_base, ev);
msec_to_tv (50, &tv);
event_add (ev, &tv);
rspamd_mempool_add_destructor (task->task_pool,
(rspamd_mempool_destruct_t)event_del, ev);
#endif
msg_info_task ("trying to check async events after spending "
"%d microseconds processing symbols",
(gint)total_microseconds);
return TRUE;
}
}
}
checkpoint->pass ++;
}
else {
/* We just go through the blocked symbols and check if they are ready */
for (i = 0; i < (gint)checkpoint->waitq->len; i ++) {
item = g_ptr_array_index (checkpoint->waitq, i);
if (item->id >= (gint)checkpoint->version) {
continue;
}
if (!isset (checkpoint->processed_bits, item->id * 2)) {
if (!rspamd_symbols_cache_check_deps (task, cache, item,
checkpoint)) {
break;
}
rspamd_symbols_cache_check_symbol (task, cache, item,
checkpoint, &total_microseconds);
}
if (total_microseconds > max_microseconds) {
/* Maybe we should stop and check pending events? */
if (rspamd_session_events_pending (task->s) >
start_events_pending) {
msg_debug_task ("trying to check async events after spending "
"%d microseconds processing symbols",
(gint)total_microseconds);
return TRUE;
}
}
}
}
return TRUE;
}
static gboolean
rspamd_symbols_cache_check_deps (struct rspamd_task *task,
struct symbols_cache *cache,
struct cache_item *item,
struct cache_savepoint *checkpoint)
{
struct cache_dependency *dep;
guint i;
gboolean ret = TRUE;
if (item->deps != NULL && item->deps->len > 0) {
for (i = 0; i < item->deps->len; i ++) {
dep = g_ptr_array_index (item->deps, i);
if (dep->item == NULL) {
/* Assume invalid deps as done */
continue;
}
if (dep->id >= (gint)checkpoint->version) {
/*
* This is dependency on some symbol that is currently
* not in this checkpoint. So we just pretend that the
* corresponding symbold does not exist
*/
continue;
}
if (!isset (checkpoint->processed_bits, dep->id * 2 + 1)) {
if (!isset (checkpoint->processed_bits, dep->id * 2)) {
/* Not started */
if (!rspamd_symbols_cache_check_deps (task, cache,
dep->item,
checkpoint)) {
g_ptr_array_add (checkpoint->waitq, item);
ret = FALSE;
msg_debug_task ("delayed dependency %d for symbol %d",
dep->id, item->id);
}
else if (!rspamd_symbols_cache_check_symbol (task, cache,
dep->item,
checkpoint,
NULL)) {
/* Now started, but has events pending */
ret = FALSE;
msg_debug_task ("started check of %d symbol as dep for "
"%d",
dep->id, item->id);
}
else {
msg_debug_task ("dependency %d for symbol %d is "
"already processed",
dep->id, item->id);
}
}
else {
/* Started but not finished */
ret = FALSE;
}
}
else {
msg_debug_task ("dependency %d for symbol %d is already "
"checked",
dep->id, item->id);
}
}
}
return ret;
}
static gboolean
rspamd_symbols_cache_check_symbol (struct rspamd_task *task,
struct symbols_cache *cache,
struct cache_item *item,
struct cache_savepoint *checkpoint,
gdouble *total_diff)
{
guint pending_before, pending_after;
double t1, t2;
gdouble diff;
struct rspamd_task **ptask;
lua_State *L;
gboolean check = TRUE;
const gdouble slow_diff_limit = 1e5;
if (item->type & (SYMBOL_TYPE_NORMAL|SYMBOL_TYPE_CALLBACK)) {
g_assert (item->func != NULL);
/* Check has been started */
setbit (checkpoint->processed_bits, item->id * 2);
if (RSPAMD_TASK_IS_EMPTY (task) && !(item->type & SYMBOL_TYPE_EMPTY)) {
check = FALSE;
}
else if (item->condition_cb != -1) {
/* We also executes condition callback to check if we need this symbol */
L = task->cfg->lua_state;
lua_rawgeti (L, LUA_REGISTRYINDEX, item->condition_cb);
ptask = lua_newuserdata (L, sizeof (struct rspamd_task *));
rspamd_lua_setclass (L, "rspamd{task}", -1);
*ptask = task;
if (lua_pcall (L, 1, 1, 0) != 0) {
msg_info_task ("call to condition for %s failed: %s",
item->symbol, lua_tostring (L, -1));
lua_pop (L, 1);
}
else {
check = lua_toboolean (L, -1);
lua_pop (L, 1);
}
}
if (check) {
t1 = rspamd_get_ticks ();
pending_before = rspamd_session_events_pending (task->s);
/* Watch for events appeared */
rspamd_session_watch_start (task->s, rspamd_symbols_cache_watcher_cb,
item);
msg_debug_task ("execute %s, %d", item->symbol, item->id);
item->func (task, item->user_data);
t2 = rspamd_get_ticks ();
diff = (t2 - t1) * 1e6;
if (total_diff) {
*total_diff += diff;
}
if (diff > slow_diff_limit) {
msg_info_task ("slow rule: %s: %d ms", item->symbol,
(gint)(diff / 1000.));
}
rspamd_set_counter (item, diff);
rspamd_session_watch_stop (task->s);
pending_after = rspamd_session_events_pending (task->s);
if (pending_before == pending_after) {
/* No new events registered */
setbit (checkpoint->processed_bits, item->id * 2 + 1);
return TRUE;
}
return FALSE;
}
else {
msg_debug_task ("skipping check of %s as its condition is false",
item->symbol);
setbit (checkpoint->processed_bits, item->id * 2 + 1);
return TRUE;
}
}
else {
setbit (checkpoint->processed_bits, item->id * 2);
setbit (checkpoint->processed_bits, item->id * 2 + 1);
return TRUE;
}
}
