static gboolean rspamd_lua_call_expression_func(
struct ucl_lua_funcdata *lua_data, struct rspamd_task *task,
GArray *args, gint *res)
{
lua_State *L = lua_data->L;
struct rspamd_task **ptask;
struct expression_argument *arg;
gint pop = 0, i, nargs = 0;
lua_rawgeti (L, LUA_REGISTRYINDEX, lua_data->idx);
/* Now we got function in top of stack */
ptask = lua_newuserdata (L, sizeof(struct rspamd_task *));
rspamd_lua_setclass (L, "rspamd{task}", -1);
*ptask = task;
/* Now push all arguments */
if (args) {
for (i = 0; i < (gint)args->len; i ++) {
arg = &g_array_index (args, struct expression_argument, i);
if (arg) {
switch (arg->type) {
case EXPRESSION_ARGUMENT_NORMAL:
lua_pushstring (L, (const gchar *) arg->data);
break;
case EXPRESSION_ARGUMENT_BOOL:
lua_pushboolean (L, (gboolean) GPOINTER_TO_SIZE(arg->data));
break;
default:
msg_err_task ("cannot pass custom params to lua function");
return FALSE;
}
}
}
nargs = args->len;
}
if (lua_pcall (L, nargs + 1, 1, 0) != 0) {
msg_info_task ("call to lua function failed: %s", lua_tostring (L, -1));
return FALSE;
}
pop++;
if (lua_type (L, -1) == LUA_TNUMBER) {
*res = lua_tonumber (L, -1);
}
else if (lua_type (L, -1) == LUA_TBOOLEAN) {
*res = lua_toboolean (L, -1);
}
else {
msg_info_task ("lua function must return a boolean");
}
lua_pop (L, pop);
return TRUE;
}
void
rspamd_symbols_cache_disable_symbol (struct rspamd_task *task,
struct symbols_cache *cache, const gchar *symbol)
{
struct cache_savepoint *checkpoint;
struct cache_item *item;
gint id;
if (task->checkpoint == NULL) {
checkpoint = rspamd_symbols_cache_make_checkpoint (task, cache);
task->checkpoint = checkpoint;
}
else {
checkpoint = task->checkpoint;
}
id = rspamd_symbols_cache_find_symbol_parent (cache, symbol);
if (id > 0) {
/* Set executed and finished flags */
item = g_ptr_array_index (cache->items_by_id, id);
setbit (checkpoint->processed_bits, item->id * 2);
setbit (checkpoint->processed_bits, item->id * 2 + 1);
msg_debug_task ("disable execution of %s", symbol);
}
else {
msg_info_task ("cannot disable %s: not found", symbol);
}
}
static gboolean
rspamd_symbols_cache_process_settings (struct rspamd_task *task,
struct symbols_cache *cache)
{
const ucl_object_t *wl, *cur, *disabled;
struct metric *def;
struct rspamd_symbols_group *gr;
GHashTableIter gr_it;
ucl_object_iter_t it = NULL;
gpointer k, v;
wl = ucl_object_lookup (task->settings, "whitelist");
if (wl != NULL) {
msg_info_task ("<%s> is whitelisted", task->message_id);
task->flags |= RSPAMD_TASK_FLAG_SKIP;
return TRUE;
}
disabled = ucl_object_lookup (task->settings, "symbols_disabled");
if (disabled) {
it = NULL;
while ((cur = ucl_iterate_object (disabled, &it, true)) != NULL) {
rspamd_symbols_cache_disable_symbol (task, cache,
ucl_object_tostring (cur));
}
}
/* Disable groups of symbols */
disabled = ucl_object_lookup (task->settings, "groups_disabled");
def = g_hash_table_lookup (task->cfg->metrics, DEFAULT_METRIC);
if (def && disabled) {
it = NULL;
while ((cur = ucl_iterate_object (disabled, &it, true)) != NULL) {
if (ucl_object_type (cur) == UCL_STRING) {
gr = g_hash_table_lookup (def->groups,
ucl_object_tostring (cur));
if (gr) {
g_hash_table_iter_init (&gr_it, gr->symbols);
while (g_hash_table_iter_next (&gr_it, &k, &v)) {
rspamd_symbols_cache_disable_symbol (task, cache, k);
}
}
}
}
}
return FALSE;
}
static void
dkim_module_key_handler (rspamd_dkim_key_t *key,
gsize keylen,
rspamd_dkim_context_t *ctx,
gpointer ud,
GError *err)
{
struct dkim_check_result *res = ud;
struct rspamd_task *task;
task = res->task;
if (key != NULL) {
/*
* We actually receive key with refcount = 1, so we just assume that
* lru hash owns this object now
*/
rspamd_lru_hash_insert (dkim_module_ctx->dkim_hash,
g_strdup (rspamd_dkim_get_dns_key (ctx)),
key, res->task->tv.tv_sec, rspamd_dkim_key_get_ttl (key));
/* Another ref belongs to the check context */
res->key = rspamd_dkim_key_ref (key);
/* Release key when task is processed */
rspamd_mempool_add_destructor (res->task->task_pool,
dkim_module_key_dtor, res->key);
}
else {
/* Insert tempfail symbol */
msg_info_task ("cannot get key for domain %s: %e",
rspamd_dkim_get_dns_key (ctx), err);
if (err != NULL) {
if (err->code == DKIM_SIGERROR_NOKEY) {
res->res = DKIM_TRYAGAIN;
}
else {
res->res = DKIM_PERM_ERROR;
}
}
}
if (err) {
g_error_free (err);
}
dkim_module_check (res);
}
static void
dkim_symbol_callback (struct rspamd_task *task, void *unused)
{
GList *hlist;
rspamd_dkim_context_t *ctx;
rspamd_dkim_key_t *key;
GError *err = NULL;
struct raw_header *rh;
struct dkim_check_result *res = NULL, *cur;
guint checked = 0;
/* First check if plugin should be enabled */
if (task->user != NULL || rspamd_inet_address_is_local (task->from_addr)) {
msg_info_task ("skip DKIM checks for local networks and authorized users");
return;
}
/* Check whitelist */
if (radix_find_compressed_addr (dkim_module_ctx->whitelist_ip,
task->from_addr) != RADIX_NO_VALUE) {
msg_info_task ("skip DKIM checks for whitelisted address");
return;
}
/* Now check if a message has its signature */
hlist = rspamd_message_get_header (task,
DKIM_SIGNHEADER,
FALSE);
if (hlist != NULL) {
msg_debug_task ("dkim signature found");
while (hlist != NULL) {
rh = (struct raw_header *)hlist->data;
if (rh->decoded == NULL || rh->decoded[0] == '\0') {
msg_info_task ("<%s> cannot load empty DKIM context",
task->message_id);
hlist = g_list_next (hlist);
continue;
}
if (res == NULL) {
res = rspamd_mempool_alloc0 (task->task_pool, sizeof (*res));
res->prev = res;
res->w = rspamd_session_get_watcher (task->s);
cur = res;
}
else {
cur = rspamd_mempool_alloc0 (task->task_pool, sizeof (*res));
}
cur->first = res;
cur->res = -1;
cur->task = task;
cur->mult_allow = 1.0;
cur->mult_deny = 1.0;
ctx = rspamd_create_dkim_context (rh->decoded,
task->task_pool,
dkim_module_ctx->time_jitter,
&err);
if (ctx == NULL) {
if (err != NULL) {
msg_info_task ("<%s> cannot parse DKIM context: %e",
task->message_id, err);
g_error_free (err);
err = NULL;
}
else {
msg_info_task ("<%s> cannot parse DKIM context: "
"unknown error",
task->message_id);
}
hlist = g_list_next (hlist);
continue;
}
else {
/* Get key */
cur->ctx = ctx;
if (dkim_module_ctx->trusted_only &&
(dkim_module_ctx->dkim_domains == NULL ||
g_hash_table_lookup (dkim_module_ctx->dkim_domains,
rspamd_dkim_get_domain (ctx)) == NULL)) {
msg_debug_task ("skip dkim check for %s domain",
rspamd_dkim_get_domain (ctx));
hlist = g_list_next (hlist);
continue;
}
key = rspamd_lru_hash_lookup (dkim_module_ctx->dkim_hash,
rspamd_dkim_get_dns_key (ctx),
task->tv.tv_sec);
if (key != NULL) {
cur->key = rspamd_dkim_key_ref (key);
/* Release key when task is processed */
rspamd_mempool_add_destructor (task->task_pool,
dkim_module_key_dtor, cur->key);
}
else {
rspamd_get_dkim_key (ctx,
task,
dkim_module_key_handler,
cur);
}
}
if (res != cur) {
DL_APPEND (res, cur);
}
if (dkim_module_ctx->skip_multi) {
if (hlist->next) {
msg_info_task ("message has multiple signatures but we"
" check only one as 'skip_multi' is set");
}
break;
}
checked ++;
if (checked > dkim_module_ctx->max_sigs) {
msg_info_task ("message has multiple signatures but we"
" stopped after %d checked signatures as limit"
" is reached", checked);
break;
}
hlist = g_list_next (hlist);
}
}
else {
rspamd_task_insert_result (task,
dkim_module_ctx->symbol_na,
1.0,
NULL);
}
if (res != NULL) {
rspamd_session_watcher_push (task->s);
dkim_module_check (res);
}
}
static void
insert_metric_result (struct rspamd_task *task,
struct metric *metric,
const gchar *symbol,
double flag,
GList * opts,
gboolean single)
{
struct metric_result *metric_res;
struct symbol *s;
gdouble w, *gr_score = NULL;
struct rspamd_symbol_def *sdef;
struct rspamd_symbols_group *gr = NULL;
const ucl_object_t *mobj, *sobj;
metric_res = rspamd_create_metric_result (task, metric->name);
sdef = g_hash_table_lookup (metric->symbols, symbol);
if (sdef == NULL) {
w = 0.0;
}
else {
w = (*sdef->weight_ptr) * flag;
gr = sdef->gr;
if (gr != NULL) {
gr_score = g_hash_table_lookup (metric_res->sym_groups, gr);
if (gr_score == NULL) {
gr_score = rspamd_mempool_alloc (task->task_pool, sizeof (gdouble));
*gr_score = 0;
g_hash_table_insert (metric_res->sym_groups, gr, gr_score);
}
}
}
if (task->settings) {
mobj = ucl_object_lookup (task->settings, metric->name);
if (mobj) {
gdouble corr;
sobj = ucl_object_lookup (mobj, symbol);
if (sobj != NULL && ucl_object_todouble_safe (sobj, &corr)) {
msg_debug ("settings: changed weight of symbol %s from %.2f to %.2f",
symbol, w, corr);
w = corr * flag;
}
}
}
/* XXX: does not take grow factor into account */
if (gr != NULL && gr_score != NULL && gr->max_score > 0.0) {
if (*gr_score >= gr->max_score) {
msg_info_task ("maximum group score %.2f for group %s has been reached,"
" ignoring symbol %s with weight %.2f", gr->max_score,
gr->name, symbol, w);
return;
}
else if (*gr_score + w > gr->max_score) {
w = gr->max_score - *gr_score;
}
*gr_score += w;
}
/* Add metric score */
if ((s = g_hash_table_lookup (metric_res->symbols, symbol)) != NULL) {
if (sdef && (sdef->flags & RSPAMD_SYMBOL_FLAG_ONESHOT)) {
/*
* For one shot symbols we do not need to add them again, so
* we just force single behaviour here
*/
single = TRUE;
}
if (s->options && opts && opts != s->options) {
/* Append new options */
s->options = g_list_concat (s->options, g_list_copy (opts));
/*
* Note that there is no need to add new destructor of GList as elements of appended
* GList are used directly, so just free initial GList
*/
}
else if (opts) {
s->options = g_list_copy (opts);
rspamd_mempool_add_destructor (task->task_pool,
(rspamd_mempool_destruct_t) g_list_free, s->options);
}
if (!single) {
/* Handle grow factor */
if (metric_res->grow_factor && w > 0) {
w *= metric_res->grow_factor;
metric_res->grow_factor *= metric->grow_factor;
}
s->score += w;
metric_res->score += w;
}
else {
if (fabs (s->score) < fabs (w)) {
/* Replace less weight with a bigger one */
metric_res->score = metric_res->score - s->score + w;
s->score = w;
}
}
}
else {
s = rspamd_mempool_alloc (task->task_pool, sizeof (struct symbol));
/* Handle grow factor */
if (metric_res->grow_factor && w > 0) {
w *= metric_res->grow_factor;
metric_res->grow_factor *= metric->grow_factor;
}
else if (w > 0) {
metric_res->grow_factor = metric->grow_factor;
}
s->score = w;
s->name = symbol;
s->def = sdef;
metric_res->score += w;
if (opts) {
s->options = g_list_copy (opts);
rspamd_mempool_add_destructor (task->task_pool,
(rspamd_mempool_destruct_t) g_list_free, s->options);
}
else {
s->options = NULL;
}
g_hash_table_insert (metric_res->symbols, (gpointer) symbol, s);
}
msg_debug ("symbol %s, score %.2f, metric %s, factor: %f",
symbol,
s->score,
metric->name,
w);
}
gboolean
rspamd_symbols_cache_process_symbols (struct rspamd_task * task,
struct symbols_cache *cache)
{
struct cache_item *item = NULL;
struct cache_savepoint *checkpoint;
gint i;
gdouble total_microseconds = 0;
const gdouble max_microseconds = 3e5;
guint start_events_pending;
g_assert (cache != NULL);
if (task->checkpoint == NULL) {
checkpoint = rspamd_symbols_cache_make_checkpoint (task, cache);
task->checkpoint = checkpoint;
}
else {
checkpoint = task->checkpoint;
}
if (task->settings) {
if (rspamd_symbols_cache_process_settings (task, cache)) {
return TRUE;
}
}
msg_debug_task ("symbols processing stage at pass: %d", checkpoint->pass);
start_events_pending = rspamd_session_events_pending (task->s);
if (checkpoint->pass == 0) {
/*
* On the first pass we check symbols that do not have dependencies
* If we figure out symbol that has no dependencies satisfied, then
* we just save it for another pass
*/
for (i = 0; i < (gint)checkpoint->version; i ++) {
if (rspamd_symbols_cache_metric_limit (task, checkpoint)) {
msg_info_task ("<%s> has already scored more than %.2f, so do "
"not "
"plan any more checks", task->message_id,
checkpoint->rs->score);
return TRUE;
}
item = g_ptr_array_index (checkpoint->order->d, i);
if (!isset (checkpoint->processed_bits, item->id * 2)) {
if (!rspamd_symbols_cache_check_deps (task, cache, item,
checkpoint)) {
msg_debug_task ("blocked execution of %d unless deps are "
"resolved",
item->id);
g_ptr_array_add (checkpoint->waitq, item);
continue;
}
rspamd_symbols_cache_check_symbol (task, cache, item,
checkpoint, &total_microseconds);
}
if (total_microseconds > max_microseconds) {
/* Maybe we should stop and check pending events? */
if (rspamd_session_events_pending (task->s) > start_events_pending) {
/* Add some timeout event to avoid too long waiting */
#if 0
struct event *ev;
struct timeval tv;
rspamd_session_add_event (task->s,
rspamd_symbols_cache_continuation, task,
rspamd_symbols_cache_quark ());
ev = rspamd_mempool_alloc (task->task_pool, sizeof (*ev));
event_set (ev, -1, EV_TIMEOUT, rspamd_symbols_cache_tm, task);
event_base_set (task->ev_base, ev);
msec_to_tv (50, &tv);
event_add (ev, &tv);
rspamd_mempool_add_destructor (task->task_pool,
(rspamd_mempool_destruct_t)event_del, ev);
#endif
msg_info_task ("trying to check async events after spending "
"%d microseconds processing symbols",
(gint)total_microseconds);
return TRUE;
}
}
}
checkpoint->pass ++;
}
else {
/* We just go through the blocked symbols and check if they are ready */
for (i = 0; i < (gint)checkpoint->waitq->len; i ++) {
item = g_ptr_array_index (checkpoint->waitq, i);
if (item->id >= (gint)checkpoint->version) {
continue;
}
if (!isset (checkpoint->processed_bits, item->id * 2)) {
if (!rspamd_symbols_cache_check_deps (task, cache, item,
checkpoint)) {
break;
}
rspamd_symbols_cache_check_symbol (task, cache, item,
checkpoint, &total_microseconds);
}
if (total_microseconds > max_microseconds) {
/* Maybe we should stop and check pending events? */
if (rspamd_session_events_pending (task->s) >
start_events_pending) {
msg_debug_task ("trying to check async events after spending "
"%d microseconds processing symbols",
(gint)total_microseconds);
return TRUE;
}
}
}
}
return TRUE;
}
static gboolean
rspamd_symbols_cache_check_symbol (struct rspamd_task *task,
struct symbols_cache *cache,
struct cache_item *item,
struct cache_savepoint *checkpoint,
gdouble *total_diff)
{
guint pending_before, pending_after;
double t1, t2;
gdouble diff;
struct rspamd_task **ptask;
lua_State *L;
gboolean check = TRUE;
const gdouble slow_diff_limit = 1e5;
if (item->type & (SYMBOL_TYPE_NORMAL|SYMBOL_TYPE_CALLBACK)) {
g_assert (item->func != NULL);
/* Check has been started */
setbit (checkpoint->processed_bits, item->id * 2);
if (RSPAMD_TASK_IS_EMPTY (task) && !(item->type & SYMBOL_TYPE_EMPTY)) {
check = FALSE;
}
else if (item->condition_cb != -1) {
/* We also executes condition callback to check if we need this symbol */
L = task->cfg->lua_state;
lua_rawgeti (L, LUA_REGISTRYINDEX, item->condition_cb);
ptask = lua_newuserdata (L, sizeof (struct rspamd_task *));
rspamd_lua_setclass (L, "rspamd{task}", -1);
*ptask = task;
if (lua_pcall (L, 1, 1, 0) != 0) {
msg_info_task ("call to condition for %s failed: %s",
item->symbol, lua_tostring (L, -1));
lua_pop (L, 1);
}
else {
check = lua_toboolean (L, -1);
lua_pop (L, 1);
}
}
if (check) {
t1 = rspamd_get_ticks ();
pending_before = rspamd_session_events_pending (task->s);
/* Watch for events appeared */
rspamd_session_watch_start (task->s, rspamd_symbols_cache_watcher_cb,
item);
msg_debug_task ("execute %s, %d", item->symbol, item->id);
item->func (task, item->user_data);
t2 = rspamd_get_ticks ();
diff = (t2 - t1) * 1e6;
if (total_diff) {
*total_diff += diff;
}
if (diff > slow_diff_limit) {
msg_info_task ("slow rule: %s: %d ms", item->symbol,
(gint)(diff / 1000.));
}
rspamd_set_counter (item, diff);
rspamd_session_watch_stop (task->s);
pending_after = rspamd_session_events_pending (task->s);
if (pending_before == pending_after) {
/* No new events registered */
setbit (checkpoint->processed_bits, item->id * 2 + 1);
return TRUE;
}
return FALSE;
}
else {
msg_debug_task ("skipping check of %s as its condition is false",
item->symbol);
setbit (checkpoint->processed_bits, item->id * 2 + 1);
return TRUE;
}
}
else {
setbit (checkpoint->processed_bits, item->id * 2);
setbit (checkpoint->processed_bits, item->id * 2 + 1);
return TRUE;
}
}
