static gboolean
rspamd_symbols_cache_check_symbol (struct rspamd_task *task,
struct symbols_cache *cache,
struct cache_item *item,
struct cache_savepoint *checkpoint)
{
guint pending_before, pending_after;
double t1, t2;
guint64 diff;
if (item->type == SYMBOL_TYPE_NORMAL || item->type == SYMBOL_TYPE_CALLBACK) {
g_assert (item->func != NULL);
/* Check has been started */
setbit (checkpoint->processed_bits, item->id * 2);
t1 = rspamd_get_ticks ();
pending_before = rspamd_session_events_pending (task->s);
/* Watch for events appeared */
rspamd_session_watch_start (task->s, rspamd_symbols_cache_watcher_cb,
item);
if (item->symbol != NULL &&
G_UNLIKELY (check_debug_symbol (task->cfg, item->symbol))) {
rspamd_log_debug (rspamd_main->logger);
item->func (task, item->user_data);
rspamd_log_nodebug (rspamd_main->logger);
}
else {
item->func (task, item->user_data);
}
t2 = rspamd_get_ticks ();
diff = (t2 - t1) * 1000000;
rspamd_set_counter (item, diff);
rspamd_session_watch_stop (task->s);
pending_after = rspamd_session_events_pending (task->s);
if (pending_before == pending_after) {
/* No new events registered */
setbit (checkpoint->processed_bits, item->id * 2 + 1);
return TRUE;
}
return FALSE;
}
else {
setbit (checkpoint->processed_bits, item->id * 2);
setbit (checkpoint->processed_bits, item->id * 2 + 1);
return TRUE;
}
}
gboolean
rspamd_task_process (struct rspamd_task *task, guint stages)
{
gint st;
gboolean ret = TRUE;
GError *stat_error = NULL;
/* Avoid nested calls */
if (task->flags & RSPAMD_TASK_FLAG_PROCESSING) {
return TRUE;
}
if (RSPAMD_TASK_IS_PROCESSED (task)) {
return TRUE;
}
task->flags |= RSPAMD_TASK_FLAG_PROCESSING;
st = rspamd_task_select_processing_stage (task, stages);
switch (st) {
case RSPAMD_TASK_STAGE_READ_MESSAGE:
if (!rspamd_message_parse (task)) {
ret = FALSE;
}
break;
case RSPAMD_TASK_STAGE_PRE_FILTERS:
rspamd_lua_call_pre_filters (task);
break;
case RSPAMD_TASK_STAGE_FILTERS:
if (!rspamd_process_filters (task)) {
ret = FALSE;
}
break;
case RSPAMD_TASK_STAGE_CLASSIFIERS:
if (rspamd_stat_classify (task, task->cfg->lua_state, &stat_error) ==
RSPAMD_STAT_PROCESS_ERROR) {
msg_err_task ("classify error: %e", stat_error);
g_error_free (stat_error);
}
break;
case RSPAMD_TASK_STAGE_COMPOSITES:
rspamd_make_composites (task);
break;
case RSPAMD_TASK_STAGE_POST_FILTERS:
rspamd_lua_call_post_filters (task);
break;
case RSPAMD_TASK_STAGE_DONE:
task->processed_stages |= RSPAMD_TASK_STAGE_DONE;
break;
default:
/* TODO: not implemented stage */
break;
}
if (RSPAMD_TASK_IS_SKIPPED (task)) {
task->processed_stages |= RSPAMD_TASK_STAGE_DONE;
}
task->flags &= ~RSPAMD_TASK_FLAG_PROCESSING;
if (!ret || RSPAMD_TASK_IS_PROCESSED (task)) {
if (!ret) {
/* Set processed flags */
task->processed_stages |= RSPAMD_TASK_STAGE_DONE;
}
msg_debug_task ("task is processed", st);
return ret;
}
if (rspamd_session_events_pending (task->s) != 0) {
/* We have events pending, so we consider this stage as incomplete */
msg_debug_task ("need more work on stage %d", st);
}
else {
/* Mark the current stage as done and go to the next stage */
msg_debug_task ("completed stage %d", st);
task->processed_stages |= st;
/* Reset checkpoint */
task->checkpoint = NULL;
/* Tail recursion */
return rspamd_task_process (task, stages);
}
return ret;
}
gboolean
rspamd_symbols_cache_process_symbols (struct rspamd_task * task,
struct symbols_cache *cache)
{
struct cache_item *item = NULL;
struct cache_savepoint *checkpoint;
gint i;
gdouble total_microseconds = 0;
const gdouble max_microseconds = 3e5;
guint start_events_pending;
g_assert (cache != NULL);
if (task->checkpoint == NULL) {
checkpoint = rspamd_symbols_cache_make_checkpoint (task, cache);
task->checkpoint = checkpoint;
}
else {
checkpoint = task->checkpoint;
}
if (task->settings) {
if (rspamd_symbols_cache_process_settings (task, cache)) {
return TRUE;
}
}
msg_debug_task ("symbols processing stage at pass: %d", checkpoint->pass);
start_events_pending = rspamd_session_events_pending (task->s);
if (checkpoint->pass == 0) {
/*
* On the first pass we check symbols that do not have dependencies
* If we figure out symbol that has no dependencies satisfied, then
* we just save it for another pass
*/
for (i = 0; i < (gint)checkpoint->version; i ++) {
if (rspamd_symbols_cache_metric_limit (task, checkpoint)) {
msg_info_task ("<%s> has already scored more than %.2f, so do "
"not "
"plan any more checks", task->message_id,
checkpoint->rs->score);
return TRUE;
}
item = g_ptr_array_index (checkpoint->order->d, i);
if (!isset (checkpoint->processed_bits, item->id * 2)) {
if (!rspamd_symbols_cache_check_deps (task, cache, item,
checkpoint)) {
msg_debug_task ("blocked execution of %d unless deps are "
"resolved",
item->id);
g_ptr_array_add (checkpoint->waitq, item);
continue;
}
rspamd_symbols_cache_check_symbol (task, cache, item,
checkpoint, &total_microseconds);
}
if (total_microseconds > max_microseconds) {
/* Maybe we should stop and check pending events? */
if (rspamd_session_events_pending (task->s) > start_events_pending) {
/* Add some timeout event to avoid too long waiting */
#if 0
struct event *ev;
struct timeval tv;
rspamd_session_add_event (task->s,
rspamd_symbols_cache_continuation, task,
rspamd_symbols_cache_quark ());
ev = rspamd_mempool_alloc (task->task_pool, sizeof (*ev));
event_set (ev, -1, EV_TIMEOUT, rspamd_symbols_cache_tm, task);
event_base_set (task->ev_base, ev);
msec_to_tv (50, &tv);
event_add (ev, &tv);
rspamd_mempool_add_destructor (task->task_pool,
(rspamd_mempool_destruct_t)event_del, ev);
#endif
msg_info_task ("trying to check async events after spending "
"%d microseconds processing symbols",
(gint)total_microseconds);
return TRUE;
}
}
}
checkpoint->pass ++;
}
else {
/* We just go through the blocked symbols and check if they are ready */
for (i = 0; i < (gint)checkpoint->waitq->len; i ++) {
item = g_ptr_array_index (checkpoint->waitq, i);
if (item->id >= (gint)checkpoint->version) {
continue;
}
if (!isset (checkpoint->processed_bits, item->id * 2)) {
if (!rspamd_symbols_cache_check_deps (task, cache, item,
checkpoint)) {
break;
}
rspamd_symbols_cache_check_symbol (task, cache, item,
checkpoint, &total_microseconds);
}
if (total_microseconds > max_microseconds) {
/* Maybe we should stop and check pending events? */
if (rspamd_session_events_pending (task->s) >
start_events_pending) {
msg_debug_task ("trying to check async events after spending "
"%d microseconds processing symbols",
(gint)total_microseconds);
return TRUE;
}
}
}
}
return TRUE;
}
static gboolean
rspamd_symbols_cache_check_symbol (struct rspamd_task *task,
struct symbols_cache *cache,
struct cache_item *item,
struct cache_savepoint *checkpoint,
gdouble *total_diff)
{
guint pending_before, pending_after;
double t1, t2;
gdouble diff;
struct rspamd_task **ptask;
lua_State *L;
gboolean check = TRUE;
const gdouble slow_diff_limit = 1e5;
if (item->type & (SYMBOL_TYPE_NORMAL|SYMBOL_TYPE_CALLBACK)) {
g_assert (item->func != NULL);
/* Check has been started */
setbit (checkpoint->processed_bits, item->id * 2);
if (RSPAMD_TASK_IS_EMPTY (task) && !(item->type & SYMBOL_TYPE_EMPTY)) {
check = FALSE;
}
else if (item->condition_cb != -1) {
/* We also executes condition callback to check if we need this symbol */
L = task->cfg->lua_state;
lua_rawgeti (L, LUA_REGISTRYINDEX, item->condition_cb);
ptask = lua_newuserdata (L, sizeof (struct rspamd_task *));
rspamd_lua_setclass (L, "rspamd{task}", -1);
*ptask = task;
if (lua_pcall (L, 1, 1, 0) != 0) {
msg_info_task ("call to condition for %s failed: %s",
item->symbol, lua_tostring (L, -1));
lua_pop (L, 1);
}
else {
check = lua_toboolean (L, -1);
lua_pop (L, 1);
}
}
if (check) {
t1 = rspamd_get_ticks ();
pending_before = rspamd_session_events_pending (task->s);
/* Watch for events appeared */
rspamd_session_watch_start (task->s, rspamd_symbols_cache_watcher_cb,
item);
msg_debug_task ("execute %s, %d", item->symbol, item->id);
item->func (task, item->user_data);
t2 = rspamd_get_ticks ();
diff = (t2 - t1) * 1e6;
if (total_diff) {
*total_diff += diff;
}
if (diff > slow_diff_limit) {
msg_info_task ("slow rule: %s: %d ms", item->symbol,
(gint)(diff / 1000.));
}
rspamd_set_counter (item, diff);
rspamd_session_watch_stop (task->s);
pending_after = rspamd_session_events_pending (task->s);
if (pending_before == pending_after) {
/* No new events registered */
setbit (checkpoint->processed_bits, item->id * 2 + 1);
return TRUE;
}
return FALSE;
}
else {
msg_debug_task ("skipping check of %s as its condition is false",
item->symbol);
setbit (checkpoint->processed_bits, item->id * 2 + 1);
return TRUE;
}
}
else {
setbit (checkpoint->processed_bits, item->id * 2);
setbit (checkpoint->processed_bits, item->id * 2 + 1);
return TRUE;
}
}
