static gboolean rspamd_symbols_cache_check_symbol (struct rspamd_task *task, struct symbols_cache *cache, struct cache_item *item, struct cache_savepoint *checkpoint) { guint pending_before, pending_after; double t1, t2; guint64 diff; if (item->type == SYMBOL_TYPE_NORMAL || item->type == SYMBOL_TYPE_CALLBACK) { g_assert (item->func != NULL); /* Check has been started */ setbit (checkpoint->processed_bits, item->id * 2); t1 = rspamd_get_ticks (); pending_before = rspamd_session_events_pending (task->s); /* Watch for events appeared */ rspamd_session_watch_start (task->s, rspamd_symbols_cache_watcher_cb, item); if (item->symbol != NULL && G_UNLIKELY (check_debug_symbol (task->cfg, item->symbol))) { rspamd_log_debug (rspamd_main->logger); item->func (task, item->user_data); rspamd_log_nodebug (rspamd_main->logger); } else { item->func (task, item->user_data); } t2 = rspamd_get_ticks (); diff = (t2 - t1) * 1000000; rspamd_set_counter (item, diff); rspamd_session_watch_stop (task->s); pending_after = rspamd_session_events_pending (task->s); if (pending_before == pending_after) { /* No new events registered */ setbit (checkpoint->processed_bits, item->id * 2 + 1); return TRUE; } return FALSE; } else { setbit (checkpoint->processed_bits, item->id * 2); setbit (checkpoint->processed_bits, item->id * 2 + 1); return TRUE; } }
gboolean rspamd_task_process (struct rspamd_task *task, guint stages) { gint st; gboolean ret = TRUE; GError *stat_error = NULL; /* Avoid nested calls */ if (task->flags & RSPAMD_TASK_FLAG_PROCESSING) { return TRUE; } if (RSPAMD_TASK_IS_PROCESSED (task)) { return TRUE; } task->flags |= RSPAMD_TASK_FLAG_PROCESSING; st = rspamd_task_select_processing_stage (task, stages); switch (st) { case RSPAMD_TASK_STAGE_READ_MESSAGE: if (!rspamd_message_parse (task)) { ret = FALSE; } break; case RSPAMD_TASK_STAGE_PRE_FILTERS: rspamd_lua_call_pre_filters (task); break; case RSPAMD_TASK_STAGE_FILTERS: if (!rspamd_process_filters (task)) { ret = FALSE; } break; case RSPAMD_TASK_STAGE_CLASSIFIERS: if (rspamd_stat_classify (task, task->cfg->lua_state, &stat_error) == RSPAMD_STAT_PROCESS_ERROR) { msg_err_task ("classify error: %e", stat_error); g_error_free (stat_error); } break; case RSPAMD_TASK_STAGE_COMPOSITES: rspamd_make_composites (task); break; case RSPAMD_TASK_STAGE_POST_FILTERS: rspamd_lua_call_post_filters (task); break; case RSPAMD_TASK_STAGE_DONE: task->processed_stages |= RSPAMD_TASK_STAGE_DONE; break; default: /* TODO: not implemented stage */ break; } if (RSPAMD_TASK_IS_SKIPPED (task)) { task->processed_stages |= RSPAMD_TASK_STAGE_DONE; } task->flags &= ~RSPAMD_TASK_FLAG_PROCESSING; if (!ret || RSPAMD_TASK_IS_PROCESSED (task)) { if (!ret) { /* Set processed flags */ task->processed_stages |= RSPAMD_TASK_STAGE_DONE; } msg_debug_task ("task is processed", st); return ret; } if (rspamd_session_events_pending (task->s) != 0) { /* We have events pending, so we consider this stage as incomplete */ msg_debug_task ("need more work on stage %d", st); } else { /* Mark the current stage as done and go to the next stage */ msg_debug_task ("completed stage %d", st); task->processed_stages |= st; /* Reset checkpoint */ task->checkpoint = NULL; /* Tail recursion */ return rspamd_task_process (task, stages); } return ret; }
gboolean rspamd_symbols_cache_process_symbols (struct rspamd_task * task, struct symbols_cache *cache) { struct cache_item *item = NULL; struct cache_savepoint *checkpoint; gint i; gdouble total_microseconds = 0; const gdouble max_microseconds = 3e5; guint start_events_pending; g_assert (cache != NULL); if (task->checkpoint == NULL) { checkpoint = rspamd_symbols_cache_make_checkpoint (task, cache); task->checkpoint = checkpoint; } else { checkpoint = task->checkpoint; } if (task->settings) { if (rspamd_symbols_cache_process_settings (task, cache)) { return TRUE; } } msg_debug_task ("symbols processing stage at pass: %d", checkpoint->pass); start_events_pending = rspamd_session_events_pending (task->s); if (checkpoint->pass == 0) { /* * On the first pass we check symbols that do not have dependencies * If we figure out symbol that has no dependencies satisfied, then * we just save it for another pass */ for (i = 0; i < (gint)checkpoint->version; i ++) { if (rspamd_symbols_cache_metric_limit (task, checkpoint)) { msg_info_task ("<%s> has already scored more than %.2f, so do " "not " "plan any more checks", task->message_id, checkpoint->rs->score); return TRUE; } item = g_ptr_array_index (checkpoint->order->d, i); if (!isset (checkpoint->processed_bits, item->id * 2)) { if (!rspamd_symbols_cache_check_deps (task, cache, item, checkpoint)) { msg_debug_task ("blocked execution of %d unless deps are " "resolved", item->id); g_ptr_array_add (checkpoint->waitq, item); continue; } rspamd_symbols_cache_check_symbol (task, cache, item, checkpoint, &total_microseconds); } if (total_microseconds > max_microseconds) { /* Maybe we should stop and check pending events? */ if (rspamd_session_events_pending (task->s) > start_events_pending) { /* Add some timeout event to avoid too long waiting */ #if 0 struct event *ev; struct timeval tv; rspamd_session_add_event (task->s, rspamd_symbols_cache_continuation, task, rspamd_symbols_cache_quark ()); ev = rspamd_mempool_alloc (task->task_pool, sizeof (*ev)); event_set (ev, -1, EV_TIMEOUT, rspamd_symbols_cache_tm, task); event_base_set (task->ev_base, ev); msec_to_tv (50, &tv); event_add (ev, &tv); rspamd_mempool_add_destructor (task->task_pool, (rspamd_mempool_destruct_t)event_del, ev); #endif msg_info_task ("trying to check async events after spending " "%d microseconds processing symbols", (gint)total_microseconds); return TRUE; } } } checkpoint->pass ++; } else { /* We just go through the blocked symbols and check if they are ready */ for (i = 0; i < (gint)checkpoint->waitq->len; i ++) { item = g_ptr_array_index (checkpoint->waitq, i); if (item->id >= (gint)checkpoint->version) { continue; } if (!isset (checkpoint->processed_bits, item->id * 2)) { if (!rspamd_symbols_cache_check_deps (task, cache, item, checkpoint)) { break; } rspamd_symbols_cache_check_symbol (task, cache, item, checkpoint, &total_microseconds); } if (total_microseconds > max_microseconds) { /* Maybe we should stop and check pending events? */ if (rspamd_session_events_pending (task->s) > start_events_pending) { msg_debug_task ("trying to check async events after spending " "%d microseconds processing symbols", (gint)total_microseconds); return TRUE; } } } } return TRUE; }
static gboolean rspamd_symbols_cache_check_symbol (struct rspamd_task *task, struct symbols_cache *cache, struct cache_item *item, struct cache_savepoint *checkpoint, gdouble *total_diff) { guint pending_before, pending_after; double t1, t2; gdouble diff; struct rspamd_task **ptask; lua_State *L; gboolean check = TRUE; const gdouble slow_diff_limit = 1e5; if (item->type & (SYMBOL_TYPE_NORMAL|SYMBOL_TYPE_CALLBACK)) { g_assert (item->func != NULL); /* Check has been started */ setbit (checkpoint->processed_bits, item->id * 2); if (RSPAMD_TASK_IS_EMPTY (task) && !(item->type & SYMBOL_TYPE_EMPTY)) { check = FALSE; } else if (item->condition_cb != -1) { /* We also executes condition callback to check if we need this symbol */ L = task->cfg->lua_state; lua_rawgeti (L, LUA_REGISTRYINDEX, item->condition_cb); ptask = lua_newuserdata (L, sizeof (struct rspamd_task *)); rspamd_lua_setclass (L, "rspamd{task}", -1); *ptask = task; if (lua_pcall (L, 1, 1, 0) != 0) { msg_info_task ("call to condition for %s failed: %s", item->symbol, lua_tostring (L, -1)); lua_pop (L, 1); } else { check = lua_toboolean (L, -1); lua_pop (L, 1); } } if (check) { t1 = rspamd_get_ticks (); pending_before = rspamd_session_events_pending (task->s); /* Watch for events appeared */ rspamd_session_watch_start (task->s, rspamd_symbols_cache_watcher_cb, item); msg_debug_task ("execute %s, %d", item->symbol, item->id); item->func (task, item->user_data); t2 = rspamd_get_ticks (); diff = (t2 - t1) * 1e6; if (total_diff) { *total_diff += diff; } if (diff > slow_diff_limit) { msg_info_task ("slow rule: %s: %d ms", item->symbol, (gint)(diff / 1000.)); } rspamd_set_counter (item, diff); rspamd_session_watch_stop (task->s); pending_after = rspamd_session_events_pending (task->s); if (pending_before == pending_after) { /* No new events registered */ setbit (checkpoint->processed_bits, item->id * 2 + 1); return TRUE; } return FALSE; } else { msg_debug_task ("skipping check of %s as its condition is false", item->symbol); setbit (checkpoint->processed_bits, item->id * 2 + 1); return TRUE; } } else { setbit (checkpoint->processed_bits, item->id * 2); setbit (checkpoint->processed_bits, item->id * 2 + 1); return TRUE; } }