static void rspamd_symbols_cache_watcher_cb (gpointer sessiond, gpointer ud) { struct rspamd_task *task = sessiond; struct cache_item *item = ud, *it; struct cache_savepoint *checkpoint; struct symbols_cache *cache; gint i, remain = 0; checkpoint = task->checkpoint; cache = task->cfg->cache; /* Specify that we are done with this item */ setbit (checkpoint->processed_bits, item->id * 2 + 1); if (checkpoint->pass > 0) { for (i = 0; i < (gint)checkpoint->waitq->len; i ++) { it = g_ptr_array_index (checkpoint->waitq, i); if (!isset (checkpoint->processed_bits, it->id * 2)) { if (!rspamd_symbols_cache_check_deps (task, cache, it, checkpoint)) { remain ++; continue; } rspamd_symbols_cache_check_symbol (task, cache, it, checkpoint); } } } msg_debug ("finished watcher, %ud symbols waiting", remain); }
static gboolean rspamd_symbols_cache_check_deps (struct rspamd_task *task, struct symbols_cache *cache, struct cache_item *item, struct cache_savepoint *checkpoint) { struct cache_dependency *dep; guint i; gboolean ret = TRUE; if (item->deps != NULL && item->deps->len > 0) { for (i = 0; i < item->deps->len; i ++) { dep = g_ptr_array_index (item->deps, i); g_assert (dep->item != NULL); if (!isset (checkpoint->processed_bits, dep->id * 2 + 1)) { if (!isset (checkpoint->processed_bits, dep->id * 2)) { /* Not started */ if (!rspamd_symbols_cache_check_deps (task, cache, dep->item, checkpoint)) { g_ptr_array_add (checkpoint->waitq, item); ret = FALSE; } else if (!rspamd_symbols_cache_check_symbol (task, cache, dep->item, checkpoint)) { /* Now started, but has events pending */ ret = FALSE; msg_debug ("started check of %d symbol as dep for %d", dep->id, item->id); } else { msg_debug ("dependency %d for symbol %d is already processed", dep->id, item->id); } } else { /* Started but not finished */ ret = FALSE; } } else { msg_debug ("dependency %d for symbol %d is already checked", dep->id, item->id); } } } return ret; }
gboolean rspamd_symbols_cache_process_symbols (struct rspamd_task * task, struct symbols_cache *cache) { struct cache_item *item = NULL; struct cache_savepoint *checkpoint; gint i; gdouble total_microseconds = 0; const gdouble max_microseconds = 3e5; guint start_events_pending; g_assert (cache != NULL); if (task->checkpoint == NULL) { checkpoint = rspamd_symbols_cache_make_checkpoint (task, cache); task->checkpoint = checkpoint; } else { checkpoint = task->checkpoint; } if (task->settings) { if (rspamd_symbols_cache_process_settings (task, cache)) { return TRUE; } } msg_debug_task ("symbols processing stage at pass: %d", checkpoint->pass); start_events_pending = rspamd_session_events_pending (task->s); if (checkpoint->pass == 0) { /* * On the first pass we check symbols that do not have dependencies * If we figure out symbol that has no dependencies satisfied, then * we just save it for another pass */ for (i = 0; i < (gint)checkpoint->version; i ++) { if (rspamd_symbols_cache_metric_limit (task, checkpoint)) { msg_info_task ("<%s> has already scored more than %.2f, so do " "not " "plan any more checks", task->message_id, checkpoint->rs->score); return TRUE; } item = g_ptr_array_index (checkpoint->order->d, i); if (!isset (checkpoint->processed_bits, item->id * 2)) { if (!rspamd_symbols_cache_check_deps (task, cache, item, checkpoint)) { msg_debug_task ("blocked execution of %d unless deps are " "resolved", item->id); g_ptr_array_add (checkpoint->waitq, item); continue; } rspamd_symbols_cache_check_symbol (task, cache, item, checkpoint, &total_microseconds); } if (total_microseconds > max_microseconds) { /* Maybe we should stop and check pending events? */ if (rspamd_session_events_pending (task->s) > start_events_pending) { /* Add some timeout event to avoid too long waiting */ #if 0 struct event *ev; struct timeval tv; rspamd_session_add_event (task->s, rspamd_symbols_cache_continuation, task, rspamd_symbols_cache_quark ()); ev = rspamd_mempool_alloc (task->task_pool, sizeof (*ev)); event_set (ev, -1, EV_TIMEOUT, rspamd_symbols_cache_tm, task); event_base_set (task->ev_base, ev); msec_to_tv (50, &tv); event_add (ev, &tv); rspamd_mempool_add_destructor (task->task_pool, (rspamd_mempool_destruct_t)event_del, ev); #endif msg_info_task ("trying to check async events after spending " "%d microseconds processing symbols", (gint)total_microseconds); return TRUE; } } } checkpoint->pass ++; } else { /* We just go through the blocked symbols and check if they are ready */ for (i = 0; i < (gint)checkpoint->waitq->len; i ++) { item = g_ptr_array_index (checkpoint->waitq, i); if (item->id >= (gint)checkpoint->version) { continue; } if (!isset (checkpoint->processed_bits, item->id * 2)) { if (!rspamd_symbols_cache_check_deps (task, cache, item, checkpoint)) { break; } rspamd_symbols_cache_check_symbol (task, cache, item, checkpoint, &total_microseconds); } if (total_microseconds > max_microseconds) { /* Maybe we should stop and check pending events? */ if (rspamd_session_events_pending (task->s) > start_events_pending) { msg_debug_task ("trying to check async events after spending " "%d microseconds processing symbols", (gint)total_microseconds); return TRUE; } } } } return TRUE; }
gboolean rspamd_symbols_cache_process_symbols (struct rspamd_task * task, struct symbols_cache *cache) { struct cache_item *item = NULL; struct cache_savepoint *checkpoint; gint i; g_assert (cache != NULL); if (task->checkpoint == NULL) { checkpoint = rspamd_mempool_alloc0 (task->task_pool, sizeof (*checkpoint)); /* Bit 0: check started, Bit 1: check finished */ checkpoint->processed_bits = rspamd_mempool_alloc0 (task->task_pool, NBYTES (cache->used_items) * 2); checkpoint->waitq = g_ptr_array_new (); rspamd_mempool_add_destructor (task->task_pool, rspamd_ptr_array_free_hard, checkpoint->waitq); task->checkpoint = checkpoint; rspamd_create_metric_result (task, DEFAULT_METRIC); if (task->settings) { const ucl_object_t *wl; wl = ucl_object_find_key (task->settings, "whitelist"); if (wl != NULL) { msg_info ("<%s> is whitelisted", task->message_id); task->flags |= RSPAMD_TASK_FLAG_SKIP; return TRUE; } } } else { checkpoint = task->checkpoint; } msg_debug ("symbols processing stage at pass: %d", checkpoint->pass); if (checkpoint->pass == 0) { /* * On the first pass we check symbols that do not have dependencies * If we figure out symbol that has no dependencies satisfied, then * we just save it for another pass */ for (i = 0; i < (gint)cache->used_items; i ++) { if (rspamd_symbols_cache_metric_limit (task, checkpoint)) { msg_info ("<%s> has already scored more than %.2f, so do not " "plan any more checks", task->message_id, checkpoint->rs->score); return TRUE; } item = g_ptr_array_index (cache->items_by_order, i); if (!isset (checkpoint->processed_bits, item->id * 2)) { if (!rspamd_symbols_cache_check_deps (task, cache, item, checkpoint)) { msg_debug ("blocked execution of %d unless deps are resolved", item->id); g_ptr_array_add (checkpoint->waitq, item); continue; } rspamd_symbols_cache_check_symbol (task, cache, item, checkpoint); } } checkpoint->pass ++; } else { /* We just go through the blocked symbols and check if they are ready */ for (i = 0; i < (gint)checkpoint->waitq->len; i ++) { item = g_ptr_array_index (checkpoint->waitq, i); if (!isset (checkpoint->processed_bits, item->id * 2)) { if (!rspamd_symbols_cache_check_deps (task, cache, item, checkpoint)) { continue; } rspamd_symbols_cache_check_symbol (task, cache, item, checkpoint); } } } return TRUE; }
static gboolean rspamd_symbols_cache_check_deps (struct rspamd_task *task, struct symbols_cache *cache, struct cache_item *item, struct cache_savepoint *checkpoint) { struct cache_dependency *dep; guint i; gboolean ret = TRUE; if (item->deps != NULL && item->deps->len > 0) { for (i = 0; i < item->deps->len; i ++) { dep = g_ptr_array_index (item->deps, i); if (dep->item == NULL) { /* Assume invalid deps as done */ continue; } if (dep->id >= (gint)checkpoint->version) { /* * This is dependency on some symbol that is currently * not in this checkpoint. So we just pretend that the * corresponding symbold does not exist */ continue; } if (!isset (checkpoint->processed_bits, dep->id * 2 + 1)) { if (!isset (checkpoint->processed_bits, dep->id * 2)) { /* Not started */ if (!rspamd_symbols_cache_check_deps (task, cache, dep->item, checkpoint)) { g_ptr_array_add (checkpoint->waitq, item); ret = FALSE; msg_debug_task ("delayed dependency %d for symbol %d", dep->id, item->id); } else if (!rspamd_symbols_cache_check_symbol (task, cache, dep->item, checkpoint, NULL)) { /* Now started, but has events pending */ ret = FALSE; msg_debug_task ("started check of %d symbol as dep for " "%d", dep->id, item->id); } else { msg_debug_task ("dependency %d for symbol %d is " "already processed", dep->id, item->id); } } else { /* Started but not finished */ ret = FALSE; } } else { msg_debug_task ("dependency %d for symbol %d is already " "checked", dep->id, item->id); } } } return ret; }