static void
rspamd_symbols_cache_watcher_cb (gpointer sessiond, gpointer ud)
{
struct rspamd_task *task = sessiond;
struct cache_item *item = ud, *it;
struct cache_savepoint *checkpoint;
struct symbols_cache *cache;
gint i, remain = 0;
checkpoint = task->checkpoint;
cache = task->cfg->cache;
/* Specify that we are done with this item */
setbit (checkpoint->processed_bits, item->id * 2 + 1);
if (checkpoint->pass > 0) {
for (i = 0; i < (gint)checkpoint->waitq->len; i ++) {
it = g_ptr_array_index (checkpoint->waitq, i);
if (!isset (checkpoint->processed_bits, it->id * 2)) {
if (!rspamd_symbols_cache_check_deps (task, cache, it,
checkpoint)) {
remain ++;
continue;
}
rspamd_symbols_cache_check_symbol (task, cache, it, checkpoint);
}
}
}
msg_debug ("finished watcher, %ud symbols waiting", remain);
}
static gboolean
rspamd_symbols_cache_check_deps (struct rspamd_task *task,
struct symbols_cache *cache,
struct cache_item *item,
struct cache_savepoint *checkpoint)
{
struct cache_dependency *dep;
guint i;
gboolean ret = TRUE;
if (item->deps != NULL && item->deps->len > 0) {
for (i = 0; i < item->deps->len; i ++) {
dep = g_ptr_array_index (item->deps, i);
g_assert (dep->item != NULL);
if (!isset (checkpoint->processed_bits, dep->id * 2 + 1)) {
if (!isset (checkpoint->processed_bits, dep->id * 2)) {
/* Not started */
if (!rspamd_symbols_cache_check_deps (task, cache,
dep->item,
checkpoint)) {
g_ptr_array_add (checkpoint->waitq, item);
ret = FALSE;
}
else if (!rspamd_symbols_cache_check_symbol (task, cache,
dep->item,
checkpoint)) {
/* Now started, but has events pending */
ret = FALSE;
msg_debug ("started check of %d symbol as dep for %d",
dep->id, item->id);
}
else {
msg_debug ("dependency %d for symbol %d is already processed",
dep->id, item->id);
}
}
else {
/* Started but not finished */
ret = FALSE;
}
}
else {
msg_debug ("dependency %d for symbol %d is already checked",
dep->id, item->id);
}
}
}
return ret;
}
gboolean
rspamd_symbols_cache_process_symbols (struct rspamd_task * task,
struct symbols_cache *cache)
{
struct cache_item *item = NULL;
struct cache_savepoint *checkpoint;
gint i;
gdouble total_microseconds = 0;
const gdouble max_microseconds = 3e5;
guint start_events_pending;
g_assert (cache != NULL);
if (task->checkpoint == NULL) {
checkpoint = rspamd_symbols_cache_make_checkpoint (task, cache);
task->checkpoint = checkpoint;
}
else {
checkpoint = task->checkpoint;
}
if (task->settings) {
if (rspamd_symbols_cache_process_settings (task, cache)) {
return TRUE;
}
}
msg_debug_task ("symbols processing stage at pass: %d", checkpoint->pass);
start_events_pending = rspamd_session_events_pending (task->s);
if (checkpoint->pass == 0) {
/*
* On the first pass we check symbols that do not have dependencies
* If we figure out symbol that has no dependencies satisfied, then
* we just save it for another pass
*/
for (i = 0; i < (gint)checkpoint->version; i ++) {
if (rspamd_symbols_cache_metric_limit (task, checkpoint)) {
msg_info_task ("<%s> has already scored more than %.2f, so do "
"not "
"plan any more checks", task->message_id,
checkpoint->rs->score);
return TRUE;
}
item = g_ptr_array_index (checkpoint->order->d, i);
if (!isset (checkpoint->processed_bits, item->id * 2)) {
if (!rspamd_symbols_cache_check_deps (task, cache, item,
checkpoint)) {
msg_debug_task ("blocked execution of %d unless deps are "
"resolved",
item->id);
g_ptr_array_add (checkpoint->waitq, item);
continue;
}
rspamd_symbols_cache_check_symbol (task, cache, item,
checkpoint, &total_microseconds);
}
if (total_microseconds > max_microseconds) {
/* Maybe we should stop and check pending events? */
if (rspamd_session_events_pending (task->s) > start_events_pending) {
/* Add some timeout event to avoid too long waiting */
#if 0
struct event *ev;
struct timeval tv;
rspamd_session_add_event (task->s,
rspamd_symbols_cache_continuation, task,
rspamd_symbols_cache_quark ());
ev = rspamd_mempool_alloc (task->task_pool, sizeof (*ev));
event_set (ev, -1, EV_TIMEOUT, rspamd_symbols_cache_tm, task);
event_base_set (task->ev_base, ev);
msec_to_tv (50, &tv);
event_add (ev, &tv);
rspamd_mempool_add_destructor (task->task_pool,
(rspamd_mempool_destruct_t)event_del, ev);
#endif
msg_info_task ("trying to check async events after spending "
"%d microseconds processing symbols",
(gint)total_microseconds);
return TRUE;
}
}
}
checkpoint->pass ++;
}
else {
/* We just go through the blocked symbols and check if they are ready */
for (i = 0; i < (gint)checkpoint->waitq->len; i ++) {
item = g_ptr_array_index (checkpoint->waitq, i);
if (item->id >= (gint)checkpoint->version) {
continue;
}
if (!isset (checkpoint->processed_bits, item->id * 2)) {
if (!rspamd_symbols_cache_check_deps (task, cache, item,
checkpoint)) {
break;
}
rspamd_symbols_cache_check_symbol (task, cache, item,
checkpoint, &total_microseconds);
}
if (total_microseconds > max_microseconds) {
/* Maybe we should stop and check pending events? */
if (rspamd_session_events_pending (task->s) >
start_events_pending) {
msg_debug_task ("trying to check async events after spending "
"%d microseconds processing symbols",
(gint)total_microseconds);
return TRUE;
}
}
}
}
return TRUE;
}
gboolean
rspamd_symbols_cache_process_symbols (struct rspamd_task * task,
struct symbols_cache *cache)
{
struct cache_item *item = NULL;
struct cache_savepoint *checkpoint;
gint i;
g_assert (cache != NULL);
if (task->checkpoint == NULL) {
checkpoint = rspamd_mempool_alloc0 (task->task_pool, sizeof (*checkpoint));
/* Bit 0: check started, Bit 1: check finished */
checkpoint->processed_bits = rspamd_mempool_alloc0 (task->task_pool,
NBYTES (cache->used_items) * 2);
checkpoint->waitq = g_ptr_array_new ();
rspamd_mempool_add_destructor (task->task_pool,
rspamd_ptr_array_free_hard, checkpoint->waitq);
task->checkpoint = checkpoint;
rspamd_create_metric_result (task, DEFAULT_METRIC);
if (task->settings) {
const ucl_object_t *wl;
wl = ucl_object_find_key (task->settings, "whitelist");
if (wl != NULL) {
msg_info ("<%s> is whitelisted", task->message_id);
task->flags |= RSPAMD_TASK_FLAG_SKIP;
return TRUE;
}
}
}
else {
checkpoint = task->checkpoint;
}
msg_debug ("symbols processing stage at pass: %d", checkpoint->pass);
if (checkpoint->pass == 0) {
/*
* On the first pass we check symbols that do not have dependencies
* If we figure out symbol that has no dependencies satisfied, then
* we just save it for another pass
*/
for (i = 0; i < (gint)cache->used_items; i ++) {
if (rspamd_symbols_cache_metric_limit (task, checkpoint)) {
msg_info ("<%s> has already scored more than %.2f, so do not "
"plan any more checks", task->message_id,
checkpoint->rs->score);
return TRUE;
}
item = g_ptr_array_index (cache->items_by_order, i);
if (!isset (checkpoint->processed_bits, item->id * 2)) {
if (!rspamd_symbols_cache_check_deps (task, cache, item,
checkpoint)) {
msg_debug ("blocked execution of %d unless deps are resolved",
item->id);
g_ptr_array_add (checkpoint->waitq, item);
continue;
}
rspamd_symbols_cache_check_symbol (task, cache, item, checkpoint);
}
}
checkpoint->pass ++;
}
else {
/* We just go through the blocked symbols and check if they are ready */
for (i = 0; i < (gint)checkpoint->waitq->len; i ++) {
item = g_ptr_array_index (checkpoint->waitq, i);
if (!isset (checkpoint->processed_bits, item->id * 2)) {
if (!rspamd_symbols_cache_check_deps (task, cache, item,
checkpoint)) {
continue;
}
rspamd_symbols_cache_check_symbol (task, cache, item, checkpoint);
}
}
}
return TRUE;
}
static gboolean
rspamd_symbols_cache_check_deps (struct rspamd_task *task,
struct symbols_cache *cache,
struct cache_item *item,
struct cache_savepoint *checkpoint)
{
struct cache_dependency *dep;
guint i;
gboolean ret = TRUE;
if (item->deps != NULL && item->deps->len > 0) {
for (i = 0; i < item->deps->len; i ++) {
dep = g_ptr_array_index (item->deps, i);
if (dep->item == NULL) {
/* Assume invalid deps as done */
continue;
}
if (dep->id >= (gint)checkpoint->version) {
/*
* This is dependency on some symbol that is currently
* not in this checkpoint. So we just pretend that the
* corresponding symbold does not exist
*/
continue;
}
if (!isset (checkpoint->processed_bits, dep->id * 2 + 1)) {
if (!isset (checkpoint->processed_bits, dep->id * 2)) {
/* Not started */
if (!rspamd_symbols_cache_check_deps (task, cache,
dep->item,
checkpoint)) {
g_ptr_array_add (checkpoint->waitq, item);
ret = FALSE;
msg_debug_task ("delayed dependency %d for symbol %d",
dep->id, item->id);
}
else if (!rspamd_symbols_cache_check_symbol (task, cache,
dep->item,
checkpoint,
NULL)) {
/* Now started, but has events pending */
ret = FALSE;
msg_debug_task ("started check of %d symbol as dep for "
"%d",
dep->id, item->id);
}
else {
msg_debug_task ("dependency %d for symbol %d is "
"already processed",
dep->id, item->id);
}
}
else {
/* Started but not finished */
ret = FALSE;
}
}
else {
msg_debug_task ("dependency %d for symbol %d is already "
"checked",
dep->id, item->id);
}
}
}
return ret;
}