static void
sync_timer_callback (gint fd, short what, void *ud)
{
struct rspamd_sync_ctx *ctx = ud;
guint32 jittered_interval;
/* Plan new event */
evtimer_del (&ctx->tm_ev);
/* Add some jittering for synchronization */
jittered_interval = g_random_int_range (ctx->sync_interval,
ctx->sync_interval * 2);
msec_to_tv (jittered_interval, &ctx->interval);
evtimer_add (&ctx->tm_ev, &ctx->interval);
log_next_sync (ctx->st->symbol, ctx->interval.tv_sec);
if (ctx->is_busy) {
/* Sync is in progress */
msg_info ("syncronization process is in progress, do not start new one");
return;
}
if ((ctx->sock =
make_universal_socket (ctx->st->binlog->master_addr,
ctx->st->binlog->master_port,
SOCK_STREAM, TRUE, FALSE, TRUE)) == -1) {
msg_info ("cannot connect to %s", ctx->st->binlog->master_addr);
return;
}
/* Now create and activate dispatcher */
msec_to_tv (ctx->timeout, &ctx->io_tv);
ctx->dispatcher = rspamd_create_dispatcher (ctx->ev_base,
ctx->sock,
BUFFER_LINE,
sync_read,
NULL,
sync_err,
&ctx->io_tv,
ctx);
ctx->state = SYNC_STATE_GREETING;
ctx->is_busy = TRUE;
msg_info ("starting synchronization of %s", ctx->st->symbol);
}
static gboolean
add_statfile_watch (statfile_pool_t *pool,
struct rspamd_statfile_config *st,
struct rspamd_config *cfg,
struct event_base *ev_base)
{
struct rspamd_sync_ctx *ctx;
guint32 jittered_interval;
if (st->binlog->master_addr != NULL) {
ctx =
rspamd_mempool_alloc (pool->pool, sizeof (struct rspamd_sync_ctx));
ctx->st = st;
ctx->timeout = cfg->statfile_sync_timeout;
ctx->sync_interval = cfg->statfile_sync_interval;
ctx->ev_base = ev_base;
/* Add some jittering for synchronization */
jittered_interval = g_random_int_range (ctx->sync_interval,
ctx->sync_interval * 2);
msec_to_tv (jittered_interval, &ctx->interval);
/* Open statfile and attach it to pool */
if ((ctx->real_statfile =
statfile_pool_is_open (pool, st->path)) == NULL) {
if ((ctx->real_statfile =
statfile_pool_open (pool, st->path, st->size, FALSE)) == NULL) {
msg_warn ("cannot open %s", st->path);
if (statfile_pool_create (pool, st->path, st->size) == -1) {
msg_err ("cannot create statfile %s", st->path);
return FALSE;
}
ctx->real_statfile = statfile_pool_open (pool,
st->path,
st->size,
FALSE);
}
}
/* Now plan event for it's future executing */
evtimer_set (&ctx->tm_ev, sync_timer_callback, ctx);
event_base_set (ctx->ev_base, &ctx->tm_ev);
evtimer_add (&ctx->tm_ev, &ctx->interval);
log_next_sync (st->symbol, ctx->interval.tv_sec);
}
else {
msg_err ("cannot add statfile watch for statfile %s: no master defined",
st->symbol);
return FALSE;
}
return TRUE;
}
/***
* @function rspamd_tcp.request({params})
* This function creates and sends TCP request to the specified host and port,
* resolves hostname (if needed) and invokes continuation callback upon data received
* from the remote peer. This function accepts table of arguments with the following
* attributes
*
* - `task`: rspamd task objects (implies `pool`, `session`, `ev_base` and `resolver` arguments)
* - `ev_base`: event base (if no task specified)
* - `resolver`: DNS resolver (no task)
* - `session`: events session (no task)
* - `pool`: memory pool (no task)
* - `host`: IP or name of the peer (required)
* - `port`: remote port to use (required)
* - `data`: a table of strings or `rspamd_text` objects that contains data pieces
* - `callback`: continuation function (required)
* - `timeout`: floating point value that specifies timeout for IO operations in seconds
* - `partial`: boolean flag that specifies that callback should be called on any data portion received
* - `stop_pattern`: stop reading on finding a certain pattern (e.g. \r\n.\r\n for smtp)
* @return {boolean} true if request has been sent
*/
static gint
lua_tcp_request (lua_State *L)
{
const gchar *host;
gchar *stop_pattern = NULL;
guint port;
gint cbref, tp;
struct event_base *ev_base;
struct lua_tcp_cbdata *cbd;
struct rspamd_dns_resolver *resolver;
struct rspamd_async_session *session;
struct rspamd_task *task = NULL;
rspamd_mempool_t *pool;
struct iovec *iov = NULL;
guint niov = 0, total_out;
gdouble timeout = default_tcp_timeout;
gboolean partial = FALSE;
if (lua_type (L, 1) == LUA_TTABLE) {
lua_pushstring (L, "host");
lua_gettable (L, -2);
host = luaL_checkstring (L, -1);
lua_pop (L, 1);
lua_pushstring (L, "port");
lua_gettable (L, -2);
port = luaL_checknumber (L, -1);
lua_pop (L, 1);
lua_pushstring (L, "callback");
lua_gettable (L, -2);
if (host == NULL || lua_type (L, -1) != LUA_TFUNCTION) {
lua_pop (L, 1);
msg_err ("tcp request has bad params");
lua_pushboolean (L, FALSE);
return 1;
}
cbref = luaL_ref (L, LUA_REGISTRYINDEX);
lua_pushstring (L, "task");
lua_gettable (L, -2);
if (lua_type (L, -1) == LUA_TUSERDATA) {
task = lua_check_task (L, -1);
ev_base = task->ev_base;
resolver = task->resolver;
session = task->s;
pool = task->task_pool;
}
lua_pop (L, 1);
if (task == NULL) {
lua_pushstring (L, "ev_base");
lua_gettable (L, -2);
if (luaL_checkudata (L, -1, "rspamd{ev_base}")) {
ev_base = *(struct event_base **)lua_touserdata (L, -1);
}
else {
ev_base = NULL;
}
lua_pop (L, 1);
lua_pushstring (L, "pool");
lua_gettable (L, -2);
if (luaL_checkudata (L, -1, "rspamd{mempool}")) {
pool = *(rspamd_mempool_t **)lua_touserdata (L, -1);
}
else {
pool = NULL;
}
lua_pop (L, 1);
lua_pushstring (L, "resolver");
lua_gettable (L, -2);
if (luaL_checkudata (L, -1, "rspamd{resolver}")) {
resolver = *(struct rspamd_dns_resolver **)lua_touserdata (L, -1);
}
else {
resolver = lua_tcp_global_resolver (ev_base);
}
lua_pop (L, 1);
lua_pushstring (L, "session");
lua_gettable (L, -2);
if (luaL_checkudata (L, -1, "rspamd{session}")) {
session = *(struct rspamd_async_session **)lua_touserdata (L, -1);
}
else {
session = NULL;
}
lua_pop (L, 1);
}
lua_pushstring (L, "timeout");
lua_gettable (L, -2);
if (lua_type (L, -1) == LUA_TNUMBER) {
timeout = lua_tonumber (L, -1) * 1000.;
}
lua_pop (L, 1);
lua_pushstring (L, "stop_pattern");
lua_gettable (L, -2);
if (lua_type (L, -1) == LUA_TSTRING) {
stop_pattern = rspamd_mempool_strdup (pool, lua_tostring (L, -1));
}
lua_pop (L, 1);
lua_pushstring (L, "partial");
lua_gettable (L, -2);
if (lua_type (L, -1) == LUA_TBOOLEAN) {
partial = lua_toboolean (L, -1);
}
lua_pop (L, 1);
if (pool == NULL) {
lua_pop (L, 1);
msg_err ("tcp request has no memory pool associated");
lua_pushboolean (L, FALSE);
return 1;
}
lua_pushstring (L, "data");
lua_gettable (L, -2);
total_out = 0;
tp = lua_type (L, -1);
if (tp == LUA_TSTRING || tp == LUA_TUSERDATA) {
iov = rspamd_mempool_alloc (pool, sizeof (*iov));
niov = 1;
if (!lua_tcp_arg_toiovec (L, -1, pool, iov)) {
lua_pop (L, 1);
msg_err ("tcp request has bad data argument");
lua_pushboolean (L, FALSE);
return 1;
}
total_out = iov[0].iov_len;
}
else if (tp == LUA_TTABLE) {
/* Count parts */
lua_pushnil (L);
while (lua_next (L, -2) != 0) {
niov ++;
lua_pop (L, 1);
}
iov = rspamd_mempool_alloc (pool, sizeof (*iov) * niov);
lua_pushnil (L);
niov = 0;
while (lua_next (L, -2) != 0) {
if (!lua_tcp_arg_toiovec (L, -1, pool, &iov[niov])) {
lua_pop (L, 2);
msg_err ("tcp request has bad data argument at pos %d", niov);
lua_pushboolean (L, FALSE);
return 1;
}
total_out += iov[niov].iov_len;
niov ++;
lua_pop (L, 1);
}
}
lua_pop (L, 1);
}
else {
msg_err ("tcp request has bad params");
lua_pushboolean (L, FALSE);
return 1;
}
cbd = g_slice_alloc0 (sizeof (*cbd));
cbd->L = L;
cbd->cbref = cbref;
cbd->ev_base = ev_base;
msec_to_tv (timeout, &cbd->tv);
cbd->fd = -1;
cbd->pool = pool;
cbd->partial = partial;
cbd->iov = iov;
cbd->iovlen = niov;
cbd->total = total_out;
cbd->pos = 0;
cbd->port = port;
cbd->stop_pattern = stop_pattern;
if (session) {
cbd->session = session;
rspamd_session_add_event (session,
(event_finalizer_t)lua_tcp_fin,
cbd,
g_quark_from_static_string ("lua tcp"));
}
if (rspamd_parse_inet_address (&cbd->addr, host)) {
rspamd_inet_address_set_port (cbd->addr, port);
/* Host is numeric IP, no need to resolve */
if (!lua_tcp_make_connection (cbd)) {
lua_tcp_maybe_free (cbd);
lua_pushboolean (L, FALSE);
return 1;
}
}
else {
if (task == NULL) {
if (!make_dns_request (resolver, session, NULL, lua_tcp_dns_handler, cbd,
RDNS_REQUEST_A, host)) {
lua_tcp_push_error (cbd, "cannot resolve host");
lua_tcp_maybe_free (cbd);
}
}
else {
if (!make_dns_request_task (task, lua_tcp_dns_handler, cbd,
RDNS_REQUEST_A, host)) {
lua_tcp_push_error (cbd, "cannot resolve host");
lua_tcp_maybe_free (cbd);
}
}
}
lua_pushboolean (L, TRUE);
return 1;
}
gboolean
rspamd_symbols_cache_process_symbols (struct rspamd_task * task,
struct symbols_cache *cache)
{
struct cache_item *item = NULL;
struct cache_savepoint *checkpoint;
gint i;
gdouble total_microseconds = 0;
const gdouble max_microseconds = 3e5;
guint start_events_pending;
g_assert (cache != NULL);
if (task->checkpoint == NULL) {
checkpoint = rspamd_symbols_cache_make_checkpoint (task, cache);
task->checkpoint = checkpoint;
}
else {
checkpoint = task->checkpoint;
}
if (task->settings) {
if (rspamd_symbols_cache_process_settings (task, cache)) {
return TRUE;
}
}
msg_debug_task ("symbols processing stage at pass: %d", checkpoint->pass);
start_events_pending = rspamd_session_events_pending (task->s);
if (checkpoint->pass == 0) {
/*
* On the first pass we check symbols that do not have dependencies
* If we figure out symbol that has no dependencies satisfied, then
* we just save it for another pass
*/
for (i = 0; i < (gint)checkpoint->version; i ++) {
if (rspamd_symbols_cache_metric_limit (task, checkpoint)) {
msg_info_task ("<%s> has already scored more than %.2f, so do "
"not "
"plan any more checks", task->message_id,
checkpoint->rs->score);
return TRUE;
}
item = g_ptr_array_index (checkpoint->order->d, i);
if (!isset (checkpoint->processed_bits, item->id * 2)) {
if (!rspamd_symbols_cache_check_deps (task, cache, item,
checkpoint)) {
msg_debug_task ("blocked execution of %d unless deps are "
"resolved",
item->id);
g_ptr_array_add (checkpoint->waitq, item);
continue;
}
rspamd_symbols_cache_check_symbol (task, cache, item,
checkpoint, &total_microseconds);
}
if (total_microseconds > max_microseconds) {
/* Maybe we should stop and check pending events? */
if (rspamd_session_events_pending (task->s) > start_events_pending) {
/* Add some timeout event to avoid too long waiting */
#if 0
struct event *ev;
struct timeval tv;
rspamd_session_add_event (task->s,
rspamd_symbols_cache_continuation, task,
rspamd_symbols_cache_quark ());
ev = rspamd_mempool_alloc (task->task_pool, sizeof (*ev));
event_set (ev, -1, EV_TIMEOUT, rspamd_symbols_cache_tm, task);
event_base_set (task->ev_base, ev);
msec_to_tv (50, &tv);
event_add (ev, &tv);
rspamd_mempool_add_destructor (task->task_pool,
(rspamd_mempool_destruct_t)event_del, ev);
#endif
msg_info_task ("trying to check async events after spending "
"%d microseconds processing symbols",
(gint)total_microseconds);
return TRUE;
}
}
}
checkpoint->pass ++;
}
else {
/* We just go through the blocked symbols and check if they are ready */
for (i = 0; i < (gint)checkpoint->waitq->len; i ++) {
item = g_ptr_array_index (checkpoint->waitq, i);
if (item->id >= (gint)checkpoint->version) {
continue;
}
if (!isset (checkpoint->processed_bits, item->id * 2)) {
if (!rspamd_symbols_cache_check_deps (task, cache, item,
checkpoint)) {
break;
}
rspamd_symbols_cache_check_symbol (task, cache, item,
checkpoint, &total_microseconds);
}
if (total_microseconds > max_microseconds) {
/* Maybe we should stop and check pending events? */
if (rspamd_session_events_pending (task->s) >
start_events_pending) {
msg_debug_task ("trying to check async events after spending "
"%d microseconds processing symbols",
(gint)total_microseconds);
return TRUE;
}
}
}
}
return TRUE;
}
