gint rspamd_stat_cache_redis_learn (struct rspamd_task *task, gboolean is_spam, gpointer runtime) { struct rspamd_redis_cache_runtime *rt = runtime; struct timeval tv; gchar *h; gint flag; h = rspamd_mempool_get_variable (task->task_pool, "words_hash"); g_assert (h != NULL); double_to_tv (rt->ctx->timeout, &tv); flag = (task->flags & RSPAMD_TASK_FLAG_LEARN_SPAM) ? 1 : -1; if (redisAsyncCommand (rt->redis, rspamd_stat_cache_redis_set, rt, "HSET %s %s %d", rt->ctx->redis_object, h, flag) == REDIS_OK) { rspamd_session_add_event (task->s, rspamd_redis_cache_fin, rt, rspamd_stat_cache_redis_quark ()); event_add (&rt->timeout_event, &tv); rt->has_event = TRUE; } /* We need to return OK every time */ return RSPAMD_LEARN_OK; }
gint rspamd_stat_cache_redis_check (struct rspamd_task *task, gboolean is_spam, gpointer runtime) { struct rspamd_redis_cache_runtime *rt = runtime; struct timeval tv; gchar *h; h = rspamd_mempool_get_variable (task->task_pool, "words_hash"); if (h == NULL) { return RSPAMD_LEARN_INGORE; } double_to_tv (rt->ctx->timeout, &tv); if (redisAsyncCommand (rt->redis, rspamd_stat_cache_redis_get, rt, "HGET %s %s", rt->ctx->redis_object, h) == REDIS_OK) { rspamd_session_add_event (task->s, rspamd_redis_cache_fin, rt, rspamd_stat_cache_redis_quark ()); event_add (&rt->timeout_event, &tv); rt->has_event = TRUE; } /* We need to return OK every time */ return RSPAMD_LEARN_OK; }
gboolean rspamd_redis_process_tokens (struct rspamd_task *task, GPtrArray *tokens, gint id, gpointer p) { struct redis_stat_runtime *rt = REDIS_RUNTIME (p); rspamd_fstring_t *query; struct timeval tv; gint ret; if (tokens == NULL || tokens->len == 0 || rt->redis == NULL) { return FALSE; } rt->id = id; if (redisAsyncCommand (rt->redis, rspamd_redis_connected, rt, "HGET %s %s", rt->redis_object_expanded, "learns") == REDIS_OK) { rspamd_session_add_event (task->s, rspamd_redis_fin, rt, rspamd_redis_stat_quark ()); rt->has_event = TRUE; if (event_get_base (&rt->timeout_event)) { event_del (&rt->timeout_event); } event_set (&rt->timeout_event, -1, EV_TIMEOUT, rspamd_redis_timeout, rt); event_base_set (task->ev_base, &rt->timeout_event); double_to_tv (rt->ctx->timeout, &tv); event_add (&rt->timeout_event, &tv); query = rspamd_redis_tokens_to_query (task, tokens, "HMGET", rt->redis_object_expanded, FALSE, -1, rt->stcf->clcf->flags & RSPAMD_FLAG_CLASSIFIER_INTEGER); g_assert (query != NULL); rspamd_mempool_add_destructor (task->task_pool, (rspamd_mempool_destruct_t)rspamd_fstring_free, query); ret = redisAsyncFormattedCommand (rt->redis, rspamd_redis_processed, rt, query->str, query->len); if (ret == REDIS_OK) { return TRUE; } else { msg_err_task ("call to redis failed: %s", rt->redis->errstr); } } return FALSE; }
gboolean create_smtp_upstream_connection (struct smtp_session *session) { struct upstream *selected; /* Try to select upstream */ selected = rspamd_upstream_get (session->ctx->upstreams, RSPAMD_UPSTREAM_ROUND_ROBIN); if (selected == NULL) { msg_err ("no upstreams suitable found"); return FALSE; } session->upstream = selected; /* Now try to create socket */ session->upstream_sock = rspamd_inet_address_connect ( rspamd_upstream_addr (selected), SOCK_STREAM, TRUE); if (session->upstream_sock == -1) { msg_err ("cannot make a connection to %s", rspamd_upstream_name (selected)); rspamd_upstream_fail (selected); return FALSE; } /* Create a dispatcher for upstream connection */ session->upstream_dispatcher = rspamd_create_dispatcher (session->ev_base, session->upstream_sock, BUFFER_LINE, smtp_upstream_read_socket, smtp_upstream_write_socket, smtp_upstream_err_socket, &session->ctx->smtp_timeout, session); session->state = SMTP_STATE_WAIT_UPSTREAM; session->upstream_state = SMTP_STATE_GREETING; rspamd_session_add_event (session->s, (event_finalizer_t)smtp_upstream_finalize_connection, session, g_quark_from_static_string ("smtp proxy")); return TRUE; }
/*** * @function rspamd_tcp.request({params}) * This function creates and sends TCP request to the specified host and port, * resolves hostname (if needed) and invokes continuation callback upon data received * from the remote peer. This function accepts table of arguments with the following * attributes * * - `task`: rspamd task objects (implies `pool`, `session`, `ev_base` and `resolver` arguments) * - `ev_base`: event base (if no task specified) * - `resolver`: DNS resolver (no task) * - `session`: events session (no task) * - `pool`: memory pool (no task) * - `host`: IP or name of the peer (required) * - `port`: remote port to use (required) * - `data`: a table of strings or `rspamd_text` objects that contains data pieces * - `callback`: continuation function (required) * - `timeout`: floating point value that specifies timeout for IO operations in seconds * - `partial`: boolean flag that specifies that callback should be called on any data portion received * - `stop_pattern`: stop reading on finding a certain pattern (e.g. \r\n.\r\n for smtp) * @return {boolean} true if request has been sent */ static gint lua_tcp_request (lua_State *L) { const gchar *host; gchar *stop_pattern = NULL; guint port; gint cbref, tp; struct event_base *ev_base; struct lua_tcp_cbdata *cbd; struct rspamd_dns_resolver *resolver; struct rspamd_async_session *session; struct rspamd_task *task = NULL; rspamd_mempool_t *pool; struct iovec *iov = NULL; guint niov = 0, total_out; gdouble timeout = default_tcp_timeout; gboolean partial = FALSE; if (lua_type (L, 1) == LUA_TTABLE) { lua_pushstring (L, "host"); lua_gettable (L, -2); host = luaL_checkstring (L, -1); lua_pop (L, 1); lua_pushstring (L, "port"); lua_gettable (L, -2); port = luaL_checknumber (L, -1); lua_pop (L, 1); lua_pushstring (L, "callback"); lua_gettable (L, -2); if (host == NULL || lua_type (L, -1) != LUA_TFUNCTION) { lua_pop (L, 1); msg_err ("tcp request has bad params"); lua_pushboolean (L, FALSE); return 1; } cbref = luaL_ref (L, LUA_REGISTRYINDEX); lua_pushstring (L, "task"); lua_gettable (L, -2); if (lua_type (L, -1) == LUA_TUSERDATA) { task = lua_check_task (L, -1); ev_base = task->ev_base; resolver = task->resolver; session = task->s; pool = task->task_pool; } lua_pop (L, 1); if (task == NULL) { lua_pushstring (L, "ev_base"); lua_gettable (L, -2); if (luaL_checkudata (L, -1, "rspamd{ev_base}")) { ev_base = *(struct event_base **)lua_touserdata (L, -1); } else { ev_base = NULL; } lua_pop (L, 1); lua_pushstring (L, "pool"); lua_gettable (L, -2); if (luaL_checkudata (L, -1, "rspamd{mempool}")) { pool = *(rspamd_mempool_t **)lua_touserdata (L, -1); } else { pool = NULL; } lua_pop (L, 1); lua_pushstring (L, "resolver"); lua_gettable (L, -2); if (luaL_checkudata (L, -1, "rspamd{resolver}")) { resolver = *(struct rspamd_dns_resolver **)lua_touserdata (L, -1); } else { resolver = lua_tcp_global_resolver (ev_base); } lua_pop (L, 1); lua_pushstring (L, "session"); lua_gettable (L, -2); if (luaL_checkudata (L, -1, "rspamd{session}")) { session = *(struct rspamd_async_session **)lua_touserdata (L, -1); } else { session = NULL; } lua_pop (L, 1); } lua_pushstring (L, "timeout"); lua_gettable (L, -2); if (lua_type (L, -1) == LUA_TNUMBER) { timeout = lua_tonumber (L, -1) * 1000.; } lua_pop (L, 1); lua_pushstring (L, "stop_pattern"); lua_gettable (L, -2); if (lua_type (L, -1) == LUA_TSTRING) { stop_pattern = rspamd_mempool_strdup (pool, lua_tostring (L, -1)); } lua_pop (L, 1); lua_pushstring (L, "partial"); lua_gettable (L, -2); if (lua_type (L, -1) == LUA_TBOOLEAN) { partial = lua_toboolean (L, -1); } lua_pop (L, 1); if (pool == NULL) { lua_pop (L, 1); msg_err ("tcp request has no memory pool associated"); lua_pushboolean (L, FALSE); return 1; } lua_pushstring (L, "data"); lua_gettable (L, -2); total_out = 0; tp = lua_type (L, -1); if (tp == LUA_TSTRING || tp == LUA_TUSERDATA) { iov = rspamd_mempool_alloc (pool, sizeof (*iov)); niov = 1; if (!lua_tcp_arg_toiovec (L, -1, pool, iov)) { lua_pop (L, 1); msg_err ("tcp request has bad data argument"); lua_pushboolean (L, FALSE); return 1; } total_out = iov[0].iov_len; } else if (tp == LUA_TTABLE) { /* Count parts */ lua_pushnil (L); while (lua_next (L, -2) != 0) { niov ++; lua_pop (L, 1); } iov = rspamd_mempool_alloc (pool, sizeof (*iov) * niov); lua_pushnil (L); niov = 0; while (lua_next (L, -2) != 0) { if (!lua_tcp_arg_toiovec (L, -1, pool, &iov[niov])) { lua_pop (L, 2); msg_err ("tcp request has bad data argument at pos %d", niov); lua_pushboolean (L, FALSE); return 1; } total_out += iov[niov].iov_len; niov ++; lua_pop (L, 1); } } lua_pop (L, 1); } else { msg_err ("tcp request has bad params"); lua_pushboolean (L, FALSE); return 1; } cbd = g_slice_alloc0 (sizeof (*cbd)); cbd->L = L; cbd->cbref = cbref; cbd->ev_base = ev_base; msec_to_tv (timeout, &cbd->tv); cbd->fd = -1; cbd->pool = pool; cbd->partial = partial; cbd->iov = iov; cbd->iovlen = niov; cbd->total = total_out; cbd->pos = 0; cbd->port = port; cbd->stop_pattern = stop_pattern; if (session) { cbd->session = session; rspamd_session_add_event (session, (event_finalizer_t)lua_tcp_fin, cbd, g_quark_from_static_string ("lua tcp")); } if (rspamd_parse_inet_address (&cbd->addr, host)) { rspamd_inet_address_set_port (cbd->addr, port); /* Host is numeric IP, no need to resolve */ if (!lua_tcp_make_connection (cbd)) { lua_tcp_maybe_free (cbd); lua_pushboolean (L, FALSE); return 1; } } else { if (task == NULL) { if (!make_dns_request (resolver, session, NULL, lua_tcp_dns_handler, cbd, RDNS_REQUEST_A, host)) { lua_tcp_push_error (cbd, "cannot resolve host"); lua_tcp_maybe_free (cbd); } } else { if (!make_dns_request_task (task, lua_tcp_dns_handler, cbd, RDNS_REQUEST_A, host)) { lua_tcp_push_error (cbd, "cannot resolve host"); lua_tcp_maybe_free (cbd); } } } lua_pushboolean (L, TRUE); return 1; }
/*** * @function rspamd_redis.make_request({params}) * Make request to redis server, params is a table of key=value arguments in any order * @param {task} task worker task object * @param {ip} host server address * @param {function} callback callback to be called in form `function (task, err, data)` * @param {string} cmd command to be sent to redis * @param {table} args numeric array of strings used as redis arguments * @param {number} timeout timeout in seconds for request (1.0 by default) * @return {boolean} `true` if a request has been scheduled */ static int lua_redis_make_request (lua_State *L) { struct lua_redis_userdata *ud; struct rspamd_lua_ip *addr = NULL; struct rspamd_task *task = NULL; const gchar *cmd = NULL; gint top, cbref = -1; struct timeval tv; gboolean ret = FALSE; gdouble timeout = REDIS_DEFAULT_TIMEOUT; if (lua_istable (L, 1)) { /* Table version */ lua_pushstring (L, "task"); lua_gettable (L, -2); if (lua_type (L, -1) == LUA_TUSERDATA) { task = lua_check_task (L, -1); } lua_pop (L, 1); lua_pushstring (L, "callback"); lua_gettable (L, -2); if (lua_type (L, -1) == LUA_TFUNCTION) { /* This also pops function from the stack */ cbref = luaL_ref (L, LUA_REGISTRYINDEX); } else { msg_err ("bad callback argument for lua redis"); lua_pop (L, 1); } lua_pushstring (L, "cmd"); lua_gettable (L, -2); cmd = lua_tostring (L, -1); lua_pop (L, 1); lua_pushstring (L, "host"); lua_gettable (L, -2); if (lua_type (L, -1) == LUA_TUSERDATA) { addr = lua_check_ip (L, -1); } lua_pop (L, 1); lua_pushstring (L, "timeout"); lua_gettable (L, -2); timeout = lua_tonumber (L, -1); lua_pop (L, 1); if (task != NULL && addr != NULL && cbref != -1 && cmd != NULL) { ud = rspamd_mempool_alloc (task->task_pool, sizeof (struct lua_redis_userdata)); ud->task = task; ud->L = L; ud->cbref = cbref; lua_pushstring (L, "args"); lua_redis_parse_args (L, -1, cmd, ud); ret = TRUE; } else { if (cbref != -1) { luaL_unref (L, LUA_REGISTRYINDEX, cbref); } msg_err ("incorrect function invocation"); } } else if ((task = lua_check_task (L, 1)) != NULL) { addr = lua_check_ip (L, 2); top = lua_gettop (L); /* Now get callback */ if (lua_isfunction (L, 3) && addr != NULL && addr->addr && top >= 4) { /* Create userdata */ ud = rspamd_mempool_alloc (task->task_pool, sizeof (struct lua_redis_userdata)); ud->task = task; ud->L = L; /* Pop other arguments */ lua_pushvalue (L, 3); /* Get a reference */ ud->cbref = luaL_ref (L, LUA_REGISTRYINDEX); cmd = luaL_checkstring (L, 4); if (top > 4) { lua_redis_parse_args (L, 5, cmd, ud); } else { lua_redis_parse_args (L, 0, cmd, ud); } ret = TRUE; } else { msg_err ("incorrect function invocation"); } } if (ret) { ud->terminated = 0; ud->ctx = redisAsyncConnect (rspamd_inet_address_to_string (addr->addr), rspamd_inet_address_get_port (addr->addr)); redisAsyncSetConnectCallback (ud->ctx, lua_redis_connect_cb); if (ud->ctx == NULL || ud->ctx->err) { ud->terminated = 1; redisAsyncFree (ud->ctx); lua_redis_free_args (ud); luaL_unref (ud->L, LUA_REGISTRYINDEX, ud->cbref); lua_pushboolean (L, FALSE); return 1; } redisLibeventAttach (ud->ctx, ud->task->ev_base); ret = redisAsyncCommandArgv (ud->ctx, lua_redis_callback, ud, ud->nargs, (const gchar **)ud->args, NULL); if (ret == REDIS_OK) { rspamd_session_add_event (ud->task->s, lua_redis_fin, ud, g_quark_from_static_string ("lua redis")); double_to_tv (timeout, &tv); event_set (&ud->timeout, -1, EV_TIMEOUT, lua_redis_timeout, ud); event_base_set (ud->task->ev_base, &ud->timeout); event_add (&ud->timeout, &tv); } else { msg_info ("call to redis failed: %s", ud->ctx->errstr); ud->terminated = 1; lua_redis_free_args (ud); redisAsyncFree (ud->ctx); luaL_unref (ud->L, LUA_REGISTRYINDEX, ud->cbref); } } lua_pushboolean (L, ret); return 1; }
gboolean rspamd_redis_learn_tokens (struct rspamd_task *task, GPtrArray *tokens, gint id, gpointer p) { struct redis_stat_runtime *rt = REDIS_RUNTIME (p); struct upstream *up; rspamd_inet_addr_t *addr; struct timeval tv; rspamd_fstring_t *query; const gchar *redis_cmd; rspamd_token_t *tok; gint ret; up = rspamd_upstream_get (rt->ctx->write_servers, RSPAMD_UPSTREAM_MASTER_SLAVE, NULL, 0); if (up == NULL) { msg_err_task ("no upstreams reachable"); return FALSE; } rt->selected = up; addr = rspamd_upstream_addr (up); g_assert (addr != NULL); rt->redis = redisAsyncConnect (rspamd_inet_address_to_string (addr), rspamd_inet_address_get_port (addr)); g_assert (rt->redis != NULL); redisLibeventAttach (rt->redis, task->ev_base); rspamd_redis_maybe_auth (rt->ctx, rt->redis); /* * Add the current key to the set of learned keys */ redisAsyncCommand (rt->redis, NULL, NULL, "SADD %s_keys %s", rt->stcf->symbol, rt->redis_object_expanded); if (rt->stcf->clcf->flags & RSPAMD_FLAG_CLASSIFIER_INTEGER) { redis_cmd = "HINCRBY"; } else { redis_cmd = "HINCRBYFLOAT"; } rt->id = id; query = rspamd_redis_tokens_to_query (task, tokens, redis_cmd, rt->redis_object_expanded, TRUE, id, rt->stcf->clcf->flags & RSPAMD_FLAG_CLASSIFIER_INTEGER); g_assert (query != NULL); /* * XXX: * Dirty hack: we get a token and check if it's value is -1 or 1, so * we could understand that we are learning or unlearning */ tok = g_ptr_array_index (task->tokens, 0); if (tok->values[id] > 0) { rspamd_printf_fstring (&query, "" "*4\r\n" "$7\r\n" "HINCRBY\r\n" "$%d\r\n" "%s\r\n" "$6\r\n" "learns\r\n" "$1\r\n" "1\r\n", (gint)strlen (rt->redis_object_expanded), rt->redis_object_expanded); } else { rspamd_printf_fstring (&query, "" "*4\r\n" "$7\r\n" "HINCRBY\r\n" "$%d\r\n" "%s\r\n" "$6\r\n" "learns\r\n" "$2\r\n" "-1\r\n", (gint)strlen (rt->redis_object_expanded), rt->redis_object_expanded); } rspamd_mempool_add_destructor (task->task_pool, (rspamd_mempool_destruct_t)rspamd_fstring_free, query); ret = redisAsyncFormattedCommand (rt->redis, rspamd_redis_learned, rt, query->str, query->len); if (ret == REDIS_OK) { rspamd_session_add_event (task->s, rspamd_redis_fin_learn, rt, rspamd_redis_stat_quark ()); rt->has_event = TRUE; /* Set timeout */ if (event_get_base (&rt->timeout_event)) { event_del (&rt->timeout_event); } event_set (&rt->timeout_event, -1, EV_TIMEOUT, rspamd_redis_timeout, rt); event_base_set (task->ev_base, &rt->timeout_event); double_to_tv (rt->ctx->timeout, &tv); event_add (&rt->timeout_event, &tv); return TRUE; } else { msg_err_task ("call to redis failed: %s", rt->redis->errstr); } return FALSE; }
gboolean rspamd_symbols_cache_process_symbols (struct rspamd_task * task, struct symbols_cache *cache) { struct cache_item *item = NULL; struct cache_savepoint *checkpoint; gint i; gdouble total_microseconds = 0; const gdouble max_microseconds = 3e5; guint start_events_pending; g_assert (cache != NULL); if (task->checkpoint == NULL) { checkpoint = rspamd_symbols_cache_make_checkpoint (task, cache); task->checkpoint = checkpoint; } else { checkpoint = task->checkpoint; } if (task->settings) { if (rspamd_symbols_cache_process_settings (task, cache)) { return TRUE; } } msg_debug_task ("symbols processing stage at pass: %d", checkpoint->pass); start_events_pending = rspamd_session_events_pending (task->s); if (checkpoint->pass == 0) { /* * On the first pass we check symbols that do not have dependencies * If we figure out symbol that has no dependencies satisfied, then * we just save it for another pass */ for (i = 0; i < (gint)checkpoint->version; i ++) { if (rspamd_symbols_cache_metric_limit (task, checkpoint)) { msg_info_task ("<%s> has already scored more than %.2f, so do " "not " "plan any more checks", task->message_id, checkpoint->rs->score); return TRUE; } item = g_ptr_array_index (checkpoint->order->d, i); if (!isset (checkpoint->processed_bits, item->id * 2)) { if (!rspamd_symbols_cache_check_deps (task, cache, item, checkpoint)) { msg_debug_task ("blocked execution of %d unless deps are " "resolved", item->id); g_ptr_array_add (checkpoint->waitq, item); continue; } rspamd_symbols_cache_check_symbol (task, cache, item, checkpoint, &total_microseconds); } if (total_microseconds > max_microseconds) { /* Maybe we should stop and check pending events? */ if (rspamd_session_events_pending (task->s) > start_events_pending) { /* Add some timeout event to avoid too long waiting */ #if 0 struct event *ev; struct timeval tv; rspamd_session_add_event (task->s, rspamd_symbols_cache_continuation, task, rspamd_symbols_cache_quark ()); ev = rspamd_mempool_alloc (task->task_pool, sizeof (*ev)); event_set (ev, -1, EV_TIMEOUT, rspamd_symbols_cache_tm, task); event_base_set (task->ev_base, ev); msec_to_tv (50, &tv); event_add (ev, &tv); rspamd_mempool_add_destructor (task->task_pool, (rspamd_mempool_destruct_t)event_del, ev); #endif msg_info_task ("trying to check async events after spending " "%d microseconds processing symbols", (gint)total_microseconds); return TRUE; } } } checkpoint->pass ++; } else { /* We just go through the blocked symbols and check if they are ready */ for (i = 0; i < (gint)checkpoint->waitq->len; i ++) { item = g_ptr_array_index (checkpoint->waitq, i); if (item->id >= (gint)checkpoint->version) { continue; } if (!isset (checkpoint->processed_bits, item->id * 2)) { if (!rspamd_symbols_cache_check_deps (task, cache, item, checkpoint)) { break; } rspamd_symbols_cache_check_symbol (task, cache, item, checkpoint, &total_microseconds); } if (total_microseconds > max_microseconds) { /* Maybe we should stop and check pending events? */ if (rspamd_session_events_pending (task->s) > start_events_pending) { msg_debug_task ("trying to check async events after spending " "%d microseconds processing symbols", (gint)total_microseconds); return TRUE; } } } } return TRUE; }