gint
rspamd_stat_cache_redis_learn (struct rspamd_task *task,
gboolean is_spam,
gpointer runtime)
{
struct rspamd_redis_cache_runtime *rt = runtime;
struct timeval tv;
gchar *h;
gint flag;
h = rspamd_mempool_get_variable (task->task_pool, "words_hash");
g_assert (h != NULL);
double_to_tv (rt->ctx->timeout, &tv);
flag = (task->flags & RSPAMD_TASK_FLAG_LEARN_SPAM) ? 1 : -1;
if (redisAsyncCommand (rt->redis, rspamd_stat_cache_redis_set, rt,
"HSET %s %s %d",
rt->ctx->redis_object, h, flag) == REDIS_OK) {
rspamd_session_add_event (task->s, rspamd_redis_cache_fin, rt,
rspamd_stat_cache_redis_quark ());
event_add (&rt->timeout_event, &tv);
rt->has_event = TRUE;
}
/* We need to return OK every time */
return RSPAMD_LEARN_OK;
}
gint
rspamd_stat_cache_redis_check (struct rspamd_task *task,
gboolean is_spam,
gpointer runtime)
{
struct rspamd_redis_cache_runtime *rt = runtime;
struct timeval tv;
gchar *h;
h = rspamd_mempool_get_variable (task->task_pool, "words_hash");
if (h == NULL) {
return RSPAMD_LEARN_INGORE;
}
double_to_tv (rt->ctx->timeout, &tv);
if (redisAsyncCommand (rt->redis, rspamd_stat_cache_redis_get, rt,
"HGET %s %s",
rt->ctx->redis_object, h) == REDIS_OK) {
rspamd_session_add_event (task->s, rspamd_redis_cache_fin, rt,
rspamd_stat_cache_redis_quark ());
event_add (&rt->timeout_event, &tv);
rt->has_event = TRUE;
}
/* We need to return OK every time */
return RSPAMD_LEARN_OK;
}
gboolean
rspamd_redis_process_tokens (struct rspamd_task *task,
GPtrArray *tokens,
gint id, gpointer p)
{
struct redis_stat_runtime *rt = REDIS_RUNTIME (p);
rspamd_fstring_t *query;
struct timeval tv;
gint ret;
if (tokens == NULL || tokens->len == 0 || rt->redis == NULL) {
return FALSE;
}
rt->id = id;
if (redisAsyncCommand (rt->redis, rspamd_redis_connected, rt, "HGET %s %s",
rt->redis_object_expanded, "learns") == REDIS_OK) {
rspamd_session_add_event (task->s, rspamd_redis_fin, rt,
rspamd_redis_stat_quark ());
rt->has_event = TRUE;
if (event_get_base (&rt->timeout_event)) {
event_del (&rt->timeout_event);
}
event_set (&rt->timeout_event, -1, EV_TIMEOUT, rspamd_redis_timeout, rt);
event_base_set (task->ev_base, &rt->timeout_event);
double_to_tv (rt->ctx->timeout, &tv);
event_add (&rt->timeout_event, &tv);
query = rspamd_redis_tokens_to_query (task, tokens,
"HMGET", rt->redis_object_expanded, FALSE, -1,
rt->stcf->clcf->flags & RSPAMD_FLAG_CLASSIFIER_INTEGER);
g_assert (query != NULL);
rspamd_mempool_add_destructor (task->task_pool,
(rspamd_mempool_destruct_t)rspamd_fstring_free, query);
ret = redisAsyncFormattedCommand (rt->redis, rspamd_redis_processed, rt,
query->str, query->len);
if (ret == REDIS_OK) {
return TRUE;
}
else {
msg_err_task ("call to redis failed: %s", rt->redis->errstr);
}
}
return FALSE;
}
gboolean
create_smtp_upstream_connection (struct smtp_session *session)
{
struct upstream *selected;
/* Try to select upstream */
selected = rspamd_upstream_get (session->ctx->upstreams,
RSPAMD_UPSTREAM_ROUND_ROBIN);
if (selected == NULL) {
msg_err ("no upstreams suitable found");
return FALSE;
}
session->upstream = selected;
/* Now try to create socket */
session->upstream_sock = rspamd_inet_address_connect (
rspamd_upstream_addr (selected), SOCK_STREAM, TRUE);
if (session->upstream_sock == -1) {
msg_err ("cannot make a connection to %s", rspamd_upstream_name (selected));
rspamd_upstream_fail (selected);
return FALSE;
}
/* Create a dispatcher for upstream connection */
session->upstream_dispatcher = rspamd_create_dispatcher (session->ev_base,
session->upstream_sock,
BUFFER_LINE,
smtp_upstream_read_socket,
smtp_upstream_write_socket,
smtp_upstream_err_socket,
&session->ctx->smtp_timeout,
session);
session->state = SMTP_STATE_WAIT_UPSTREAM;
session->upstream_state = SMTP_STATE_GREETING;
rspamd_session_add_event (session->s,
(event_finalizer_t)smtp_upstream_finalize_connection,
session,
g_quark_from_static_string ("smtp proxy"));
return TRUE;
}
/***
* @function rspamd_tcp.request({params})
* This function creates and sends TCP request to the specified host and port,
* resolves hostname (if needed) and invokes continuation callback upon data received
* from the remote peer. This function accepts table of arguments with the following
* attributes
*
* - `task`: rspamd task objects (implies `pool`, `session`, `ev_base` and `resolver` arguments)
* - `ev_base`: event base (if no task specified)
* - `resolver`: DNS resolver (no task)
* - `session`: events session (no task)
* - `pool`: memory pool (no task)
* - `host`: IP or name of the peer (required)
* - `port`: remote port to use (required)
* - `data`: a table of strings or `rspamd_text` objects that contains data pieces
* - `callback`: continuation function (required)
* - `timeout`: floating point value that specifies timeout for IO operations in seconds
* - `partial`: boolean flag that specifies that callback should be called on any data portion received
* - `stop_pattern`: stop reading on finding a certain pattern (e.g. \r\n.\r\n for smtp)
* @return {boolean} true if request has been sent
*/
static gint
lua_tcp_request (lua_State *L)
{
const gchar *host;
gchar *stop_pattern = NULL;
guint port;
gint cbref, tp;
struct event_base *ev_base;
struct lua_tcp_cbdata *cbd;
struct rspamd_dns_resolver *resolver;
struct rspamd_async_session *session;
struct rspamd_task *task = NULL;
rspamd_mempool_t *pool;
struct iovec *iov = NULL;
guint niov = 0, total_out;
gdouble timeout = default_tcp_timeout;
gboolean partial = FALSE;
if (lua_type (L, 1) == LUA_TTABLE) {
lua_pushstring (L, "host");
lua_gettable (L, -2);
host = luaL_checkstring (L, -1);
lua_pop (L, 1);
lua_pushstring (L, "port");
lua_gettable (L, -2);
port = luaL_checknumber (L, -1);
lua_pop (L, 1);
lua_pushstring (L, "callback");
lua_gettable (L, -2);
if (host == NULL || lua_type (L, -1) != LUA_TFUNCTION) {
lua_pop (L, 1);
msg_err ("tcp request has bad params");
lua_pushboolean (L, FALSE);
return 1;
}
cbref = luaL_ref (L, LUA_REGISTRYINDEX);
lua_pushstring (L, "task");
lua_gettable (L, -2);
if (lua_type (L, -1) == LUA_TUSERDATA) {
task = lua_check_task (L, -1);
ev_base = task->ev_base;
resolver = task->resolver;
session = task->s;
pool = task->task_pool;
}
lua_pop (L, 1);
if (task == NULL) {
lua_pushstring (L, "ev_base");
lua_gettable (L, -2);
if (luaL_checkudata (L, -1, "rspamd{ev_base}")) {
ev_base = *(struct event_base **)lua_touserdata (L, -1);
}
else {
ev_base = NULL;
}
lua_pop (L, 1);
lua_pushstring (L, "pool");
lua_gettable (L, -2);
if (luaL_checkudata (L, -1, "rspamd{mempool}")) {
pool = *(rspamd_mempool_t **)lua_touserdata (L, -1);
}
else {
pool = NULL;
}
lua_pop (L, 1);
lua_pushstring (L, "resolver");
lua_gettable (L, -2);
if (luaL_checkudata (L, -1, "rspamd{resolver}")) {
resolver = *(struct rspamd_dns_resolver **)lua_touserdata (L, -1);
}
else {
resolver = lua_tcp_global_resolver (ev_base);
}
lua_pop (L, 1);
lua_pushstring (L, "session");
lua_gettable (L, -2);
if (luaL_checkudata (L, -1, "rspamd{session}")) {
session = *(struct rspamd_async_session **)lua_touserdata (L, -1);
}
else {
session = NULL;
}
lua_pop (L, 1);
}
lua_pushstring (L, "timeout");
lua_gettable (L, -2);
if (lua_type (L, -1) == LUA_TNUMBER) {
timeout = lua_tonumber (L, -1) * 1000.;
}
lua_pop (L, 1);
lua_pushstring (L, "stop_pattern");
lua_gettable (L, -2);
if (lua_type (L, -1) == LUA_TSTRING) {
stop_pattern = rspamd_mempool_strdup (pool, lua_tostring (L, -1));
}
lua_pop (L, 1);
lua_pushstring (L, "partial");
lua_gettable (L, -2);
if (lua_type (L, -1) == LUA_TBOOLEAN) {
partial = lua_toboolean (L, -1);
}
lua_pop (L, 1);
if (pool == NULL) {
lua_pop (L, 1);
msg_err ("tcp request has no memory pool associated");
lua_pushboolean (L, FALSE);
return 1;
}
lua_pushstring (L, "data");
lua_gettable (L, -2);
total_out = 0;
tp = lua_type (L, -1);
if (tp == LUA_TSTRING || tp == LUA_TUSERDATA) {
iov = rspamd_mempool_alloc (pool, sizeof (*iov));
niov = 1;
if (!lua_tcp_arg_toiovec (L, -1, pool, iov)) {
lua_pop (L, 1);
msg_err ("tcp request has bad data argument");
lua_pushboolean (L, FALSE);
return 1;
}
total_out = iov[0].iov_len;
}
else if (tp == LUA_TTABLE) {
/* Count parts */
lua_pushnil (L);
while (lua_next (L, -2) != 0) {
niov ++;
lua_pop (L, 1);
}
iov = rspamd_mempool_alloc (pool, sizeof (*iov) * niov);
lua_pushnil (L);
niov = 0;
while (lua_next (L, -2) != 0) {
if (!lua_tcp_arg_toiovec (L, -1, pool, &iov[niov])) {
lua_pop (L, 2);
msg_err ("tcp request has bad data argument at pos %d", niov);
lua_pushboolean (L, FALSE);
return 1;
}
total_out += iov[niov].iov_len;
niov ++;
lua_pop (L, 1);
}
}
lua_pop (L, 1);
}
else {
msg_err ("tcp request has bad params");
lua_pushboolean (L, FALSE);
return 1;
}
cbd = g_slice_alloc0 (sizeof (*cbd));
cbd->L = L;
cbd->cbref = cbref;
cbd->ev_base = ev_base;
msec_to_tv (timeout, &cbd->tv);
cbd->fd = -1;
cbd->pool = pool;
cbd->partial = partial;
cbd->iov = iov;
cbd->iovlen = niov;
cbd->total = total_out;
cbd->pos = 0;
cbd->port = port;
cbd->stop_pattern = stop_pattern;
if (session) {
cbd->session = session;
rspamd_session_add_event (session,
(event_finalizer_t)lua_tcp_fin,
cbd,
g_quark_from_static_string ("lua tcp"));
}
if (rspamd_parse_inet_address (&cbd->addr, host)) {
rspamd_inet_address_set_port (cbd->addr, port);
/* Host is numeric IP, no need to resolve */
if (!lua_tcp_make_connection (cbd)) {
lua_tcp_maybe_free (cbd);
lua_pushboolean (L, FALSE);
return 1;
}
}
else {
if (task == NULL) {
if (!make_dns_request (resolver, session, NULL, lua_tcp_dns_handler, cbd,
RDNS_REQUEST_A, host)) {
lua_tcp_push_error (cbd, "cannot resolve host");
lua_tcp_maybe_free (cbd);
}
}
else {
if (!make_dns_request_task (task, lua_tcp_dns_handler, cbd,
RDNS_REQUEST_A, host)) {
lua_tcp_push_error (cbd, "cannot resolve host");
lua_tcp_maybe_free (cbd);
}
}
}
lua_pushboolean (L, TRUE);
return 1;
}
/***
* @function rspamd_redis.make_request({params})
* Make request to redis server, params is a table of key=value arguments in any order
* @param {task} task worker task object
* @param {ip} host server address
* @param {function} callback callback to be called in form `function (task, err, data)`
* @param {string} cmd command to be sent to redis
* @param {table} args numeric array of strings used as redis arguments
* @param {number} timeout timeout in seconds for request (1.0 by default)
* @return {boolean} `true` if a request has been scheduled
*/
static int
lua_redis_make_request (lua_State *L)
{
struct lua_redis_userdata *ud;
struct rspamd_lua_ip *addr = NULL;
struct rspamd_task *task = NULL;
const gchar *cmd = NULL;
gint top, cbref = -1;
struct timeval tv;
gboolean ret = FALSE;
gdouble timeout = REDIS_DEFAULT_TIMEOUT;
if (lua_istable (L, 1)) {
/* Table version */
lua_pushstring (L, "task");
lua_gettable (L, -2);
if (lua_type (L, -1) == LUA_TUSERDATA) {
task = lua_check_task (L, -1);
}
lua_pop (L, 1);
lua_pushstring (L, "callback");
lua_gettable (L, -2);
if (lua_type (L, -1) == LUA_TFUNCTION) {
/* This also pops function from the stack */
cbref = luaL_ref (L, LUA_REGISTRYINDEX);
}
else {
msg_err ("bad callback argument for lua redis");
lua_pop (L, 1);
}
lua_pushstring (L, "cmd");
lua_gettable (L, -2);
cmd = lua_tostring (L, -1);
lua_pop (L, 1);
lua_pushstring (L, "host");
lua_gettable (L, -2);
if (lua_type (L, -1) == LUA_TUSERDATA) {
addr = lua_check_ip (L, -1);
}
lua_pop (L, 1);
lua_pushstring (L, "timeout");
lua_gettable (L, -2);
timeout = lua_tonumber (L, -1);
lua_pop (L, 1);
if (task != NULL && addr != NULL && cbref != -1 && cmd != NULL) {
ud =
rspamd_mempool_alloc (task->task_pool,
sizeof (struct lua_redis_userdata));
ud->task = task;
ud->L = L;
ud->cbref = cbref;
lua_pushstring (L, "args");
lua_redis_parse_args (L, -1, cmd, ud);
ret = TRUE;
}
else {
if (cbref != -1) {
luaL_unref (L, LUA_REGISTRYINDEX, cbref);
}
msg_err ("incorrect function invocation");
}
}
else if ((task = lua_check_task (L, 1)) != NULL) {
addr = lua_check_ip (L, 2);
top = lua_gettop (L);
/* Now get callback */
if (lua_isfunction (L, 3) && addr != NULL && addr->addr && top >= 4) {
/* Create userdata */
ud =
rspamd_mempool_alloc (task->task_pool,
sizeof (struct lua_redis_userdata));
ud->task = task;
ud->L = L;
/* Pop other arguments */
lua_pushvalue (L, 3);
/* Get a reference */
ud->cbref = luaL_ref (L, LUA_REGISTRYINDEX);
cmd = luaL_checkstring (L, 4);
if (top > 4) {
lua_redis_parse_args (L, 5, cmd, ud);
}
else {
lua_redis_parse_args (L, 0, cmd, ud);
}
ret = TRUE;
}
else {
msg_err ("incorrect function invocation");
}
}
if (ret) {
ud->terminated = 0;
ud->ctx = redisAsyncConnect (rspamd_inet_address_to_string (addr->addr),
rspamd_inet_address_get_port (addr->addr));
redisAsyncSetConnectCallback (ud->ctx, lua_redis_connect_cb);
if (ud->ctx == NULL || ud->ctx->err) {
ud->terminated = 1;
redisAsyncFree (ud->ctx);
lua_redis_free_args (ud);
luaL_unref (ud->L, LUA_REGISTRYINDEX, ud->cbref);
lua_pushboolean (L, FALSE);
return 1;
}
redisLibeventAttach (ud->ctx, ud->task->ev_base);
ret = redisAsyncCommandArgv (ud->ctx,
lua_redis_callback,
ud,
ud->nargs,
(const gchar **)ud->args,
NULL);
if (ret == REDIS_OK) {
rspamd_session_add_event (ud->task->s,
lua_redis_fin,
ud,
g_quark_from_static_string ("lua redis"));
double_to_tv (timeout, &tv);
event_set (&ud->timeout, -1, EV_TIMEOUT, lua_redis_timeout, ud);
event_base_set (ud->task->ev_base, &ud->timeout);
event_add (&ud->timeout, &tv);
}
else {
msg_info ("call to redis failed: %s", ud->ctx->errstr);
ud->terminated = 1;
lua_redis_free_args (ud);
redisAsyncFree (ud->ctx);
luaL_unref (ud->L, LUA_REGISTRYINDEX, ud->cbref);
}
}
lua_pushboolean (L, ret);
return 1;
}
gboolean
rspamd_redis_learn_tokens (struct rspamd_task *task, GPtrArray *tokens,
gint id, gpointer p)
{
struct redis_stat_runtime *rt = REDIS_RUNTIME (p);
struct upstream *up;
rspamd_inet_addr_t *addr;
struct timeval tv;
rspamd_fstring_t *query;
const gchar *redis_cmd;
rspamd_token_t *tok;
gint ret;
up = rspamd_upstream_get (rt->ctx->write_servers,
RSPAMD_UPSTREAM_MASTER_SLAVE,
NULL,
0);
if (up == NULL) {
msg_err_task ("no upstreams reachable");
return FALSE;
}
rt->selected = up;
addr = rspamd_upstream_addr (up);
g_assert (addr != NULL);
rt->redis = redisAsyncConnect (rspamd_inet_address_to_string (addr),
rspamd_inet_address_get_port (addr));
g_assert (rt->redis != NULL);
redisLibeventAttach (rt->redis, task->ev_base);
rspamd_redis_maybe_auth (rt->ctx, rt->redis);
/*
* Add the current key to the set of learned keys
*/
redisAsyncCommand (rt->redis, NULL, NULL, "SADD %s_keys %s",
rt->stcf->symbol, rt->redis_object_expanded);
if (rt->stcf->clcf->flags & RSPAMD_FLAG_CLASSIFIER_INTEGER) {
redis_cmd = "HINCRBY";
}
else {
redis_cmd = "HINCRBYFLOAT";
}
rt->id = id;
query = rspamd_redis_tokens_to_query (task, tokens,
redis_cmd, rt->redis_object_expanded, TRUE, id,
rt->stcf->clcf->flags & RSPAMD_FLAG_CLASSIFIER_INTEGER);
g_assert (query != NULL);
/*
* XXX:
* Dirty hack: we get a token and check if it's value is -1 or 1, so
* we could understand that we are learning or unlearning
*/
tok = g_ptr_array_index (task->tokens, 0);
if (tok->values[id] > 0) {
rspamd_printf_fstring (&query, ""
"*4\r\n"
"$7\r\n"
"HINCRBY\r\n"
"$%d\r\n"
"%s\r\n"
"$6\r\n"
"learns\r\n"
"$1\r\n"
"1\r\n",
(gint)strlen (rt->redis_object_expanded),
rt->redis_object_expanded);
}
else {
rspamd_printf_fstring (&query, ""
"*4\r\n"
"$7\r\n"
"HINCRBY\r\n"
"$%d\r\n"
"%s\r\n"
"$6\r\n"
"learns\r\n"
"$2\r\n"
"-1\r\n",
(gint)strlen (rt->redis_object_expanded),
rt->redis_object_expanded);
}
rspamd_mempool_add_destructor (task->task_pool,
(rspamd_mempool_destruct_t)rspamd_fstring_free, query);
ret = redisAsyncFormattedCommand (rt->redis, rspamd_redis_learned, rt,
query->str, query->len);
if (ret == REDIS_OK) {
rspamd_session_add_event (task->s, rspamd_redis_fin_learn, rt,
rspamd_redis_stat_quark ());
rt->has_event = TRUE;
/* Set timeout */
if (event_get_base (&rt->timeout_event)) {
event_del (&rt->timeout_event);
}
event_set (&rt->timeout_event, -1, EV_TIMEOUT, rspamd_redis_timeout, rt);
event_base_set (task->ev_base, &rt->timeout_event);
double_to_tv (rt->ctx->timeout, &tv);
event_add (&rt->timeout_event, &tv);
return TRUE;
}
else {
msg_err_task ("call to redis failed: %s", rt->redis->errstr);
}
return FALSE;
}
gboolean
rspamd_symbols_cache_process_symbols (struct rspamd_task * task,
struct symbols_cache *cache)
{
struct cache_item *item = NULL;
struct cache_savepoint *checkpoint;
gint i;
gdouble total_microseconds = 0;
const gdouble max_microseconds = 3e5;
guint start_events_pending;
g_assert (cache != NULL);
if (task->checkpoint == NULL) {
checkpoint = rspamd_symbols_cache_make_checkpoint (task, cache);
task->checkpoint = checkpoint;
}
else {
checkpoint = task->checkpoint;
}
if (task->settings) {
if (rspamd_symbols_cache_process_settings (task, cache)) {
return TRUE;
}
}
msg_debug_task ("symbols processing stage at pass: %d", checkpoint->pass);
start_events_pending = rspamd_session_events_pending (task->s);
if (checkpoint->pass == 0) {
/*
* On the first pass we check symbols that do not have dependencies
* If we figure out symbol that has no dependencies satisfied, then
* we just save it for another pass
*/
for (i = 0; i < (gint)checkpoint->version; i ++) {
if (rspamd_symbols_cache_metric_limit (task, checkpoint)) {
msg_info_task ("<%s> has already scored more than %.2f, so do "
"not "
"plan any more checks", task->message_id,
checkpoint->rs->score);
return TRUE;
}
item = g_ptr_array_index (checkpoint->order->d, i);
if (!isset (checkpoint->processed_bits, item->id * 2)) {
if (!rspamd_symbols_cache_check_deps (task, cache, item,
checkpoint)) {
msg_debug_task ("blocked execution of %d unless deps are "
"resolved",
item->id);
g_ptr_array_add (checkpoint->waitq, item);
continue;
}
rspamd_symbols_cache_check_symbol (task, cache, item,
checkpoint, &total_microseconds);
}
if (total_microseconds > max_microseconds) {
/* Maybe we should stop and check pending events? */
if (rspamd_session_events_pending (task->s) > start_events_pending) {
/* Add some timeout event to avoid too long waiting */
#if 0
struct event *ev;
struct timeval tv;
rspamd_session_add_event (task->s,
rspamd_symbols_cache_continuation, task,
rspamd_symbols_cache_quark ());
ev = rspamd_mempool_alloc (task->task_pool, sizeof (*ev));
event_set (ev, -1, EV_TIMEOUT, rspamd_symbols_cache_tm, task);
event_base_set (task->ev_base, ev);
msec_to_tv (50, &tv);
event_add (ev, &tv);
rspamd_mempool_add_destructor (task->task_pool,
(rspamd_mempool_destruct_t)event_del, ev);
#endif
msg_info_task ("trying to check async events after spending "
"%d microseconds processing symbols",
(gint)total_microseconds);
return TRUE;
}
}
}
checkpoint->pass ++;
}
else {
/* We just go through the blocked symbols and check if they are ready */
for (i = 0; i < (gint)checkpoint->waitq->len; i ++) {
item = g_ptr_array_index (checkpoint->waitq, i);
if (item->id >= (gint)checkpoint->version) {
continue;
}
if (!isset (checkpoint->processed_bits, item->id * 2)) {
if (!rspamd_symbols_cache_check_deps (task, cache, item,
checkpoint)) {
break;
}
rspamd_symbols_cache_check_symbol (task, cache, item,
checkpoint, &total_microseconds);
}
if (total_microseconds > max_microseconds) {
/* Maybe we should stop and check pending events? */
if (rspamd_session_events_pending (task->s) >
start_events_pending) {
msg_debug_task ("trying to check async events after spending "
"%d microseconds processing symbols",
(gint)total_microseconds);
return TRUE;
}
}
}
}
return TRUE;
}
