gpointer rspamd_redis_runtime (struct rspamd_task *task, struct rspamd_statfile_config *stcf, gboolean learn, gpointer c) { struct redis_stat_ctx *ctx = REDIS_CTX (c); struct redis_stat_runtime *rt; struct upstream *up; rspamd_inet_addr_t *addr; g_assert (ctx != NULL); g_assert (stcf != NULL); if (learn && ctx->write_servers == NULL) { msg_err_task ("no write servers defined for %s, cannot learn", stcf->symbol); return NULL; } if (learn) { up = rspamd_upstream_get (ctx->write_servers, RSPAMD_UPSTREAM_MASTER_SLAVE, NULL, 0); } else { up = rspamd_upstream_get (ctx->read_servers, RSPAMD_UPSTREAM_ROUND_ROBIN, NULL, 0); } if (up == NULL) { msg_err_task ("no upstreams reachable"); return NULL; } rt = rspamd_mempool_alloc0 (task->task_pool, sizeof (*rt)); rspamd_redis_expand_object (ctx->redis_object, ctx, task, &rt->redis_object_expanded); rt->selected = up; rt->task = task; rt->ctx = ctx; rt->stcf = stcf; addr = rspamd_upstream_addr (up); g_assert (addr != NULL); rt->redis = redisAsyncConnect (rspamd_inet_address_to_string (addr), rspamd_inet_address_get_port (addr)); if (rt->redis == NULL) { msg_err_task ("cannot connect redis"); return NULL; } redisLibeventAttach (rt->redis, task->ev_base); rspamd_redis_maybe_auth (ctx, rt->redis); return rt; }
static void rspamd_redis_async_stat_cb (struct rspamd_stat_async_elt *elt, gpointer d) { struct redis_stat_ctx *ctx; struct rspamd_redis_stat_elt *redis_elt = elt->ud; struct rspamd_redis_stat_cbdata *cbdata; rspamd_inet_addr_t *addr; g_assert (redis_elt != NULL); ctx = redis_elt->ctx; if (redis_elt->cbdata) { /* We have some other process pending */ rspamd_redis_async_cbdata_cleanup (redis_elt->cbdata); } /* Disable further events unless needed */ elt->enabled = FALSE; cbdata = g_slice_alloc0 (sizeof (*cbdata)); cbdata->selected = rspamd_upstream_get (ctx->read_servers, RSPAMD_UPSTREAM_ROUND_ROBIN, NULL, 0); g_assert (cbdata->selected != NULL); addr = rspamd_upstream_addr (cbdata->selected); g_assert (addr != NULL); cbdata->redis = redisAsyncConnect (rspamd_inet_address_to_string (addr), rspamd_inet_address_get_port (addr)); g_assert (cbdata->redis != NULL); redisLibeventAttach (cbdata->redis, redis_elt->ev_base); cbdata->inflight = 1; cbdata->cur = ucl_object_typed_new (UCL_OBJECT); cbdata->elt = redis_elt; cbdata->cur_keys = g_ptr_array_new (); redis_elt->cbdata = cbdata; /* XXX: deal with timeouts maybe */ /* Get keys in redis that match our symbol */ rspamd_redis_maybe_auth (ctx, cbdata->redis); redisAsyncCommand (cbdata->redis, rspamd_redis_stat_keys, cbdata, "SMEMBERS %s_keys", ctx->stcf->symbol); }
static gboolean lua_tcp_make_connection (struct lua_tcp_cbdata *cbd) { int fd; rspamd_inet_address_set_port (cbd->addr, cbd->port); fd = rspamd_inet_address_connect (cbd->addr, SOCK_STREAM, TRUE); if (fd == -1) { msg_info ("cannot connect to %s", rspamd_inet_address_to_string (cbd->addr)); return FALSE; } cbd->fd = fd; event_set (&cbd->ev, cbd->fd, EV_WRITE, lua_tcp_handler, cbd); event_base_set (cbd->ev_base, &cbd->ev); event_add (&cbd->ev, &cbd->tv); return TRUE; }
void rspamd_fuzzy_backend_count_redis (struct rspamd_fuzzy_backend *bk, rspamd_fuzzy_count_cb cb, void *ud, void *subr_ud) { struct rspamd_fuzzy_backend_redis *backend = subr_ud; struct rspamd_fuzzy_redis_session *session; struct upstream *up; struct timeval tv; rspamd_inet_addr_t *addr; g_assert (backend != NULL); session = g_slice_alloc0 (sizeof (*session)); session->backend = backend; REF_RETAIN (session->backend); session->callback.cb_count = cb; session->cbdata = ud; session->command = RSPAMD_FUZZY_REDIS_COMMAND_COUNT; session->nargs = 2; session->argv = g_malloc (sizeof (gchar *) * 2); session->argv[0] = g_strdup ("HLEN"); session->argv[1] = g_strdup (backend->redis_object); up = rspamd_upstream_get (backend->read_servers, RSPAMD_UPSTREAM_ROUND_ROBIN, NULL, 0); session->up = up; addr = rspamd_upstream_addr (up); g_assert (addr != NULL); session->ctx = rspamd_redis_pool_connect (backend->pool, backend->dbname, backend->password, rspamd_inet_address_to_string (addr), rspamd_inet_address_get_port (addr)); if (session->ctx == NULL) { rspamd_fuzzy_redis_session_dtor (session); if (cb) { cb (0, subr_ud); } } else { if (redisAsyncCommandArgv (session->ctx, rspamd_fuzzy_redis_count_callback, session, session->nargs, (const gchar **)session->argv, NULL) != REDIS_OK) { rspamd_fuzzy_redis_session_dtor (session); if (cb) { cb (0, subr_ud); } } else { /* Add timeout */ event_set (&session->timeout, -1, EV_TIMEOUT, rspamd_fuzzy_redis_timeout, session); event_base_set (rspamd_fuzzy_backend_event_base (bk), &session->timeout); double_to_tv (backend->timeout, &tv); event_add (&session->timeout, &tv); } } }
gpointer rspamd_stat_cache_redis_runtime (struct rspamd_task *task, gpointer c, gboolean learn) { struct rspamd_redis_cache_ctx *ctx = c; struct rspamd_redis_cache_runtime *rt; struct upstream *up; rspamd_inet_addr_t *addr; g_assert (ctx != NULL); if (learn && ctx->write_servers == NULL) { msg_err_task ("no write servers defined for %s, cannot learn", ctx->stcf->symbol); return NULL; } if (task->tokens == NULL || task->tokens->len == 0) { return NULL; } if (learn) { up = rspamd_upstream_get (ctx->write_servers, RSPAMD_UPSTREAM_MASTER_SLAVE, NULL, 0); } else { up = rspamd_upstream_get (ctx->read_servers, RSPAMD_UPSTREAM_ROUND_ROBIN, NULL, 0); } if (up == NULL) { msg_err_task ("no upstreams reachable"); return NULL; } rt = rspamd_mempool_alloc0 (task->task_pool, sizeof (*rt)); rt->selected = up; rt->task = task; rt->ctx = ctx; addr = rspamd_upstream_addr (up); g_assert (addr != NULL); rt->redis = redisAsyncConnect (rspamd_inet_address_to_string (addr), rspamd_inet_address_get_port (addr)); g_assert (rt->redis != NULL); redisLibeventAttach (rt->redis, task->ev_base); /* Now check stats */ event_set (&rt->timeout_event, -1, EV_TIMEOUT, rspamd_redis_cache_timeout, rt); event_base_set (task->ev_base, &rt->timeout_event); rspamd_redis_cache_maybe_auth (ctx, rt->redis); if (!learn) { rspamd_stat_cache_redis_generate_id (task); } return rt; }
/*** * @function rspamd_redis.make_request({params}) * Make request to redis server, params is a table of key=value arguments in any order * @param {task} task worker task object * @param {ip} host server address * @param {function} callback callback to be called in form `function (task, err, data)` * @param {string} cmd command to be sent to redis * @param {table} args numeric array of strings used as redis arguments * @param {number} timeout timeout in seconds for request (1.0 by default) * @return {boolean} `true` if a request has been scheduled */ static int lua_redis_make_request (lua_State *L) { struct lua_redis_userdata *ud; struct rspamd_lua_ip *addr = NULL; struct rspamd_task *task = NULL; const gchar *cmd = NULL; gint top, cbref = -1; struct timeval tv; gboolean ret = FALSE; gdouble timeout = REDIS_DEFAULT_TIMEOUT; if (lua_istable (L, 1)) { /* Table version */ lua_pushstring (L, "task"); lua_gettable (L, -2); if (lua_type (L, -1) == LUA_TUSERDATA) { task = lua_check_task (L, -1); } lua_pop (L, 1); lua_pushstring (L, "callback"); lua_gettable (L, -2); if (lua_type (L, -1) == LUA_TFUNCTION) { /* This also pops function from the stack */ cbref = luaL_ref (L, LUA_REGISTRYINDEX); } else { msg_err ("bad callback argument for lua redis"); lua_pop (L, 1); } lua_pushstring (L, "cmd"); lua_gettable (L, -2); cmd = lua_tostring (L, -1); lua_pop (L, 1); lua_pushstring (L, "host"); lua_gettable (L, -2); if (lua_type (L, -1) == LUA_TUSERDATA) { addr = lua_check_ip (L, -1); } lua_pop (L, 1); lua_pushstring (L, "timeout"); lua_gettable (L, -2); timeout = lua_tonumber (L, -1); lua_pop (L, 1); if (task != NULL && addr != NULL && cbref != -1 && cmd != NULL) { ud = rspamd_mempool_alloc (task->task_pool, sizeof (struct lua_redis_userdata)); ud->task = task; ud->L = L; ud->cbref = cbref; lua_pushstring (L, "args"); lua_redis_parse_args (L, -1, cmd, ud); ret = TRUE; } else { if (cbref != -1) { luaL_unref (L, LUA_REGISTRYINDEX, cbref); } msg_err ("incorrect function invocation"); } } else if ((task = lua_check_task (L, 1)) != NULL) { addr = lua_check_ip (L, 2); top = lua_gettop (L); /* Now get callback */ if (lua_isfunction (L, 3) && addr != NULL && addr->addr && top >= 4) { /* Create userdata */ ud = rspamd_mempool_alloc (task->task_pool, sizeof (struct lua_redis_userdata)); ud->task = task; ud->L = L; /* Pop other arguments */ lua_pushvalue (L, 3); /* Get a reference */ ud->cbref = luaL_ref (L, LUA_REGISTRYINDEX); cmd = luaL_checkstring (L, 4); if (top > 4) { lua_redis_parse_args (L, 5, cmd, ud); } else { lua_redis_parse_args (L, 0, cmd, ud); } ret = TRUE; } else { msg_err ("incorrect function invocation"); } } if (ret) { ud->terminated = 0; ud->ctx = redisAsyncConnect (rspamd_inet_address_to_string (addr->addr), rspamd_inet_address_get_port (addr->addr)); redisAsyncSetConnectCallback (ud->ctx, lua_redis_connect_cb); if (ud->ctx == NULL || ud->ctx->err) { ud->terminated = 1; redisAsyncFree (ud->ctx); lua_redis_free_args (ud); luaL_unref (ud->L, LUA_REGISTRYINDEX, ud->cbref); lua_pushboolean (L, FALSE); return 1; } redisLibeventAttach (ud->ctx, ud->task->ev_base); ret = redisAsyncCommandArgv (ud->ctx, lua_redis_callback, ud, ud->nargs, (const gchar **)ud->args, NULL); if (ret == REDIS_OK) { rspamd_session_add_event (ud->task->s, lua_redis_fin, ud, g_quark_from_static_string ("lua redis")); double_to_tv (timeout, &tv); event_set (&ud->timeout, -1, EV_TIMEOUT, lua_redis_timeout, ud); event_base_set (ud->task->ev_base, &ud->timeout); event_add (&ud->timeout, &tv); } else { msg_info ("call to redis failed: %s", ud->ctx->errstr); ud->terminated = 1; lua_redis_free_args (ud); redisAsyncFree (ud->ctx); luaL_unref (ud->L, LUA_REGISTRYINDEX, ud->cbref); } } lua_pushboolean (L, ret); return 1; }
gboolean rspamd_redis_learn_tokens (struct rspamd_task *task, GPtrArray *tokens, gint id, gpointer p) { struct redis_stat_runtime *rt = REDIS_RUNTIME (p); struct upstream *up; rspamd_inet_addr_t *addr; struct timeval tv; rspamd_fstring_t *query; const gchar *redis_cmd; rspamd_token_t *tok; gint ret; up = rspamd_upstream_get (rt->ctx->write_servers, RSPAMD_UPSTREAM_MASTER_SLAVE, NULL, 0); if (up == NULL) { msg_err_task ("no upstreams reachable"); return FALSE; } rt->selected = up; addr = rspamd_upstream_addr (up); g_assert (addr != NULL); rt->redis = redisAsyncConnect (rspamd_inet_address_to_string (addr), rspamd_inet_address_get_port (addr)); g_assert (rt->redis != NULL); redisLibeventAttach (rt->redis, task->ev_base); rspamd_redis_maybe_auth (rt->ctx, rt->redis); /* * Add the current key to the set of learned keys */ redisAsyncCommand (rt->redis, NULL, NULL, "SADD %s_keys %s", rt->stcf->symbol, rt->redis_object_expanded); if (rt->stcf->clcf->flags & RSPAMD_FLAG_CLASSIFIER_INTEGER) { redis_cmd = "HINCRBY"; } else { redis_cmd = "HINCRBYFLOAT"; } rt->id = id; query = rspamd_redis_tokens_to_query (task, tokens, redis_cmd, rt->redis_object_expanded, TRUE, id, rt->stcf->clcf->flags & RSPAMD_FLAG_CLASSIFIER_INTEGER); g_assert (query != NULL); /* * XXX: * Dirty hack: we get a token and check if it's value is -1 or 1, so * we could understand that we are learning or unlearning */ tok = g_ptr_array_index (task->tokens, 0); if (tok->values[id] > 0) { rspamd_printf_fstring (&query, "" "*4\r\n" "$7\r\n" "HINCRBY\r\n" "$%d\r\n" "%s\r\n" "$6\r\n" "learns\r\n" "$1\r\n" "1\r\n", (gint)strlen (rt->redis_object_expanded), rt->redis_object_expanded); } else { rspamd_printf_fstring (&query, "" "*4\r\n" "$7\r\n" "HINCRBY\r\n" "$%d\r\n" "%s\r\n" "$6\r\n" "learns\r\n" "$2\r\n" "-1\r\n", (gint)strlen (rt->redis_object_expanded), rt->redis_object_expanded); } rspamd_mempool_add_destructor (task->task_pool, (rspamd_mempool_destruct_t)rspamd_fstring_free, query); ret = redisAsyncFormattedCommand (rt->redis, rspamd_redis_learned, rt, query->str, query->len); if (ret == REDIS_OK) { rspamd_session_add_event (task->s, rspamd_redis_fin_learn, rt, rspamd_redis_stat_quark ()); rt->has_event = TRUE; /* Set timeout */ if (event_get_base (&rt->timeout_event)) { event_del (&rt->timeout_event); } event_set (&rt->timeout_event, -1, EV_TIMEOUT, rspamd_redis_timeout, rt); event_base_set (task->ev_base, &rt->timeout_event); double_to_tv (rt->ctx->timeout, &tv); event_add (&rt->timeout_event, &tv); return TRUE; } else { msg_err_task ("call to redis failed: %s", rt->redis->errstr); } return FALSE; }