gint
rspamd_stat_cache_redis_learn (struct rspamd_task *task,
gboolean is_spam,
gpointer runtime)
{
struct rspamd_redis_cache_runtime *rt = runtime;
struct timeval tv;
gchar *h;
gint flag;
h = rspamd_mempool_get_variable (task->task_pool, "words_hash");
g_assert (h != NULL);
double_to_tv (rt->ctx->timeout, &tv);
flag = (task->flags & RSPAMD_TASK_FLAG_LEARN_SPAM) ? 1 : -1;
if (redisAsyncCommand (rt->redis, rspamd_stat_cache_redis_set, rt,
"HSET %s %s %d",
rt->ctx->redis_object, h, flag) == REDIS_OK) {
rspamd_session_add_event (task->s, rspamd_redis_cache_fin, rt,
rspamd_stat_cache_redis_quark ());
event_add (&rt->timeout_event, &tv);
rt->has_event = TRUE;
}
/* We need to return OK every time */
return RSPAMD_LEARN_OK;
}
static void
jitter_timeout_event (struct rspamd_map *map,
gboolean locked, gboolean initial, gboolean errored)
{
const gdouble error_mult = 20.0, lock_mult = 0.5;
gdouble jittered_sec;
gdouble timeout;
if (initial) {
timeout = 0.0;
}
else if (errored) {
timeout = map->cfg->map_timeout * error_mult;
}
else if (locked) {
timeout = map->cfg->map_timeout * lock_mult;
}
else {
timeout = map->cfg->map_timeout;
}
/* Plan event again with jitter */
evtimer_del (&map->ev);
jittered_sec = rspamd_time_jitter (timeout, 0);
double_to_tv (jittered_sec, &map->tv);
evtimer_add (&map->ev, &map->tv);
}
gint
rspamd_stat_cache_redis_check (struct rspamd_task *task,
gboolean is_spam,
gpointer runtime)
{
struct rspamd_redis_cache_runtime *rt = runtime;
struct timeval tv;
gchar *h;
h = rspamd_mempool_get_variable (task->task_pool, "words_hash");
if (h == NULL) {
return RSPAMD_LEARN_INGORE;
}
double_to_tv (rt->ctx->timeout, &tv);
if (redisAsyncCommand (rt->redis, rspamd_stat_cache_redis_get, rt,
"HGET %s %s",
rt->ctx->redis_object, h) == REDIS_OK) {
rspamd_session_add_event (task->s, rspamd_redis_cache_fin, rt,
rspamd_stat_cache_redis_quark ());
event_add (&rt->timeout_event, &tv);
rt->has_event = TRUE;
}
/* We need to return OK every time */
return RSPAMD_LEARN_OK;
}
static void
rspamd_symbols_cache_resort_cb (gint fd, short what, gpointer ud)
{
struct timeval tv;
gdouble tm;
struct symbols_cache *cache = ud;
struct cache_item *item, *parent;
guint i;
/* Plan new event */
tm = rspamd_time_jitter (cache->reload_time, 0);
msg_debug_cache ("resort symbols cache, next reload in %.2f seconds", tm);
g_assert (cache != NULL);
evtimer_set (&cache->resort_ev, rspamd_symbols_cache_resort_cb, cache);
double_to_tv (tm, &tv);
event_add (&cache->resort_ev, &tv);
rspamd_mempool_lock_mutex (cache->mtx);
/* Gather stats from shared execution times */
for (i = 0; i < cache->items_by_id->len; i ++) {
item = g_ptr_array_index (cache->items_by_id, i);
if (item->type & (SYMBOL_TYPE_CALLBACK|SYMBOL_TYPE_NORMAL)) {
if (item->cd->number > 0) {
item->avg_counter += item->cd->number + 1;
item->avg_time = item->avg_time +
(item->cd->value - item->avg_time) /
(gdouble)item->avg_counter;
item->cd->value = item->avg_time;
item->cd->number = item->avg_counter;
}
}
}
/* Sync virtual symbols */
for (i = 0; i < cache->items_by_id->len; i ++) {
item = g_ptr_array_index (cache->items_by_id, i);
if (item->parent != -1) {
parent = g_ptr_array_index (cache->items_by_id, item->parent);
if (parent) {
item->avg_time = parent->avg_time;
item->avg_counter = parent->avg_counter;
}
}
}
rspamd_mempool_unlock_mutex (cache->mtx);
rspamd_symbols_cache_resort (cache);
}
static void
jitter_timeout_event (struct rspamd_map *map, gboolean locked, gboolean initial)
{
gdouble jittered_sec;
gdouble timeout = initial ? 1.0 : map->cfg->map_timeout;
/* Plan event again with jitter */
evtimer_del (&map->ev);
jittered_sec = rspamd_time_jitter (locked ? timeout * 4 : timeout, 0);
double_to_tv (jittered_sec, &map->tv);
evtimer_add (&map->ev, &map->tv);
}
gboolean
rspamd_redis_process_tokens (struct rspamd_task *task,
GPtrArray *tokens,
gint id, gpointer p)
{
struct redis_stat_runtime *rt = REDIS_RUNTIME (p);
rspamd_fstring_t *query;
struct timeval tv;
gint ret;
if (tokens == NULL || tokens->len == 0 || rt->redis == NULL) {
return FALSE;
}
rt->id = id;
if (redisAsyncCommand (rt->redis, rspamd_redis_connected, rt, "HGET %s %s",
rt->redis_object_expanded, "learns") == REDIS_OK) {
rspamd_session_add_event (task->s, rspamd_redis_fin, rt,
rspamd_redis_stat_quark ());
rt->has_event = TRUE;
if (event_get_base (&rt->timeout_event)) {
event_del (&rt->timeout_event);
}
event_set (&rt->timeout_event, -1, EV_TIMEOUT, rspamd_redis_timeout, rt);
event_base_set (task->ev_base, &rt->timeout_event);
double_to_tv (rt->ctx->timeout, &tv);
event_add (&rt->timeout_event, &tv);
query = rspamd_redis_tokens_to_query (task, tokens,
"HMGET", rt->redis_object_expanded, FALSE, -1,
rt->stcf->clcf->flags & RSPAMD_FLAG_CLASSIFIER_INTEGER);
g_assert (query != NULL);
rspamd_mempool_add_destructor (task->task_pool,
(rspamd_mempool_destruct_t)rspamd_fstring_free, query);
ret = redisAsyncFormattedCommand (rt->redis, rspamd_redis_processed, rt,
query->str, query->len);
if (ret == REDIS_OK) {
return TRUE;
}
else {
msg_err_task ("call to redis failed: %s", rt->redis->errstr);
}
}
return FALSE;
}
void
rspamd_symbols_cache_start_refresh (struct symbols_cache * cache,
struct event_base *ev_base)
{
struct timeval tv;
gdouble tm;
tm = rspamd_time_jitter (cache->reload_time, 0);
g_assert (cache != NULL);
evtimer_set (&cache->resort_ev, rspamd_symbols_cache_resort_cb, cache);
event_base_set (ev_base, &cache->resort_ev);
double_to_tv (tm, &tv);
event_add (&cache->resort_ev, &tv);
}
struct rspamd_client_connection *
rspamd_client_init (struct event_base *ev_base, const gchar *name,
guint16 port, gdouble timeout, const gchar *key)
{
struct rspamd_client_connection *conn;
gint fd;
fd = rspamd_socket (name, port, SOCK_STREAM, TRUE, FALSE, TRUE);
if (fd == -1) {
return NULL;
}
conn = g_slice_alloc0 (sizeof (struct rspamd_client_connection));
conn->ev_base = ev_base;
conn->fd = fd;
conn->req_sent = FALSE;
conn->keys_cache = rspamd_keypair_cache_new (32);
conn->http_conn = rspamd_http_connection_new (rspamd_client_body_handler,
rspamd_client_error_handler,
rspamd_client_finish_handler,
0,
RSPAMD_HTTP_CLIENT,
conn->keys_cache,
NULL);
conn->server_name = g_string_new (name);
if (port != 0) {
rspamd_printf_gstring (conn->server_name, ":%d", (int)port);
}
double_to_tv (timeout, &conn->timeout);
if (key) {
conn->key = rspamd_pubkey_from_base32 (key, 0, RSPAMD_KEYPAIR_KEX,
RSPAMD_CRYPTOBOX_MODE_25519);
if (conn->key) {
conn->keypair = rspamd_keypair_new (RSPAMD_KEYPAIR_KEX,
RSPAMD_CRYPTOBOX_MODE_25519);
rspamd_http_connection_set_key (conn->http_conn, conn->keypair);
}
else {
rspamd_client_destroy (conn);
return NULL;
}
}
return conn;
}
static void
rspamd_async_elt_on_timer (gint fd, short what, gpointer d)
{
struct rspamd_stat_async_elt *elt = d;
gdouble jittered_time;
event_del (&elt->timer_ev);
if (elt->enabled) {
elt->handler (elt, elt->ud);
}
jittered_time = rspamd_time_jitter (elt->timeout, 0);
double_to_tv (jittered_time, &elt->tv);
event_add (&elt->timer_ev, &elt->tv);
}
static void
rspamd_map_schedule_periodic (struct rspamd_map *map,
gboolean locked, gboolean initial, gboolean errored)
{
const gdouble error_mult = 20.0, lock_mult = 0.1;
gdouble jittered_sec;
gdouble timeout;
struct map_periodic_cbdata *cbd;
timeout = map->poll_timeout;
if (initial) {
timeout = 0.0;
}
if (errored) {
timeout = map->poll_timeout * error_mult;
}
else if (locked) {
timeout = lock_mult;
}
cbd = g_slice_alloc0 (sizeof (*cbd));
cbd->cbdata.state = 0;
cbd->cbdata.prev_data = *map->user_data;
cbd->cbdata.cur_data = NULL;
cbd->cbdata.map = map;
cbd->map = map;
REF_INIT_RETAIN (cbd, rspamd_map_periodic_dtor);
if (initial) {
evtimer_set (&map->ev, rspamd_map_periodic_callback, cbd);
event_base_set (map->ev_base, &map->ev);
}
else {
evtimer_del (&map->ev);
evtimer_set (&map->ev, rspamd_map_periodic_callback, cbd);
event_base_set (map->ev_base, &map->ev);
}
jittered_sec = rspamd_time_jitter (timeout, 0);
msg_debug_map ("schedule new periodic event %p in %.2f seconds", cbd, jittered_sec);
double_to_tv (jittered_sec, &map->tv);
evtimer_add (&map->ev, &map->tv);
}
static void
rspamadm_control (gint argc, gchar **argv)
{
GOptionContext *context;
GError *error = NULL;
struct event_base *ev_base;
const gchar *cmd, *path = NULL;
struct rspamd_http_connection *conn;
struct rspamd_http_message *msg;
rspamd_inet_addr_t *addr;
struct timeval tv;
static struct rspamadm_control_cbdata cbdata;
lua_State *L;
gint sock;
context = g_option_context_new (
"control - manage rspamd main control interface");
g_option_context_set_summary (context,
"Summary:\n Rspamd administration utility version "
RVERSION
"\n Release id: "
RID);
g_option_context_add_main_entries (context, entries, NULL);
g_option_context_set_ignore_unknown_options (context, TRUE);
if (!g_option_context_parse (context, &argc, &argv, &error)) {
rspamd_fprintf (stderr, "option parsing failed: %s\n", error->message);
g_error_free (error);
exit (1);
}
if (argc <= 1) {
rspamd_fprintf (stderr, "command required\n");
exit (1);
}
cmd = argv[1];
if (g_ascii_strcasecmp (cmd, "stat") == 0) {
path = "/stat";
}
else if (g_ascii_strcasecmp (cmd, "reload") == 0) {
path = "/reload";
}
else if (g_ascii_strcasecmp (cmd, "reresolve") == 0) {
path = "/reresolve";
}
else if (g_ascii_strcasecmp (cmd, "recompile") == 0) {
path = "/recompile";
}
else if (g_ascii_strcasecmp (cmd, "fuzzystat") == 0 ||
g_ascii_strcasecmp (cmd, "fuzzy_stat") == 0) {
path = "/fuzzystat";
}
else if (g_ascii_strcasecmp (cmd, "fuzzysync") == 0 ||
g_ascii_strcasecmp (cmd, "fuzzy_sync") == 0) {
path = "/fuzzysync";
}
else {
rspamd_fprintf (stderr, "unknown command: %s\n", cmd);
exit (1);
}
if (!rspamd_parse_inet_address (&addr, control_path, 0)) {
rspamd_fprintf (stderr, "bad control path: %s\n", control_path);
exit (1);
}
ev_base = event_init ();
sock = rspamd_inet_address_connect (addr, SOCK_STREAM, TRUE);
if (sock == -1) {
rspamd_fprintf (stderr, "cannot connect to: %s\n", control_path);
rspamd_inet_address_destroy (addr);
exit (1);
}
L = rspamd_lua_init ();
conn = rspamd_http_connection_new (NULL,
rspamd_control_error_handler,
rspamd_control_finish_handler,
RSPAMD_HTTP_CLIENT_SIMPLE,
RSPAMD_HTTP_CLIENT,
NULL,
NULL);
msg = rspamd_http_new_message (HTTP_REQUEST);
msg->url = rspamd_fstring_new_init (path, strlen (path));
double_to_tv (timeout, &tv);
cbdata.L = L;
cbdata.argc = argc;
cbdata.argv = argv;
cbdata.path = path;
rspamd_http_connection_write_message (conn, msg, NULL, NULL, &cbdata, sock,
&tv, ev_base);
event_base_loop (ev_base, 0);
rspamd_http_connection_unref (conn);
rspamd_inet_address_destroy (addr);
lua_close (L);
close (sock);
}
gboolean
rspamd_redis_learn_tokens (struct rspamd_task *task, GPtrArray *tokens,
gint id, gpointer p)
{
struct redis_stat_runtime *rt = REDIS_RUNTIME (p);
struct upstream *up;
rspamd_inet_addr_t *addr;
struct timeval tv;
rspamd_fstring_t *query;
const gchar *redis_cmd;
rspamd_token_t *tok;
gint ret;
up = rspamd_upstream_get (rt->ctx->write_servers,
RSPAMD_UPSTREAM_MASTER_SLAVE,
NULL,
0);
if (up == NULL) {
msg_err_task ("no upstreams reachable");
return FALSE;
}
rt->selected = up;
addr = rspamd_upstream_addr (up);
g_assert (addr != NULL);
rt->redis = redisAsyncConnect (rspamd_inet_address_to_string (addr),
rspamd_inet_address_get_port (addr));
g_assert (rt->redis != NULL);
redisLibeventAttach (rt->redis, task->ev_base);
rspamd_redis_maybe_auth (rt->ctx, rt->redis);
/*
* Add the current key to the set of learned keys
*/
redisAsyncCommand (rt->redis, NULL, NULL, "SADD %s_keys %s",
rt->stcf->symbol, rt->redis_object_expanded);
if (rt->stcf->clcf->flags & RSPAMD_FLAG_CLASSIFIER_INTEGER) {
redis_cmd = "HINCRBY";
}
else {
redis_cmd = "HINCRBYFLOAT";
}
rt->id = id;
query = rspamd_redis_tokens_to_query (task, tokens,
redis_cmd, rt->redis_object_expanded, TRUE, id,
rt->stcf->clcf->flags & RSPAMD_FLAG_CLASSIFIER_INTEGER);
g_assert (query != NULL);
/*
* XXX:
* Dirty hack: we get a token and check if it's value is -1 or 1, so
* we could understand that we are learning or unlearning
*/
tok = g_ptr_array_index (task->tokens, 0);
if (tok->values[id] > 0) {
rspamd_printf_fstring (&query, ""
"*4\r\n"
"$7\r\n"
"HINCRBY\r\n"
"$%d\r\n"
"%s\r\n"
"$6\r\n"
"learns\r\n"
"$1\r\n"
"1\r\n",
(gint)strlen (rt->redis_object_expanded),
rt->redis_object_expanded);
}
else {
rspamd_printf_fstring (&query, ""
"*4\r\n"
"$7\r\n"
"HINCRBY\r\n"
"$%d\r\n"
"%s\r\n"
"$6\r\n"
"learns\r\n"
"$2\r\n"
"-1\r\n",
(gint)strlen (rt->redis_object_expanded),
rt->redis_object_expanded);
}
rspamd_mempool_add_destructor (task->task_pool,
(rspamd_mempool_destruct_t)rspamd_fstring_free, query);
ret = redisAsyncFormattedCommand (rt->redis, rspamd_redis_learned, rt,
query->str, query->len);
if (ret == REDIS_OK) {
rspamd_session_add_event (task->s, rspamd_redis_fin_learn, rt,
rspamd_redis_stat_quark ());
rt->has_event = TRUE;
/* Set timeout */
if (event_get_base (&rt->timeout_event)) {
event_del (&rt->timeout_event);
}
event_set (&rt->timeout_event, -1, EV_TIMEOUT, rspamd_redis_timeout, rt);
event_base_set (task->ev_base, &rt->timeout_event);
double_to_tv (rt->ctx->timeout, &tv);
event_add (&rt->timeout_event, &tv);
return TRUE;
}
else {
msg_err_task ("call to redis failed: %s", rt->redis->errstr);
}
return FALSE;
}
/**
* Async HTTP callback
*/
static void
http_callback (gint fd, short what, void *ud)
{
struct rspamd_map *map = ud;
struct http_map_data *data;
struct http_callback_data *cbd;
rspamd_mempool_t *pool;
gchar tmpbuf[PATH_MAX];
data = map->map_data;
pool = map->pool;
if (!g_atomic_int_compare_and_exchange (map->locked, 0, 1)) {
msg_debug_pool (
"don't try to reread map as it is locked by other process, will reread it later");
jitter_timeout_event (map, TRUE, FALSE, FALSE);
return;
}
/* Plan event */
cbd = g_slice_alloc0 (sizeof (struct http_callback_data));
rspamd_snprintf (tmpbuf, sizeof (tmpbuf),
"%s" G_DIR_SEPARATOR_S "rspamd_map%d-XXXXXX",
map->cfg->temp_dir, map->id);
cbd->out_fd = mkstemp (tmpbuf);
if (cbd->out_fd == -1) {
g_slice_free1 (sizeof (*cbd), cbd);
msg_err_pool ("cannot create tempfile: %s", strerror (errno));
jitter_timeout_event (map, FALSE, FALSE, TRUE);
g_atomic_int_set (map->locked, 0);
return;
}
cbd->tmpfile = g_strdup (tmpbuf);
cbd->ev_base = map->ev_base;
cbd->map = map;
cbd->data = data;
cbd->fd = -1;
cbd->cbdata.state = 0;
cbd->cbdata.prev_data = *cbd->map->user_data;
cbd->cbdata.cur_data = NULL;
cbd->cbdata.map = cbd->map;
cbd->stage = map_resolve_host2;
double_to_tv (map->cfg->map_timeout, &cbd->tv);
REF_INIT_RETAIN (cbd, free_http_cbdata);
msg_debug_pool ("reading map data from %s", data->host);
/* Send both A and AAAA requests */
if (map->r->r) {
if (rdns_make_request_full (map->r->r, rspamd_map_dns_callback, cbd,
map->cfg->dns_timeout, map->cfg->dns_retransmits, 1,
data->host, RDNS_REQUEST_A)) {
REF_RETAIN (cbd);
}
if (rdns_make_request_full (map->r->r, rspamd_map_dns_callback, cbd,
map->cfg->dns_timeout, map->cfg->dns_retransmits, 1,
data->host, RDNS_REQUEST_AAAA)) {
REF_RETAIN (cbd);
}
jitter_timeout_event (map, FALSE, FALSE, FALSE);
map->dtor = free_http_cbdata_dtor;
map->dtor_data = cbd;
}
else {
msg_warn_pool ("cannot load map: DNS resolver is not initialized");
jitter_timeout_event (map, FALSE, FALSE, TRUE);
}
/* We don't need own ref as it is now refcounted by DNS requests */
REF_RELEASE (cbd);
}
/***
* @function rspamd_redis.make_request({params})
* Make request to redis server, params is a table of key=value arguments in any order
* @param {task} task worker task object
* @param {ip} host server address
* @param {function} callback callback to be called in form `function (task, err, data)`
* @param {string} cmd command to be sent to redis
* @param {table} args numeric array of strings used as redis arguments
* @param {number} timeout timeout in seconds for request (1.0 by default)
* @return {boolean} `true` if a request has been scheduled
*/
static int
lua_redis_make_request (lua_State *L)
{
struct lua_redis_userdata *ud;
struct rspamd_lua_ip *addr = NULL;
struct rspamd_task *task = NULL;
const gchar *cmd = NULL;
gint top, cbref = -1;
struct timeval tv;
gboolean ret = FALSE;
gdouble timeout = REDIS_DEFAULT_TIMEOUT;
if (lua_istable (L, 1)) {
/* Table version */
lua_pushstring (L, "task");
lua_gettable (L, -2);
if (lua_type (L, -1) == LUA_TUSERDATA) {
task = lua_check_task (L, -1);
}
lua_pop (L, 1);
lua_pushstring (L, "callback");
lua_gettable (L, -2);
if (lua_type (L, -1) == LUA_TFUNCTION) {
/* This also pops function from the stack */
cbref = luaL_ref (L, LUA_REGISTRYINDEX);
}
else {
msg_err ("bad callback argument for lua redis");
lua_pop (L, 1);
}
lua_pushstring (L, "cmd");
lua_gettable (L, -2);
cmd = lua_tostring (L, -1);
lua_pop (L, 1);
lua_pushstring (L, "host");
lua_gettable (L, -2);
if (lua_type (L, -1) == LUA_TUSERDATA) {
addr = lua_check_ip (L, -1);
}
lua_pop (L, 1);
lua_pushstring (L, "timeout");
lua_gettable (L, -2);
timeout = lua_tonumber (L, -1);
lua_pop (L, 1);
if (task != NULL && addr != NULL && cbref != -1 && cmd != NULL) {
ud =
rspamd_mempool_alloc (task->task_pool,
sizeof (struct lua_redis_userdata));
ud->task = task;
ud->L = L;
ud->cbref = cbref;
lua_pushstring (L, "args");
lua_redis_parse_args (L, -1, cmd, ud);
ret = TRUE;
}
else {
if (cbref != -1) {
luaL_unref (L, LUA_REGISTRYINDEX, cbref);
}
msg_err ("incorrect function invocation");
}
}
else if ((task = lua_check_task (L, 1)) != NULL) {
addr = lua_check_ip (L, 2);
top = lua_gettop (L);
/* Now get callback */
if (lua_isfunction (L, 3) && addr != NULL && addr->addr && top >= 4) {
/* Create userdata */
ud =
rspamd_mempool_alloc (task->task_pool,
sizeof (struct lua_redis_userdata));
ud->task = task;
ud->L = L;
/* Pop other arguments */
lua_pushvalue (L, 3);
/* Get a reference */
ud->cbref = luaL_ref (L, LUA_REGISTRYINDEX);
cmd = luaL_checkstring (L, 4);
if (top > 4) {
lua_redis_parse_args (L, 5, cmd, ud);
}
else {
lua_redis_parse_args (L, 0, cmd, ud);
}
ret = TRUE;
}
else {
msg_err ("incorrect function invocation");
}
}
if (ret) {
ud->terminated = 0;
ud->ctx = redisAsyncConnect (rspamd_inet_address_to_string (addr->addr),
rspamd_inet_address_get_port (addr->addr));
redisAsyncSetConnectCallback (ud->ctx, lua_redis_connect_cb);
if (ud->ctx == NULL || ud->ctx->err) {
ud->terminated = 1;
redisAsyncFree (ud->ctx);
lua_redis_free_args (ud);
luaL_unref (ud->L, LUA_REGISTRYINDEX, ud->cbref);
lua_pushboolean (L, FALSE);
return 1;
}
redisLibeventAttach (ud->ctx, ud->task->ev_base);
ret = redisAsyncCommandArgv (ud->ctx,
lua_redis_callback,
ud,
ud->nargs,
(const gchar **)ud->args,
NULL);
if (ret == REDIS_OK) {
rspamd_session_add_event (ud->task->s,
lua_redis_fin,
ud,
g_quark_from_static_string ("lua redis"));
double_to_tv (timeout, &tv);
event_set (&ud->timeout, -1, EV_TIMEOUT, lua_redis_timeout, ud);
event_base_set (ud->task->ev_base, &ud->timeout);
event_add (&ud->timeout, &tv);
}
else {
msg_info ("call to redis failed: %s", ud->ctx->errstr);
ud->terminated = 1;
lua_redis_free_args (ud);
redisAsyncFree (ud->ctx);
luaL_unref (ud->L, LUA_REGISTRYINDEX, ud->cbref);
}
}
lua_pushboolean (L, ret);
return 1;
}
static int
http_map_finish (struct rspamd_http_connection *conn,
struct rspamd_http_message *msg)
{
struct http_callback_data *cbd = conn->ud;
struct rspamd_map *map;
struct rspamd_map_backend *bk;
guchar *aux_data, *in = NULL;
gsize inlen = 0, dlen = 0;
map = cbd->map;
bk = cbd->bk;
if (msg->code == 200) {
if (cbd->check) {
cbd->periodic->need_modify = TRUE;
/* Reset the whole chain */
cbd->periodic->cur_backend = 0;
rspamd_map_periodic_callback (-1, EV_TIMEOUT, cbd->periodic);
MAP_RELEASE (cbd, "http_callback_data");
return 0;
}
if (cbd->stage == map_load_file) {
if (msg->last_modified) {
cbd->data->last_checked = msg->last_modified;
}
else {
cbd->data->last_checked = msg->date;
}
/* Maybe we need to check signature ? */
if (bk->is_signed) {
if (bk->trusted_pubkey) {
/* No need to load key */
cbd->stage = map_load_signature;
cbd->pk = rspamd_pubkey_ref (bk->trusted_pubkey);
}
else {
cbd->stage = map_load_pubkey;
}
cbd->shmem_data = rspamd_http_message_shmem_ref (msg);
cbd->data_len = msg->body_buf.len;
rspamd_http_connection_reset (cbd->conn);
write_http_request (cbd);
MAP_RELEASE (cbd, "http_callback_data");
return 0;
}
else {
/* Unsinged version - just open file */
cbd->shmem_data = rspamd_http_message_shmem_ref (msg);
cbd->data_len = msg->body_buf.len;
goto read_data;
}
}
else if (cbd->stage == map_load_pubkey) {
/* We now can load pubkey */
cbd->shmem_pubkey = rspamd_http_message_shmem_ref (msg);
cbd->pubkey_len = msg->body_buf.len;
aux_data = rspamd_shmem_xmap (cbd->shmem_pubkey->shm_name,
PROT_READ, &inlen);
if (aux_data == NULL) {
msg_err_map ("cannot map pubkey file %s: %s",
cbd->shmem_pubkey->shm_name, strerror (errno));
goto err;
}
if (inlen < cbd->pubkey_len) {
msg_err_map ("cannot map pubkey file %s: %s",
cbd->shmem_pubkey->shm_name, strerror (errno));
munmap (aux_data, inlen);
goto err;
}
cbd->pk = rspamd_pubkey_from_base32 (aux_data, cbd->pubkey_len,
RSPAMD_KEYPAIR_SIGN, RSPAMD_CRYPTOBOX_MODE_25519);
munmap (aux_data, inlen);
if (cbd->pk == NULL) {
msg_err_map ("cannot load pubkey file %s: bad pubkey",
cbd->shmem_pubkey->shm_name);
goto err;
}
cbd->stage = map_load_signature;
rspamd_http_connection_reset (cbd->conn);
write_http_request (cbd);
MAP_RELEASE (cbd, "http_callback_data");
return 0;
}
else if (cbd->stage == map_load_signature) {
/* We can now check signature */
cbd->shmem_sig = rspamd_http_message_shmem_ref (msg);
cbd->sig_len = msg->body_buf.len;
aux_data = rspamd_shmem_xmap (cbd->shmem_sig->shm_name,
PROT_READ, &inlen);
if (aux_data == NULL) {
msg_err_map ("cannot map signature file %s: %s",
cbd->shmem_sig->shm_name, strerror (errno));
goto err;
}
if (inlen < cbd->sig_len) {
msg_err_map ("cannot map pubkey file %s: %s",
cbd->shmem_pubkey->shm_name, strerror (errno));
munmap (aux_data, inlen);
goto err;
}
in = rspamd_shmem_xmap (cbd->shmem_data->shm_name, PROT_READ, &dlen);
if (in == NULL) {
msg_err_map ("cannot read tempfile %s: %s",
cbd->shmem_data->shm_name,
strerror (errno));
munmap (aux_data, inlen);
goto err;
}
if (!rspamd_map_check_sig_pk_mem (aux_data, cbd->sig_len, map, in,
cbd->data_len, cbd->pk)) {
munmap (aux_data, inlen);
munmap (in, dlen);
goto err;
}
munmap (in, dlen);
}
read_data:
g_assert (cbd->shmem_data != NULL);
in = rspamd_shmem_xmap (cbd->shmem_data->shm_name, PROT_READ, &dlen);
if (in == NULL) {
msg_err_map ("cannot read tempfile %s: %s",
cbd->shmem_data->shm_name,
strerror (errno));
goto err;
}
map->read_callback (in, cbd->data_len, &cbd->periodic->cbdata, TRUE);
msg_info_map ("read map data from %s", cbd->data->host);
/*
* We know that a map is in the locked state
*/
if (g_atomic_int_compare_and_exchange (&map->cache->available, 0, 1)) {
/* Store cached data */
struct rspamd_http_map_cached_cbdata *cache_cbd;
struct timeval tv;
rspamd_strlcpy (map->cache->shmem_name, cbd->shmem_data->shm_name,
sizeof (map->cache->shmem_name));
map->cache->len = cbd->data_len;
map->cache->last_checked = cbd->data->last_checked;
cache_cbd = g_slice_alloc0 (sizeof (*cache_cbd));
cache_cbd->shm = cbd->shmem_data;
cache_cbd->map = map;
MAP_RETAIN (cache_cbd->shm, "shmem_data");
event_set (&cache_cbd->timeout, -1, EV_TIMEOUT, rspamd_map_cache_cb,
cache_cbd);
event_base_set (cbd->ev_base, &cache_cbd->timeout);
double_to_tv (map->poll_timeout, &tv);
event_add (&cache_cbd->timeout, &tv);
}
cbd->periodic->cur_backend ++;
munmap (in, dlen);
rspamd_map_periodic_callback (-1, EV_TIMEOUT, cbd->periodic);
}
else if (msg->code == 304 && (cbd->check && cbd->stage == map_load_file)) {
msg_debug_map ("data is not modified for server %s",
cbd->data->host);
if (msg->last_modified) {
cbd->data->last_checked = msg->last_modified;
}
else {
cbd->data->last_checked = msg->date;
}
cbd->periodic->cur_backend ++;
rspamd_map_periodic_callback (-1, EV_TIMEOUT, cbd->periodic);
}
else {
msg_info_map ("cannot load map %s from %s: HTTP error %d",
bk->uri, cbd->data->host, msg->code);
}
MAP_RELEASE (cbd, "http_callback_data");
return 0;
err:
cbd->periodic->errored = 1;
rspamd_map_periodic_callback (-1, EV_TIMEOUT, cbd->periodic);
MAP_RELEASE (cbd, "http_callback_data");
return 0;
}
/**
* Async HTTP callback
*/
static void
rspamd_map_common_http_callback (struct rspamd_map *map, struct rspamd_map_backend *bk,
struct map_periodic_cbdata *periodic, gboolean check)
{
struct http_map_data *data;
struct http_callback_data *cbd;
gchar tmpbuf[PATH_MAX];
data = bk->data.hd;
cbd = g_slice_alloc0 (sizeof (struct http_callback_data));
rspamd_snprintf (tmpbuf, sizeof (tmpbuf),
"%s" G_DIR_SEPARATOR_S "rspamd_map%d-XXXXXX",
map->cfg->temp_dir, map->id);
cbd->out_fd = mkstemp (tmpbuf);
if (cbd->out_fd == -1) {
msg_err_map ("cannot create tempfile: %s", strerror (errno));
g_atomic_int_set (map->locked, 0);
g_slice_free1 (sizeof (*cbd), cbd);
periodic->errored = TRUE;
rspamd_map_periodic_callback (-1, EV_TIMEOUT, periodic);
return;
}
cbd->tmpfile = g_strdup (tmpbuf);
cbd->ev_base = map->ev_base;
cbd->map = map;
cbd->data = data;
cbd->fd = -1;
cbd->check = check;
cbd->periodic = periodic;
MAP_RETAIN (periodic);
cbd->bk = bk;
MAP_RETAIN (bk);
cbd->stage = map_resolve_host2;
double_to_tv (map->cfg->map_timeout, &cbd->tv);
REF_INIT_RETAIN (cbd, free_http_cbdata);
msg_debug_map ("%s map data from %s", check ? "checking" : "reading",
data->host);
/* Send both A and AAAA requests */
if (map->r->r) {
if (rdns_make_request_full (map->r->r, rspamd_map_dns_callback, cbd,
map->cfg->dns_timeout, map->cfg->dns_retransmits, 1,
data->host, RDNS_REQUEST_A)) {
MAP_RETAIN (cbd);
}
if (rdns_make_request_full (map->r->r, rspamd_map_dns_callback, cbd,
map->cfg->dns_timeout, map->cfg->dns_retransmits, 1,
data->host, RDNS_REQUEST_AAAA)) {
MAP_RETAIN (cbd);
}
map->dtor = free_http_cbdata_dtor;
map->dtor_data = cbd;
}
else {
msg_warn_map ("cannot load map: DNS resolver is not initialized");
cbd->periodic->errored = TRUE;
}
/* We don't need own ref as it is now ref counted by DNS handlers */
MAP_RELEASE (cbd);
}
void
rspamd_fuzzy_backend_count_redis (struct rspamd_fuzzy_backend *bk,
rspamd_fuzzy_count_cb cb, void *ud,
void *subr_ud)
{
struct rspamd_fuzzy_backend_redis *backend = subr_ud;
struct rspamd_fuzzy_redis_session *session;
struct upstream *up;
struct timeval tv;
rspamd_inet_addr_t *addr;
g_assert (backend != NULL);
session = g_slice_alloc0 (sizeof (*session));
session->backend = backend;
REF_RETAIN (session->backend);
session->callback.cb_count = cb;
session->cbdata = ud;
session->command = RSPAMD_FUZZY_REDIS_COMMAND_COUNT;
session->nargs = 2;
session->argv = g_malloc (sizeof (gchar *) * 2);
session->argv[0] = g_strdup ("HLEN");
session->argv[1] = g_strdup (backend->redis_object);
up = rspamd_upstream_get (backend->read_servers,
RSPAMD_UPSTREAM_ROUND_ROBIN,
NULL,
0);
session->up = up;
addr = rspamd_upstream_addr (up);
g_assert (addr != NULL);
session->ctx = rspamd_redis_pool_connect (backend->pool,
backend->dbname, backend->password,
rspamd_inet_address_to_string (addr),
rspamd_inet_address_get_port (addr));
if (session->ctx == NULL) {
rspamd_fuzzy_redis_session_dtor (session);
if (cb) {
cb (0, subr_ud);
}
}
else {
if (redisAsyncCommandArgv (session->ctx, rspamd_fuzzy_redis_count_callback,
session, session->nargs,
(const gchar **)session->argv, NULL) != REDIS_OK) {
rspamd_fuzzy_redis_session_dtor (session);
if (cb) {
cb (0, subr_ud);
}
}
else {
/* Add timeout */
event_set (&session->timeout, -1, EV_TIMEOUT, rspamd_fuzzy_redis_timeout,
session);
event_base_set (rspamd_fuzzy_backend_event_base (bk),
&session->timeout);
double_to_tv (backend->timeout, &tv);
event_add (&session->timeout, &tv);
}
}
}
/**
* Async HTTP callback
*/
static void
rspamd_map_common_http_callback (struct rspamd_map *map, struct rspamd_map_backend *bk,
struct map_periodic_cbdata *periodic, gboolean check)
{
struct http_map_data *data;
struct http_callback_data *cbd;
data = bk->data.hd;
if (g_atomic_int_get (&map->cache->available) == 1) {
/* Read cached data */
if (check) {
if (data->last_checked < map->cache->last_checked) {
periodic->need_modify = TRUE;
/* Reset the whole chain */
periodic->cur_backend = 0;
rspamd_map_periodic_callback (-1, EV_TIMEOUT, periodic);
}
return;
}
else if (rspamd_map_read_cached (map, periodic, data->host)) {
/* Switch to the next backend */
periodic->cur_backend ++;
rspamd_map_periodic_callback (-1, EV_TIMEOUT, periodic);
data->last_checked = map->cache->last_checked;
return;
}
}
cbd = g_slice_alloc0 (sizeof (struct http_callback_data));
cbd->ev_base = map->ev_base;
cbd->map = map;
cbd->data = data;
cbd->fd = -1;
cbd->check = check;
cbd->periodic = periodic;
MAP_RETAIN (periodic, "periodic");
cbd->bk = bk;
MAP_RETAIN (bk, "rspamd_map_backend");
cbd->stage = map_resolve_host2;
double_to_tv (map->cfg->map_timeout, &cbd->tv);
REF_INIT_RETAIN (cbd, free_http_cbdata);
msg_debug_map ("%s map data from %s", check ? "checking" : "reading",
data->host);
/* Send both A and AAAA requests */
if (map->r->r) {
if (rdns_make_request_full (map->r->r, rspamd_map_dns_callback, cbd,
map->cfg->dns_timeout, map->cfg->dns_retransmits, 1,
data->host, RDNS_REQUEST_A)) {
MAP_RETAIN (cbd, "http_callback_data");
}
if (rdns_make_request_full (map->r->r, rspamd_map_dns_callback, cbd,
map->cfg->dns_timeout, map->cfg->dns_retransmits, 1,
data->host, RDNS_REQUEST_AAAA)) {
MAP_RETAIN (cbd, "http_callback_data");
}
map->dtor = free_http_cbdata_dtor;
map->dtor_data = cbd;
}
else {
msg_warn_map ("cannot load map: DNS resolver is not initialized");
cbd->periodic->errored = TRUE;
}
/* We don't need own ref as it is now ref counted by DNS handlers */
MAP_RELEASE (cbd, "http_callback_data");
}
