Exemple #1
0
gint
rspamd_stat_cache_redis_learn (struct rspamd_task *task,
		gboolean is_spam,
		gpointer runtime)
{
	struct rspamd_redis_cache_runtime *rt = runtime;
	struct timeval tv;
	gchar *h;
	gint flag;

	h = rspamd_mempool_get_variable (task->task_pool, "words_hash");
	g_assert (h != NULL);

	double_to_tv (rt->ctx->timeout, &tv);
	flag = (task->flags & RSPAMD_TASK_FLAG_LEARN_SPAM) ? 1 : -1;

	if (redisAsyncCommand (rt->redis, rspamd_stat_cache_redis_set, rt,
			"HSET %s %s %d",
			rt->ctx->redis_object, h, flag) == REDIS_OK) {
		rspamd_session_add_event (task->s, rspamd_redis_cache_fin, rt,
				rspamd_stat_cache_redis_quark ());
		event_add (&rt->timeout_event, &tv);
		rt->has_event = TRUE;
	}

	/* We need to return OK every time */
	return RSPAMD_LEARN_OK;
}
Exemple #2
0
static void
jitter_timeout_event (struct rspamd_map *map,
		gboolean locked, gboolean initial, gboolean errored)
{
	const gdouble error_mult = 20.0, lock_mult = 0.5;
	gdouble jittered_sec;
	gdouble timeout;

	if (initial) {
		timeout = 0.0;
	}
	else if (errored) {
		timeout = map->cfg->map_timeout * error_mult;
	}
	else if (locked) {
		timeout = map->cfg->map_timeout * lock_mult;
	}
	else {
		timeout = map->cfg->map_timeout;
	}

	/* Plan event again with jitter */
	evtimer_del (&map->ev);
	jittered_sec = rspamd_time_jitter (timeout, 0);
	double_to_tv (jittered_sec, &map->tv);

	evtimer_add (&map->ev, &map->tv);
}
Exemple #3
0
gint
rspamd_stat_cache_redis_check (struct rspamd_task *task,
		gboolean is_spam,
		gpointer runtime)
{
	struct rspamd_redis_cache_runtime *rt = runtime;
	struct timeval tv;
	gchar *h;

	h = rspamd_mempool_get_variable (task->task_pool, "words_hash");

	if (h == NULL) {
		return RSPAMD_LEARN_INGORE;
	}

	double_to_tv (rt->ctx->timeout, &tv);

	if (redisAsyncCommand (rt->redis, rspamd_stat_cache_redis_get, rt,
			"HGET %s %s",
			rt->ctx->redis_object, h) == REDIS_OK) {
		rspamd_session_add_event (task->s, rspamd_redis_cache_fin, rt,
				rspamd_stat_cache_redis_quark ());
		event_add (&rt->timeout_event, &tv);
		rt->has_event = TRUE;
	}

	/* We need to return OK every time */
	return RSPAMD_LEARN_OK;
}
Exemple #4
0
static void
rspamd_symbols_cache_resort_cb (gint fd, short what, gpointer ud)
{
	struct timeval tv;
	gdouble tm;
	struct symbols_cache *cache = ud;
	struct cache_item *item, *parent;
	guint i;

	/* Plan new event */
	tm = rspamd_time_jitter (cache->reload_time, 0);
	msg_debug_cache ("resort symbols cache, next reload in %.2f seconds", tm);
	g_assert (cache != NULL);
	evtimer_set (&cache->resort_ev, rspamd_symbols_cache_resort_cb, cache);
	double_to_tv (tm, &tv);
	event_add (&cache->resort_ev, &tv);

	rspamd_mempool_lock_mutex (cache->mtx);

	/* Gather stats from shared execution times */
	for (i = 0; i < cache->items_by_id->len; i ++) {
		item = g_ptr_array_index (cache->items_by_id, i);

		if (item->type & (SYMBOL_TYPE_CALLBACK|SYMBOL_TYPE_NORMAL)) {
			if (item->cd->number > 0) {
				item->avg_counter += item->cd->number + 1;
				item->avg_time = item->avg_time +
						(item->cd->value - item->avg_time) /
								(gdouble)item->avg_counter;
				item->cd->value = item->avg_time;
				item->cd->number = item->avg_counter;
			}
		}
	}
	/* Sync virtual symbols */
	for (i = 0; i < cache->items_by_id->len; i ++) {
		item = g_ptr_array_index (cache->items_by_id, i);

		if (item->parent != -1) {
			parent = g_ptr_array_index (cache->items_by_id, item->parent);

			if (parent) {
				item->avg_time = parent->avg_time;
				item->avg_counter = parent->avg_counter;
			}
		}
	}

	rspamd_mempool_unlock_mutex (cache->mtx);

	rspamd_symbols_cache_resort (cache);
}
Exemple #5
0
static void
jitter_timeout_event (struct rspamd_map *map, gboolean locked, gboolean initial)
{
    gdouble jittered_sec;
    gdouble timeout = initial ? 1.0 : map->cfg->map_timeout;

    /* Plan event again with jitter */
    evtimer_del (&map->ev);
    jittered_sec = rspamd_time_jitter (locked ? timeout * 4 : timeout, 0);
    double_to_tv (jittered_sec, &map->tv);

    evtimer_add (&map->ev, &map->tv);
}
Exemple #6
0
gboolean
rspamd_redis_process_tokens (struct rspamd_task *task,
		GPtrArray *tokens,
		gint id, gpointer p)
{
	struct redis_stat_runtime *rt = REDIS_RUNTIME (p);
	rspamd_fstring_t *query;
	struct timeval tv;
	gint ret;

	if (tokens == NULL || tokens->len == 0 || rt->redis == NULL) {
		return FALSE;
	}

	rt->id = id;

	if (redisAsyncCommand (rt->redis, rspamd_redis_connected, rt, "HGET %s %s",
			rt->redis_object_expanded, "learns") == REDIS_OK) {

		rspamd_session_add_event (task->s, rspamd_redis_fin, rt,
				rspamd_redis_stat_quark ());
		rt->has_event = TRUE;

		if (event_get_base (&rt->timeout_event)) {
			event_del (&rt->timeout_event);
		}
		event_set (&rt->timeout_event, -1, EV_TIMEOUT, rspamd_redis_timeout, rt);
		event_base_set (task->ev_base, &rt->timeout_event);
		double_to_tv (rt->ctx->timeout, &tv);
		event_add (&rt->timeout_event, &tv);

		query = rspamd_redis_tokens_to_query (task, tokens,
				"HMGET", rt->redis_object_expanded, FALSE, -1,
				rt->stcf->clcf->flags & RSPAMD_FLAG_CLASSIFIER_INTEGER);
		g_assert (query != NULL);
		rspamd_mempool_add_destructor (task->task_pool,
				(rspamd_mempool_destruct_t)rspamd_fstring_free, query);

		ret = redisAsyncFormattedCommand (rt->redis, rspamd_redis_processed, rt,
				query->str, query->len);

		if (ret == REDIS_OK) {
			return TRUE;
		}
		else {
			msg_err_task ("call to redis failed: %s", rt->redis->errstr);
		}
	}

	return FALSE;
}
Exemple #7
0
void
rspamd_symbols_cache_start_refresh (struct symbols_cache * cache,
		struct event_base *ev_base)
{
	struct timeval tv;
	gdouble tm;

	tm = rspamd_time_jitter (cache->reload_time, 0);
	g_assert (cache != NULL);
	evtimer_set (&cache->resort_ev, rspamd_symbols_cache_resort_cb, cache);
	event_base_set (ev_base, &cache->resort_ev);
	double_to_tv (tm, &tv);
	event_add (&cache->resort_ev, &tv);
}
Exemple #8
0
struct rspamd_client_connection *
rspamd_client_init (struct event_base *ev_base, const gchar *name,
	guint16 port, gdouble timeout, const gchar *key)
{
	struct rspamd_client_connection *conn;
	gint fd;

	fd = rspamd_socket (name, port, SOCK_STREAM, TRUE, FALSE, TRUE);
	if (fd == -1) {
		return NULL;
	}

	conn = g_slice_alloc0 (sizeof (struct rspamd_client_connection));
	conn->ev_base = ev_base;
	conn->fd = fd;
	conn->req_sent = FALSE;
	conn->keys_cache = rspamd_keypair_cache_new (32);
	conn->http_conn = rspamd_http_connection_new (rspamd_client_body_handler,
			rspamd_client_error_handler,
			rspamd_client_finish_handler,
			0,
			RSPAMD_HTTP_CLIENT,
			conn->keys_cache,
			NULL);

	conn->server_name = g_string_new (name);
	if (port != 0) {
		rspamd_printf_gstring (conn->server_name, ":%d", (int)port);
	}

	double_to_tv (timeout, &conn->timeout);

	if (key) {
		conn->key = rspamd_pubkey_from_base32 (key, 0, RSPAMD_KEYPAIR_KEX,
				RSPAMD_CRYPTOBOX_MODE_25519);

		if (conn->key) {
			conn->keypair = rspamd_keypair_new (RSPAMD_KEYPAIR_KEX,
					RSPAMD_CRYPTOBOX_MODE_25519);
			rspamd_http_connection_set_key (conn->http_conn, conn->keypair);
		}
		else {
			rspamd_client_destroy (conn);
			return NULL;
		}
	}

	return conn;
}
Exemple #9
0
static void
rspamd_async_elt_on_timer (gint fd, short what, gpointer d)
{
	struct rspamd_stat_async_elt *elt = d;
	gdouble jittered_time;

	event_del (&elt->timer_ev);

	if (elt->enabled) {
		elt->handler (elt, elt->ud);
	}

	jittered_time = rspamd_time_jitter (elt->timeout, 0);
	double_to_tv (jittered_time, &elt->tv);
	event_add (&elt->timer_ev, &elt->tv);
}
Exemple #10
0
static void
rspamd_map_schedule_periodic (struct rspamd_map *map,
		gboolean locked, gboolean initial, gboolean errored)
{
	const gdouble error_mult = 20.0, lock_mult = 0.1;
	gdouble jittered_sec;
	gdouble timeout;
	struct map_periodic_cbdata *cbd;

	timeout = map->poll_timeout;

	if (initial) {
		timeout = 0.0;
	}

	if (errored) {
		timeout = map->poll_timeout * error_mult;
	}
	else if (locked) {
		timeout = lock_mult;
	}

	cbd = g_slice_alloc0 (sizeof (*cbd));
	cbd->cbdata.state = 0;
	cbd->cbdata.prev_data = *map->user_data;
	cbd->cbdata.cur_data = NULL;
	cbd->cbdata.map = map;
	cbd->map = map;
	REF_INIT_RETAIN (cbd, rspamd_map_periodic_dtor);

	if (initial) {
		evtimer_set (&map->ev, rspamd_map_periodic_callback, cbd);
		event_base_set (map->ev_base, &map->ev);
	}
	else {
		evtimer_del (&map->ev);
		evtimer_set (&map->ev, rspamd_map_periodic_callback, cbd);
		event_base_set (map->ev_base, &map->ev);
	}

	jittered_sec = rspamd_time_jitter (timeout, 0);
	msg_debug_map ("schedule new periodic event %p in %.2f seconds", cbd, jittered_sec);
	double_to_tv (jittered_sec, &map->tv);

	evtimer_add (&map->ev, &map->tv);
}
Exemple #11
0
static void
rspamadm_control (gint argc, gchar **argv)
{
	GOptionContext *context;
	GError *error = NULL;
	struct event_base *ev_base;
	const gchar *cmd, *path = NULL;
	struct rspamd_http_connection *conn;
	struct rspamd_http_message *msg;
	rspamd_inet_addr_t *addr;
	struct timeval tv;
	static struct rspamadm_control_cbdata cbdata;
	lua_State *L;
	gint sock;

	context = g_option_context_new (
			"control - manage rspamd main control interface");
	g_option_context_set_summary (context,
			"Summary:\n  Rspamd administration utility version "
					RVERSION
					"\n  Release id: "
					RID);
	g_option_context_add_main_entries (context, entries, NULL);
	g_option_context_set_ignore_unknown_options (context, TRUE);

	if (!g_option_context_parse (context, &argc, &argv, &error)) {
		rspamd_fprintf (stderr, "option parsing failed: %s\n", error->message);
		g_error_free (error);
		exit (1);
	}

	if (argc <= 1) {
		rspamd_fprintf (stderr, "command required\n");
		exit (1);
	}

	cmd = argv[1];

	if (g_ascii_strcasecmp (cmd, "stat") == 0) {
		path = "/stat";
	}
	else if (g_ascii_strcasecmp (cmd, "reload") == 0) {
		path = "/reload";
	}
	else if (g_ascii_strcasecmp (cmd, "reresolve") == 0) {
		path = "/reresolve";
	}
	else if (g_ascii_strcasecmp (cmd, "recompile") == 0) {
		path = "/recompile";
	}
	else if (g_ascii_strcasecmp (cmd, "fuzzystat") == 0 ||
			g_ascii_strcasecmp (cmd, "fuzzy_stat") == 0) {
		path = "/fuzzystat";
	}
	else if (g_ascii_strcasecmp (cmd, "fuzzysync") == 0 ||
			g_ascii_strcasecmp (cmd, "fuzzy_sync") == 0) {
		path = "/fuzzysync";
	}
	else {
		rspamd_fprintf (stderr, "unknown command: %s\n", cmd);
		exit (1);
	}

	if (!rspamd_parse_inet_address (&addr, control_path, 0)) {
		rspamd_fprintf (stderr, "bad control path: %s\n", control_path);
		exit (1);
	}

	ev_base = event_init ();
	sock = rspamd_inet_address_connect (addr, SOCK_STREAM, TRUE);

	if (sock == -1) {
		rspamd_fprintf (stderr, "cannot connect to: %s\n", control_path);
		rspamd_inet_address_destroy (addr);
		exit (1);
	}

	L = rspamd_lua_init ();

	conn = rspamd_http_connection_new (NULL,
			rspamd_control_error_handler,
			rspamd_control_finish_handler,
			RSPAMD_HTTP_CLIENT_SIMPLE,
			RSPAMD_HTTP_CLIENT,
			NULL,
			NULL);
	msg = rspamd_http_new_message (HTTP_REQUEST);
	msg->url = rspamd_fstring_new_init (path, strlen (path));
	double_to_tv (timeout, &tv);

	cbdata.L = L;
	cbdata.argc = argc;
	cbdata.argv = argv;
	cbdata.path = path;

	rspamd_http_connection_write_message (conn, msg, NULL, NULL, &cbdata, sock,
			&tv, ev_base);

	event_base_loop (ev_base, 0);

	rspamd_http_connection_unref (conn);
	rspamd_inet_address_destroy (addr);
	lua_close (L);
	close (sock);
}
Exemple #12
0
gboolean
rspamd_redis_learn_tokens (struct rspamd_task *task, GPtrArray *tokens,
		gint id, gpointer p)
{
	struct redis_stat_runtime *rt = REDIS_RUNTIME (p);
	struct upstream *up;
	rspamd_inet_addr_t *addr;
	struct timeval tv;
	rspamd_fstring_t *query;
	const gchar *redis_cmd;
	rspamd_token_t *tok;
	gint ret;

	up = rspamd_upstream_get (rt->ctx->write_servers,
			RSPAMD_UPSTREAM_MASTER_SLAVE,
			NULL,
			0);

	if (up == NULL) {
		msg_err_task ("no upstreams reachable");
		return FALSE;
	}

	rt->selected = up;

	addr = rspamd_upstream_addr (up);
	g_assert (addr != NULL);
	rt->redis = redisAsyncConnect (rspamd_inet_address_to_string (addr),
			rspamd_inet_address_get_port (addr));
	g_assert (rt->redis != NULL);

	redisLibeventAttach (rt->redis, task->ev_base);
	rspamd_redis_maybe_auth (rt->ctx, rt->redis);

	/*
	 * Add the current key to the set of learned keys
	 */
	redisAsyncCommand (rt->redis, NULL, NULL, "SADD %s_keys %s",
			rt->stcf->symbol, rt->redis_object_expanded);

	if (rt->stcf->clcf->flags & RSPAMD_FLAG_CLASSIFIER_INTEGER) {
		redis_cmd = "HINCRBY";
	}
	else {
		redis_cmd = "HINCRBYFLOAT";
	}

	rt->id = id;
	query = rspamd_redis_tokens_to_query (task, tokens,
			redis_cmd, rt->redis_object_expanded, TRUE, id,
			rt->stcf->clcf->flags & RSPAMD_FLAG_CLASSIFIER_INTEGER);
	g_assert (query != NULL);

	/*
	 * XXX:
	 * Dirty hack: we get a token and check if it's value is -1 or 1, so
	 * we could understand that we are learning or unlearning
	 */

	tok = g_ptr_array_index (task->tokens, 0);

	if (tok->values[id] > 0) {
		rspamd_printf_fstring (&query, ""
				"*4\r\n"
				"$7\r\n"
				"HINCRBY\r\n"
				"$%d\r\n"
				"%s\r\n"
				"$6\r\n"
				"learns\r\n"
				"$1\r\n"
				"1\r\n",
				(gint)strlen (rt->redis_object_expanded),
				rt->redis_object_expanded);
	}
	else {
		rspamd_printf_fstring (&query, ""
				"*4\r\n"
				"$7\r\n"
				"HINCRBY\r\n"
				"$%d\r\n"
				"%s\r\n"
				"$6\r\n"
				"learns\r\n"
				"$2\r\n"
				"-1\r\n",
				(gint)strlen (rt->redis_object_expanded),
				rt->redis_object_expanded);
	}

	rspamd_mempool_add_destructor (task->task_pool,
				(rspamd_mempool_destruct_t)rspamd_fstring_free, query);

	ret = redisAsyncFormattedCommand (rt->redis, rspamd_redis_learned, rt,
			query->str, query->len);

	if (ret == REDIS_OK) {
		rspamd_session_add_event (task->s, rspamd_redis_fin_learn, rt,
				rspamd_redis_stat_quark ());
		rt->has_event = TRUE;

		/* Set timeout */
		if (event_get_base (&rt->timeout_event)) {
			event_del (&rt->timeout_event);
		}
		event_set (&rt->timeout_event, -1, EV_TIMEOUT, rspamd_redis_timeout, rt);
		event_base_set (task->ev_base, &rt->timeout_event);
		double_to_tv (rt->ctx->timeout, &tv);
		event_add (&rt->timeout_event, &tv);

		return TRUE;
	}
	else {
		msg_err_task ("call to redis failed: %s", rt->redis->errstr);
	}

	return FALSE;
}
Exemple #13
0
/**
 * Async HTTP callback
 */
static void
http_callback (gint fd, short what, void *ud)
{
	struct rspamd_map *map = ud;
	struct http_map_data *data;
	struct http_callback_data *cbd;
	rspamd_mempool_t *pool;
	gchar tmpbuf[PATH_MAX];

	data = map->map_data;
	pool = map->pool;

	if (!g_atomic_int_compare_and_exchange (map->locked, 0, 1)) {
		msg_debug_pool (
				"don't try to reread map as it is locked by other process, will reread it later");
		jitter_timeout_event (map, TRUE, FALSE, FALSE);
		return;
	}

	/* Plan event */
	cbd = g_slice_alloc0 (sizeof (struct http_callback_data));

	rspamd_snprintf (tmpbuf, sizeof (tmpbuf),
			"%s" G_DIR_SEPARATOR_S "rspamd_map%d-XXXXXX",
			map->cfg->temp_dir, map->id);
	cbd->out_fd = mkstemp (tmpbuf);

	if (cbd->out_fd == -1) {
		g_slice_free1 (sizeof (*cbd), cbd);
		msg_err_pool ("cannot create tempfile: %s", strerror (errno));
		jitter_timeout_event (map, FALSE, FALSE, TRUE);
		g_atomic_int_set (map->locked, 0);

		return;
	}

	cbd->tmpfile = g_strdup (tmpbuf);
	cbd->ev_base = map->ev_base;
	cbd->map = map;
	cbd->data = data;
	cbd->fd = -1;
	cbd->cbdata.state = 0;
	cbd->cbdata.prev_data = *cbd->map->user_data;
	cbd->cbdata.cur_data = NULL;
	cbd->cbdata.map = cbd->map;
	cbd->stage = map_resolve_host2;
	double_to_tv (map->cfg->map_timeout, &cbd->tv);
	REF_INIT_RETAIN (cbd, free_http_cbdata);

	msg_debug_pool ("reading map data from %s", data->host);
	/* Send both A and AAAA requests */
	if (map->r->r) {
		if (rdns_make_request_full (map->r->r, rspamd_map_dns_callback, cbd,
				map->cfg->dns_timeout, map->cfg->dns_retransmits, 1,
				data->host, RDNS_REQUEST_A)) {
			REF_RETAIN (cbd);
		}
		if (rdns_make_request_full (map->r->r, rspamd_map_dns_callback, cbd,
				map->cfg->dns_timeout, map->cfg->dns_retransmits, 1,
				data->host, RDNS_REQUEST_AAAA)) {
			REF_RETAIN (cbd);
		}

		jitter_timeout_event (map, FALSE, FALSE, FALSE);
		map->dtor = free_http_cbdata_dtor;
		map->dtor_data = cbd;
	}
	else {
		msg_warn_pool ("cannot load map: DNS resolver is not initialized");
		jitter_timeout_event (map, FALSE, FALSE, TRUE);
	}

	/* We don't need own ref as it is now refcounted by DNS requests */
	REF_RELEASE (cbd);
}
Exemple #14
0
/***
 * @function rspamd_redis.make_request({params})
 * Make request to redis server, params is a table of key=value arguments in any order
 * @param {task} task worker task object
 * @param {ip} host server address
 * @param {function} callback callback to be called in form `function (task, err, data)`
 * @param {string} cmd command to be sent to redis
 * @param {table} args numeric array of strings used as redis arguments
 * @param {number} timeout timeout in seconds for request (1.0 by default)
 * @return {boolean} `true` if a request has been scheduled
 */
static int
lua_redis_make_request (lua_State *L)
{
	struct lua_redis_userdata *ud;
	struct rspamd_lua_ip *addr = NULL;
	struct rspamd_task *task = NULL;
	const gchar *cmd = NULL;
	gint top, cbref = -1;
	struct timeval tv;
	gboolean ret = FALSE;
	gdouble timeout = REDIS_DEFAULT_TIMEOUT;

	if (lua_istable (L, 1)) {
		/* Table version */
		lua_pushstring (L, "task");
		lua_gettable (L, -2);
		if (lua_type (L, -1) == LUA_TUSERDATA) {
			task = lua_check_task (L, -1);
		}
		lua_pop (L, 1);

		lua_pushstring (L, "callback");
		lua_gettable (L, -2);
		if (lua_type (L, -1) == LUA_TFUNCTION) {
			/* This also pops function from the stack */
			cbref = luaL_ref (L, LUA_REGISTRYINDEX);
		}
		else {
			msg_err ("bad callback argument for lua redis");
			lua_pop (L, 1);
		}

		lua_pushstring (L, "cmd");
		lua_gettable (L, -2);
		cmd = lua_tostring (L, -1);
		lua_pop (L, 1);

		lua_pushstring (L, "host");
		lua_gettable (L, -2);
		if (lua_type (L, -1) == LUA_TUSERDATA) {
			addr = lua_check_ip (L, -1);
		}
		lua_pop (L, 1);

		lua_pushstring (L, "timeout");
		lua_gettable (L, -2);
		timeout = lua_tonumber (L, -1);
		lua_pop (L, 1);

		if (task != NULL && addr != NULL && cbref != -1 && cmd != NULL) {
			ud =
					rspamd_mempool_alloc (task->task_pool,
							sizeof (struct lua_redis_userdata));
			ud->task = task;
			ud->L = L;
			ud->cbref = cbref;
			lua_pushstring (L, "args");
			lua_redis_parse_args (L, -1, cmd, ud);
			ret = TRUE;
		}
		else {
			if (cbref != -1) {
				luaL_unref (L, LUA_REGISTRYINDEX, cbref);
			}

			msg_err ("incorrect function invocation");
		}
	}
	else if ((task = lua_check_task (L, 1)) != NULL) {
		addr = lua_check_ip (L, 2);
		top = lua_gettop (L);
		/* Now get callback */
		if (lua_isfunction (L, 3) && addr != NULL && addr->addr && top >= 4) {
			/* Create userdata */
			ud =
				rspamd_mempool_alloc (task->task_pool,
					sizeof (struct lua_redis_userdata));
			ud->task = task;
			ud->L = L;

			/* Pop other arguments */
			lua_pushvalue (L, 3);
			/* Get a reference */
			ud->cbref = luaL_ref (L, LUA_REGISTRYINDEX);

			cmd = luaL_checkstring (L, 4);
			if (top > 4) {
				lua_redis_parse_args (L, 5, cmd, ud);
			}
			else {
				lua_redis_parse_args (L, 0, cmd, ud);
			}

			ret = TRUE;
		}
		else {
			msg_err ("incorrect function invocation");
		}
	}

	if (ret) {
		ud->terminated = 0;
		ud->ctx = redisAsyncConnect (rspamd_inet_address_to_string (addr->addr),
				rspamd_inet_address_get_port (addr->addr));
		redisAsyncSetConnectCallback (ud->ctx, lua_redis_connect_cb);

		if (ud->ctx == NULL || ud->ctx->err) {
			ud->terminated = 1;
			redisAsyncFree (ud->ctx);
			lua_redis_free_args (ud);
			luaL_unref (ud->L, LUA_REGISTRYINDEX, ud->cbref);
			lua_pushboolean (L, FALSE);

			return 1;
		}
		redisLibeventAttach (ud->ctx, ud->task->ev_base);
		ret = redisAsyncCommandArgv (ud->ctx,
					lua_redis_callback,
					ud,
					ud->nargs,
					(const gchar **)ud->args,
					NULL);
		if (ret == REDIS_OK) {
			rspamd_session_add_event (ud->task->s,
					lua_redis_fin,
					ud,
					g_quark_from_static_string ("lua redis"));

			double_to_tv (timeout, &tv);
			event_set (&ud->timeout, -1, EV_TIMEOUT, lua_redis_timeout, ud);
			event_base_set (ud->task->ev_base, &ud->timeout);
			event_add (&ud->timeout, &tv);
		}
		else {
			msg_info ("call to redis failed: %s", ud->ctx->errstr);
			ud->terminated = 1;
			lua_redis_free_args (ud);
			redisAsyncFree (ud->ctx);
			luaL_unref (ud->L, LUA_REGISTRYINDEX, ud->cbref);
		}
	}

	lua_pushboolean (L, ret);

	return 1;
}
Exemple #15
0
static int
http_map_finish (struct rspamd_http_connection *conn,
		struct rspamd_http_message *msg)
{
	struct http_callback_data *cbd = conn->ud;
	struct rspamd_map *map;
	struct rspamd_map_backend *bk;
	guchar *aux_data, *in = NULL;
	gsize inlen = 0, dlen = 0;

	map = cbd->map;
	bk = cbd->bk;

	if (msg->code == 200) {

		if (cbd->check) {
			cbd->periodic->need_modify = TRUE;
			/* Reset the whole chain */
			cbd->periodic->cur_backend = 0;
			rspamd_map_periodic_callback (-1, EV_TIMEOUT, cbd->periodic);
			MAP_RELEASE (cbd, "http_callback_data");

			return 0;
		}

		if (cbd->stage == map_load_file) {
			if (msg->last_modified) {
				cbd->data->last_checked = msg->last_modified;
			}
			else {
				cbd->data->last_checked = msg->date;
			}

			/* Maybe we need to check signature ? */
			if (bk->is_signed) {

				if (bk->trusted_pubkey) {
					/* No need to load key */
					cbd->stage = map_load_signature;
					cbd->pk = rspamd_pubkey_ref (bk->trusted_pubkey);
				}
				else {
					cbd->stage = map_load_pubkey;
				}

				cbd->shmem_data = rspamd_http_message_shmem_ref (msg);
				cbd->data_len = msg->body_buf.len;
				rspamd_http_connection_reset (cbd->conn);
				write_http_request (cbd);
				MAP_RELEASE (cbd, "http_callback_data");

				return 0;
			}
			else {
				/* Unsinged version - just open file */
				cbd->shmem_data = rspamd_http_message_shmem_ref (msg);
				cbd->data_len = msg->body_buf.len;

				goto read_data;
			}
		}
		else if (cbd->stage == map_load_pubkey) {
			/* We now can load pubkey */
			cbd->shmem_pubkey = rspamd_http_message_shmem_ref (msg);
			cbd->pubkey_len = msg->body_buf.len;

			aux_data = rspamd_shmem_xmap (cbd->shmem_pubkey->shm_name,
					PROT_READ, &inlen);

			if (aux_data == NULL) {
				msg_err_map ("cannot map pubkey file %s: %s",
						cbd->shmem_pubkey->shm_name, strerror (errno));
				goto err;
			}

			if (inlen < cbd->pubkey_len) {
				msg_err_map ("cannot map pubkey file %s: %s",
						cbd->shmem_pubkey->shm_name, strerror (errno));
				munmap (aux_data, inlen);
				goto err;
			}

			cbd->pk = rspamd_pubkey_from_base32 (aux_data, cbd->pubkey_len,
					RSPAMD_KEYPAIR_SIGN, RSPAMD_CRYPTOBOX_MODE_25519);
			munmap (aux_data, inlen);

			if (cbd->pk == NULL) {
				msg_err_map ("cannot load pubkey file %s: bad pubkey",
						cbd->shmem_pubkey->shm_name);
				goto err;
			}

			cbd->stage = map_load_signature;
			rspamd_http_connection_reset (cbd->conn);
			write_http_request (cbd);
			MAP_RELEASE (cbd, "http_callback_data");

			return 0;
		}
		else if (cbd->stage == map_load_signature) {
			/* We can now check signature */
			cbd->shmem_sig = rspamd_http_message_shmem_ref (msg);
			cbd->sig_len = msg->body_buf.len;

			aux_data = rspamd_shmem_xmap (cbd->shmem_sig->shm_name,
					PROT_READ, &inlen);

			if (aux_data == NULL) {
				msg_err_map ("cannot map signature file %s: %s",
						cbd->shmem_sig->shm_name, strerror (errno));
				goto err;
			}

			if (inlen < cbd->sig_len) {
				msg_err_map ("cannot map pubkey file %s: %s",
						cbd->shmem_pubkey->shm_name, strerror (errno));
				munmap (aux_data, inlen);
				goto err;
			}

			in = rspamd_shmem_xmap (cbd->shmem_data->shm_name, PROT_READ, &dlen);

			if (in == NULL) {
				msg_err_map ("cannot read tempfile %s: %s",
						cbd->shmem_data->shm_name,
						strerror (errno));
				munmap (aux_data, inlen);
				goto err;
			}

			if (!rspamd_map_check_sig_pk_mem (aux_data, cbd->sig_len, map, in,
					cbd->data_len, cbd->pk)) {
				munmap (aux_data, inlen);
				munmap (in, dlen);
				goto err;
			}

			munmap (in, dlen);
		}

read_data:
		g_assert (cbd->shmem_data != NULL);

		in = rspamd_shmem_xmap (cbd->shmem_data->shm_name, PROT_READ, &dlen);

		if (in == NULL) {
			msg_err_map ("cannot read tempfile %s: %s",
					cbd->shmem_data->shm_name,
					strerror (errno));
			goto err;
		}

		map->read_callback (in, cbd->data_len, &cbd->periodic->cbdata, TRUE);
		msg_info_map ("read map data from %s", cbd->data->host);

		/*
		 * We know that a map is in the locked state
		 */
		if (g_atomic_int_compare_and_exchange (&map->cache->available, 0, 1)) {
			/* Store cached data */
			struct rspamd_http_map_cached_cbdata *cache_cbd;
			struct timeval tv;

			rspamd_strlcpy (map->cache->shmem_name, cbd->shmem_data->shm_name,
					sizeof (map->cache->shmem_name));
			map->cache->len = cbd->data_len;
			map->cache->last_checked = cbd->data->last_checked;
			cache_cbd = g_slice_alloc0 (sizeof (*cache_cbd));
			cache_cbd->shm = cbd->shmem_data;
			cache_cbd->map = map;
			MAP_RETAIN (cache_cbd->shm, "shmem_data");
			event_set (&cache_cbd->timeout, -1, EV_TIMEOUT, rspamd_map_cache_cb,
					cache_cbd);
			event_base_set (cbd->ev_base, &cache_cbd->timeout);
			double_to_tv (map->poll_timeout, &tv);
			event_add (&cache_cbd->timeout, &tv);
		}

		cbd->periodic->cur_backend ++;
		munmap (in, dlen);
		rspamd_map_periodic_callback (-1, EV_TIMEOUT, cbd->periodic);
	}
	else if (msg->code == 304 && (cbd->check && cbd->stage == map_load_file)) {
		msg_debug_map ("data is not modified for server %s",
				cbd->data->host);

		if (msg->last_modified) {
			cbd->data->last_checked = msg->last_modified;
		}
		else {
			cbd->data->last_checked = msg->date;
		}

		cbd->periodic->cur_backend ++;
		rspamd_map_periodic_callback (-1, EV_TIMEOUT, cbd->periodic);
	}
	else {
		msg_info_map ("cannot load map %s from %s: HTTP error %d",
				bk->uri, cbd->data->host, msg->code);
	}

	MAP_RELEASE (cbd, "http_callback_data");
	return 0;

err:
	cbd->periodic->errored = 1;
	rspamd_map_periodic_callback (-1, EV_TIMEOUT, cbd->periodic);
	MAP_RELEASE (cbd, "http_callback_data");

	return 0;
}
Exemple #16
0
/**
 * Async HTTP callback
 */
static void
rspamd_map_common_http_callback (struct rspamd_map *map, struct rspamd_map_backend *bk,
		struct map_periodic_cbdata *periodic, gboolean check)
{
	struct http_map_data *data;
	struct http_callback_data *cbd;
	gchar tmpbuf[PATH_MAX];

	data = bk->data.hd;
	cbd = g_slice_alloc0 (sizeof (struct http_callback_data));

	rspamd_snprintf (tmpbuf, sizeof (tmpbuf),
			"%s" G_DIR_SEPARATOR_S "rspamd_map%d-XXXXXX",
			map->cfg->temp_dir, map->id);
	cbd->out_fd = mkstemp (tmpbuf);

	if (cbd->out_fd == -1) {
		msg_err_map ("cannot create tempfile: %s", strerror (errno));
		g_atomic_int_set (map->locked, 0);
		g_slice_free1 (sizeof (*cbd), cbd);
		periodic->errored = TRUE;
		rspamd_map_periodic_callback (-1, EV_TIMEOUT, periodic);

		return;
	}

	cbd->tmpfile = g_strdup (tmpbuf);
	cbd->ev_base = map->ev_base;
	cbd->map = map;
	cbd->data = data;
	cbd->fd = -1;
	cbd->check = check;
	cbd->periodic = periodic;
	MAP_RETAIN (periodic);
	cbd->bk = bk;
	MAP_RETAIN (bk);
	cbd->stage = map_resolve_host2;
	double_to_tv (map->cfg->map_timeout, &cbd->tv);
	REF_INIT_RETAIN (cbd, free_http_cbdata);

	msg_debug_map ("%s map data from %s", check ? "checking" : "reading",
			data->host);
	/* Send both A and AAAA requests */
	if (map->r->r) {
		if (rdns_make_request_full (map->r->r, rspamd_map_dns_callback, cbd,
				map->cfg->dns_timeout, map->cfg->dns_retransmits, 1,
				data->host, RDNS_REQUEST_A)) {
			MAP_RETAIN (cbd);
		}
		if (rdns_make_request_full (map->r->r, rspamd_map_dns_callback, cbd,
				map->cfg->dns_timeout, map->cfg->dns_retransmits, 1,
				data->host, RDNS_REQUEST_AAAA)) {
			MAP_RETAIN (cbd);
		}

		map->dtor = free_http_cbdata_dtor;
		map->dtor_data = cbd;
	}
	else {
		msg_warn_map ("cannot load map: DNS resolver is not initialized");
		cbd->periodic->errored = TRUE;
	}

	/* We don't need own ref as it is now ref counted by DNS handlers */
	MAP_RELEASE (cbd);
}
void
rspamd_fuzzy_backend_count_redis (struct rspamd_fuzzy_backend *bk,
		rspamd_fuzzy_count_cb cb, void *ud,
		void *subr_ud)
{
	struct rspamd_fuzzy_backend_redis *backend = subr_ud;
	struct rspamd_fuzzy_redis_session *session;
	struct upstream *up;
	struct timeval tv;
	rspamd_inet_addr_t *addr;

	g_assert (backend != NULL);

	session = g_slice_alloc0 (sizeof (*session));
	session->backend = backend;
	REF_RETAIN (session->backend);

	session->callback.cb_count = cb;
	session->cbdata = ud;
	session->command = RSPAMD_FUZZY_REDIS_COMMAND_COUNT;

	session->nargs = 2;
	session->argv = g_malloc (sizeof (gchar *) * 2);
	session->argv[0] = g_strdup ("HLEN");
	session->argv[1] = g_strdup (backend->redis_object);

	up = rspamd_upstream_get (backend->read_servers,
			RSPAMD_UPSTREAM_ROUND_ROBIN,
			NULL,
			0);

	session->up = up;
	addr = rspamd_upstream_addr (up);
	g_assert (addr != NULL);
	session->ctx = rspamd_redis_pool_connect (backend->pool,
			backend->dbname, backend->password,
			rspamd_inet_address_to_string (addr),
			rspamd_inet_address_get_port (addr));

	if (session->ctx == NULL) {
		rspamd_fuzzy_redis_session_dtor (session);

		if (cb) {
			cb (0, subr_ud);
		}
	}
	else {
		if (redisAsyncCommandArgv (session->ctx, rspamd_fuzzy_redis_count_callback,
				session, session->nargs,
				(const gchar **)session->argv, NULL) != REDIS_OK) {
			rspamd_fuzzy_redis_session_dtor (session);

			if (cb) {
				cb (0, subr_ud);
			}
		}
		else {
			/* Add timeout */
			event_set (&session->timeout, -1, EV_TIMEOUT, rspamd_fuzzy_redis_timeout,
					session);
			event_base_set (rspamd_fuzzy_backend_event_base (bk),
					&session->timeout);
			double_to_tv (backend->timeout, &tv);
			event_add (&session->timeout, &tv);
		}
	}
}
Exemple #18
0
/**
 * Async HTTP callback
 */
static void
rspamd_map_common_http_callback (struct rspamd_map *map, struct rspamd_map_backend *bk,
		struct map_periodic_cbdata *periodic, gboolean check)
{
	struct http_map_data *data;
	struct http_callback_data *cbd;


	data = bk->data.hd;

	if (g_atomic_int_get (&map->cache->available) == 1) {
		/* Read cached data */
		if (check) {
			if (data->last_checked < map->cache->last_checked) {
				periodic->need_modify = TRUE;
				/* Reset the whole chain */
				periodic->cur_backend = 0;
				rspamd_map_periodic_callback (-1, EV_TIMEOUT, periodic);
			}

			return;
		}
		else if (rspamd_map_read_cached (map, periodic, data->host)) {
			/* Switch to the next backend */
			periodic->cur_backend ++;
			rspamd_map_periodic_callback (-1, EV_TIMEOUT, periodic);
			data->last_checked = map->cache->last_checked;

			return;
		}
	}

	cbd = g_slice_alloc0 (sizeof (struct http_callback_data));

	cbd->ev_base = map->ev_base;
	cbd->map = map;
	cbd->data = data;
	cbd->fd = -1;
	cbd->check = check;
	cbd->periodic = periodic;
	MAP_RETAIN (periodic, "periodic");
	cbd->bk = bk;
	MAP_RETAIN (bk, "rspamd_map_backend");
	cbd->stage = map_resolve_host2;
	double_to_tv (map->cfg->map_timeout, &cbd->tv);
	REF_INIT_RETAIN (cbd, free_http_cbdata);

	msg_debug_map ("%s map data from %s", check ? "checking" : "reading",
			data->host);
	/* Send both A and AAAA requests */
	if (map->r->r) {
		if (rdns_make_request_full (map->r->r, rspamd_map_dns_callback, cbd,
				map->cfg->dns_timeout, map->cfg->dns_retransmits, 1,
				data->host, RDNS_REQUEST_A)) {
			MAP_RETAIN (cbd, "http_callback_data");
		}
		if (rdns_make_request_full (map->r->r, rspamd_map_dns_callback, cbd,
				map->cfg->dns_timeout, map->cfg->dns_retransmits, 1,
				data->host, RDNS_REQUEST_AAAA)) {
			MAP_RETAIN (cbd, "http_callback_data");
		}

		map->dtor = free_http_cbdata_dtor;
		map->dtor_data = cbd;
	}
	else {
		msg_warn_map ("cannot load map: DNS resolver is not initialized");
		cbd->periodic->errored = TRUE;
	}

	/* We don't need own ref as it is now ref counted by DNS handlers */
	MAP_RELEASE (cbd, "http_callback_data");
}