/**
* Common file callback
*/
static void
file_callback (gint fd, short what, void *ud)
{
struct rspamd_map *map = ud;
struct file_map_data *data = map->map_data;
struct stat st;
rspamd_mempool_t *pool;
pool = map->pool;
if (g_atomic_int_get (map->locked)) {
msg_info_pool (
"don't try to reread map as it is locked by other process, will reread it later");
jitter_timeout_event (map, TRUE, FALSE);
return;
}
g_atomic_int_inc (map->locked);
jitter_timeout_event (map, FALSE, FALSE);
if (stat (data->filename,
&st) != -1 &&
(st.st_mtime > data->st.st_mtime || data->st.st_mtime == -1)) {
/* File was modified since last check */
memcpy (&data->st, &st, sizeof (struct stat));
}
else {
g_atomic_int_set (map->locked, 0);
return;
}
msg_info_pool ("rereading map file %s", data->filename);
read_map_file (map, data);
g_atomic_int_set (map->locked, 0);
}
gpointer
rspamd_tokenizer_osb_get_config (rspamd_mempool_t *pool,
struct rspamd_tokenizer_config *cf,
gsize *len)
{
struct rspamd_osb_tokenizer_config *osb_cf, *def;
if (cf != NULL && cf->opts != NULL) {
osb_cf = rspamd_tokenizer_osb_config_from_ucl (pool, cf->opts);
}
else {
def = rspamd_tokenizer_osb_default_config ();
osb_cf = rspamd_mempool_alloc (pool, sizeof (*osb_cf));
memcpy (osb_cf, def, sizeof (*osb_cf));
/* Do not write sipkey to statfile */
}
if (osb_cf->ht == RSPAMD_OSB_HASH_SIPHASH) {
msg_info_pool ("siphash key is not stored into statfiles, so you'd "
"need to keep it inside the configuration");
}
memset (osb_cf->sk, 0, sizeof (osb_cf->sk));
if (len != NULL) {
*len = sizeof (*osb_cf);
}
return osb_cf;
}
static guint
rspamd_re_cache_process_pcre (struct rspamd_re_runtime *rt,
rspamd_regexp_t *re, rspamd_mempool_t *pool,
const guchar *in, gsize len,
gboolean is_raw)
{
guint r = 0;
const gchar *start = NULL, *end = NULL;
guint max_hits = rspamd_regexp_get_maxhits (re);
guint64 id = rspamd_regexp_get_cache_id (re);
gdouble t1, t2;
const gdouble slow_time = 0.1;
if (len == 0) {
len = strlen (in);
}
if (rt->cache->max_re_data > 0 && len > rt->cache->max_re_data) {
len = rt->cache->max_re_data;
}
r = rt->results[id];
t1 = rspamd_get_ticks ();
if (max_hits == 0 || r < max_hits) {
while (rspamd_regexp_search (re,
in,
len,
&start,
&end,
is_raw,
NULL)) {
r++;
if (max_hits > 0 && r >= max_hits) {
break;
}
}
rt->stat.regexp_checked++;
rt->stat.bytes_scanned_pcre += len;
rt->stat.bytes_scanned += len;
if (r > 0) {
rt->stat.regexp_matched += r;
}
}
t2 = rspamd_get_ticks ();
if (t2 - t1 > slow_time) {
msg_info_pool ("regexp '%16s' took %.2f seconds to execute",
rspamd_regexp_get_pattern (re), t2 - t1);
}
return r;
}
void
rspamd_radix_fin (rspamd_mempool_t * pool, struct map_cb_data *data)
{
if (data->prev_data) {
radix_destroy_compressed (data->prev_data);
}
if (data->cur_data) {
msg_info_pool ("read radix trie of %z elements", radix_get_size
(data->cur_data));
}
}
void
rspamd_kv_list_fin (rspamd_mempool_t * pool, struct map_cb_data *data)
{
if (data->prev_data) {
g_hash_table_destroy (data->prev_data);
}
if (data->cur_data) {
msg_info_pool ("read hash of %z elements", g_hash_table_size
(data->cur_data));
}
}
static int
http_map_finish (struct rspamd_http_connection *conn,
struct rspamd_http_message *msg)
{
struct http_callback_data *cbd = conn->ud;
struct rspamd_map *map;
rspamd_mempool_t *pool;
map = cbd->map;
pool = cbd->map->pool;
if (msg->code == 200) {
if (cbd->remain_buf != NULL) {
/* Append \n to avoid issues */
g_string_append_c (cbd->remain_buf, '\n');
map->read_callback (map->pool, cbd->remain_buf->str,
cbd->remain_buf->len, &cbd->cbdata);
}
map->fin_callback (map->pool, &cbd->cbdata);
*map->user_data = cbd->cbdata.cur_data;
cbd->data->last_checked = msg->date;
msg_info_pool ("read map data from %s", cbd->data->host);
}
else if (msg->code == 304) {
msg_debug_pool ("data is not modified for server %s",
cbd->data->host);
cbd->data->last_checked = msg->date;
}
else {
msg_info_pool ("cannot load map %s from %s: HTTP error %d",
map->uri, cbd->data->host, msg->code);
}
free_http_cbdata (cbd);
return 0;
}
/**
* Async HTTP callback
*/
static void
http_callback (gint fd, short what, void *ud)
{
struct rspamd_map *map = ud;
struct http_map_data *data;
gint sock;
struct http_callback_data *cbd;
rspamd_mempool_t *pool;
data = map->map_data;
pool = map->pool;
if (g_atomic_int_get (map->locked)) {
msg_info_pool (
"don't try to reread map as it is locked by other process, will reread it later");
if (data->conn->ud == NULL) {
jitter_timeout_event (map, TRUE, TRUE);
}
else {
jitter_timeout_event (map, TRUE, FALSE);
}
return;
}
g_atomic_int_inc (map->locked);
jitter_timeout_event (map, FALSE, FALSE);
/* Connect asynced */
if ((sock = connect_http (map, data, TRUE)) == -1) {
g_atomic_int_set (map->locked, 0);
return;
}
else {
/* Plan event */
cbd = g_slice_alloc (sizeof (struct http_callback_data));
cbd->ev_base = map->ev_base;
cbd->map = map;
cbd->data = data;
cbd->remain_buf = NULL;
cbd->cbdata.state = 0;
cbd->cbdata.prev_data = *cbd->map->user_data;
cbd->cbdata.cur_data = NULL;
cbd->cbdata.map = cbd->map;
cbd->tv.tv_sec = HTTP_CONNECT_TIMEOUT;
cbd->tv.tv_usec = 0;
cbd->fd = sock;
data->conn->ud = cbd;
msg_debug_pool ("reading map data from %s", data->host);
write_http_request (cbd);
}
}
static gboolean
rspamd_map_check_sig_pk (const char *fname,
struct rspamd_map *map,
const guchar *input,
gsize inlen,
struct rspamd_cryptobox_pubkey *pk)
{
gchar fpath[PATH_MAX];
rspamd_mempool_t *pool = map->pool;
guchar *data;
GString *b32_key;
gsize len = 0;
/* Now load signature */
rspamd_snprintf (fpath, sizeof (fpath), "%s.sig", fname);
data = rspamd_file_xmap (fpath, PROT_READ, &len);
if (data == NULL) {
msg_err_pool ("can't open signature %s: %s", fpath, strerror (errno));
return FALSE;
}
if (len != rspamd_cryptobox_signature_bytes (RSPAMD_CRYPTOBOX_MODE_25519)) {
msg_err_pool ("can't open signature %s: invalid signature", fpath);
munmap (data, len);
return FALSE;
}
if (!rspamd_cryptobox_verify (data, input, inlen,
rspamd_pubkey_get_pk (pk, NULL), RSPAMD_CRYPTOBOX_MODE_25519)) {
msg_err_pool ("can't verify signature %s: incorrect signature", fpath);
munmap (data, len);
return FALSE;
}
b32_key = rspamd_pubkey_print (pk,
RSPAMD_KEYPAIR_BASE32|RSPAMD_KEYPAIR_PUBKEY);
msg_info_pool ("verified signature in file %s using trusted key %v",
fpath, b32_key);
g_string_free (b32_key, TRUE);
munmap (data, len);
return TRUE;
}
/**
* Helper for HTTP connection establishment
*/
static gint
connect_http (struct rspamd_map *map,
struct http_map_data *data,
gboolean is_async)
{
gint sock;
rspamd_mempool_t *pool;
pool = map->pool;
if ((sock = rspamd_socket_tcp (data->addr, FALSE, is_async)) == -1) {
msg_info_pool ("cannot connect to http server %s: %d, %s",
data->host,
errno,
strerror (errno));
return -1;
}
return sock;
}
static void
rspamd_html_url_is_phished (rspamd_mempool_t *pool,
struct rspamd_url *href_url,
const guchar *url_text,
gsize len,
gboolean *url_found)
{
struct rspamd_url *text_url;
gint rc;
gchar *url_str = NULL;
*url_found = FALSE;
if (rspamd_url_find (pool, url_text, len, &url_str, TRUE) && url_str != NULL) {
text_url = rspamd_mempool_alloc0 (pool, sizeof (struct rspamd_url));
rc = rspamd_url_parse (text_url, url_str, strlen (url_str), pool);
if (rc == URI_ERRNO_OK) {
if (href_url->hostlen != text_url->hostlen || memcmp (href_url->host,
text_url->host, href_url->hostlen) != 0) {
if (href_url->tldlen != text_url->tldlen || memcmp (href_url->tld,
text_url->tld, href_url->tldlen) != 0) {
href_url->flags |= RSPAMD_URL_FLAG_PHISHED;
href_url->phished_url = text_url;
}
}
*url_found = TRUE;
}
else {
msg_info_pool ("extract of url '%s' failed: %s",
url_str,
rspamd_url_strerror (rc));
}
}
}
struct rspamd_map *
rspamd_map_add (struct rspamd_config *cfg,
const gchar *map_line,
const gchar *description,
map_cb_t read_callback,
map_fin_cb_t fin_callback,
void **user_data)
{
struct rspamd_map *new_map;
const gchar *def;
struct file_map_data *fdata;
struct http_map_data *hdata;
gchar *cksum_encoded, cksum[rspamd_cryptobox_HASHBYTES];
rspamd_mempool_t *pool;
struct http_parser_url up;
rspamd_ftok_t tok;
if (cfg->map_pool == NULL) {
cfg->map_pool = rspamd_mempool_new (rspamd_mempool_suggest_size (),
"map");
memcpy (cfg->map_pool->tag.uid, cfg->cfg_pool->tag.uid,
sizeof (cfg->map_pool->tag.uid));
}
new_map = rspamd_mempool_alloc0 (cfg->map_pool, sizeof (struct rspamd_map));
/* First of all detect protocol line */
if (rspamd_map_check_proto (cfg, map_line, new_map) == NULL) {
return NULL;
}
new_map->read_callback = read_callback;
new_map->fin_callback = fin_callback;
new_map->user_data = user_data;
new_map->cfg = cfg;
new_map->id = g_random_int ();
new_map->locked =
rspamd_mempool_alloc0_shared (cfg->cfg_pool, sizeof (gint));
def = new_map->uri;
if (description != NULL) {
new_map->description =
rspamd_mempool_strdup (cfg->cfg_pool, description);
}
/* Now check for each proto separately */
if (new_map->protocol == MAP_PROTO_FILE) {
fdata =
rspamd_mempool_alloc0 (cfg->map_pool,
sizeof (struct file_map_data));
if (access (def, R_OK) == -1) {
if (errno != ENOENT) {
msg_err_config ("cannot open file '%s': %s", def, strerror
(errno));
return NULL;
}
msg_info_config (
"map '%s' is not found, but it can be loaded automatically later",
def);
/* We still can add this file */
fdata->st.st_mtime = -1;
}
else {
stat (def, &fdata->st);
}
fdata->filename = rspamd_mempool_strdup (cfg->map_pool, def);
new_map->map_data = fdata;
}
else if (new_map->protocol == MAP_PROTO_HTTP) {
hdata =
rspamd_mempool_alloc0 (cfg->map_pool,
sizeof (struct http_map_data));
memset (&up, 0, sizeof (up));
if (http_parser_parse_url (new_map->uri, strlen (new_map->uri), FALSE,
&up) != 0) {
msg_err_config ("cannot parse HTTP url: %s", new_map->uri);
return NULL;
}
else {
if (!(up.field_set & 1 << UF_HOST)) {
msg_err_config ("cannot parse HTTP url: %s: no host", new_map->uri);
return NULL;
}
tok.begin = new_map->uri + up.field_data[UF_HOST].off;
tok.len = up.field_data[UF_HOST].len;
hdata->host = rspamd_mempool_ftokdup (cfg->map_pool, &tok);
if (up.field_set & 1 << UF_PORT) {
hdata->port = up.port;
}
else {
hdata->port = 80;
}
if (up.field_set & 1 << UF_PATH) {
tok.begin = new_map->uri + up.field_data[UF_PATH].off;
tok.len = strlen (tok.begin);
hdata->path = rspamd_mempool_ftokdup (cfg->map_pool, &tok);
}
}
new_map->map_data = hdata;
}
/* Temp pool */
rspamd_cryptobox_hash (cksum, new_map->uri, strlen (new_map->uri), NULL, 0);
cksum_encoded = rspamd_encode_base32 (cksum, sizeof (cksum));
new_map->pool = rspamd_mempool_new (rspamd_mempool_suggest_size (), "map");
rspamd_strlcpy (new_map->pool->tag.uid, cksum_encoded,
sizeof (new_map->pool->tag.uid));
g_free (cksum_encoded);
pool = new_map->pool;
msg_info_pool ("added map %s", new_map->uri);
cfg->maps = g_list_prepend (cfg->maps, new_map);
return new_map;
}
static int
http_map_finish (struct rspamd_http_connection *conn,
struct rspamd_http_message *msg)
{
struct http_callback_data *cbd = conn->ud;
struct rspamd_map *map;
rspamd_mempool_t *pool;
char fpath[PATH_MAX];
guchar *aux_data, *in = NULL;
gsize inlen = 0;
struct stat st;
map = cbd->map;
pool = cbd->map->pool;
if (msg->code == 200) {
if (cbd->stage == map_load_file) {
if (msg->last_modified) {
cbd->data->last_checked = msg->last_modified;
}
else {
cbd->data->last_checked = msg->date;
}
/* Maybe we need to check signature ? */
if (map->is_signed) {
close (cbd->out_fd);
if (map->trusted_pubkey) {
/* No need to load key */
cbd->stage = map_load_signature;
cbd->pk = rspamd_pubkey_ref (map->trusted_pubkey);
rspamd_snprintf (fpath, sizeof (fpath), "%s.sig",
cbd->tmpfile);
}
else {
rspamd_snprintf (fpath, sizeof (fpath), "%s.pub",
cbd->tmpfile);
cbd->stage = map_load_pubkey;
}
cbd->out_fd = rspamd_file_xopen (fpath, O_RDWR|O_CREAT, 00644);
if (cbd->out_fd == -1) {
msg_err_pool ("cannot open pubkey file %s for writing: %s",
fpath, strerror (errno));
goto end;
}
rspamd_http_connection_reset (cbd->conn);
write_http_request (cbd);
goto end;
}
else {
/* Unsinged version - just open file */
in = rspamd_file_xmap (cbd->tmpfile, PROT_READ, &inlen);
if (in == NULL) {
msg_err_pool ("cannot read tempfile %s: %s", cbd->tmpfile,
strerror (errno));
goto end;
}
}
}
else if (cbd->stage == map_load_pubkey) {
/* We now can load pubkey */
(void)lseek (cbd->out_fd, 0, SEEK_SET);
if (fstat (cbd->out_fd, &st) == -1) {
msg_err_pool ("cannot stat pubkey file %s: %s",
fpath, strerror (errno));
goto end;
}
aux_data = mmap (NULL, st.st_size, PROT_READ, MAP_SHARED,
cbd->out_fd, 0);
close (cbd->out_fd);
cbd->out_fd = -1;
if (aux_data == MAP_FAILED) {
msg_err_pool ("cannot map pubkey file %s: %s",
fpath, strerror (errno));
goto end;
}
cbd->pk = rspamd_pubkey_from_base32 (aux_data, st.st_size,
RSPAMD_KEYPAIR_SIGN, RSPAMD_CRYPTOBOX_MODE_25519);
munmap (aux_data, st.st_size);
if (cbd->pk == NULL) {
msg_err_pool ("cannot load pubkey file %s: bad pubkey",
fpath);
goto end;
}
rspamd_snprintf (fpath, sizeof (fpath), "%s.sig", cbd->tmpfile);
cbd->out_fd = rspamd_file_xopen (fpath, O_RDWR|O_CREAT, 00644);
if (cbd->out_fd == -1) {
msg_err_pool ("cannot open signature file %s for writing: %s",
fpath, strerror (errno));
goto end;
}
cbd->stage = map_load_signature;
rspamd_http_connection_reset (cbd->conn);
write_http_request (cbd);
goto end;
}
else if (cbd->stage == map_load_signature) {
/* We can now check signature */
close (cbd->out_fd);
cbd->out_fd = -1;
in = rspamd_file_xmap (cbd->tmpfile, PROT_READ, &inlen);
if (in == NULL) {
msg_err_pool ("cannot read tempfile %s: %s", cbd->tmpfile,
strerror (errno));
goto end;
}
if (!rspamd_map_check_sig_pk (cbd->tmpfile, map, in, inlen, cbd->pk)) {
goto end;
}
}
g_assert (in != NULL);
map->read_callback (map->pool, in, inlen, &cbd->cbdata, TRUE);
map->fin_callback (map->pool, &cbd->cbdata);
*map->user_data = cbd->cbdata.cur_data;
msg_info_pool ("read map data from %s", cbd->data->host);
}
else if (msg->code == 304 && cbd->stage == map_load_file) {
msg_debug_pool ("data is not modified for server %s",
cbd->data->host);
if (msg->last_modified) {
cbd->data->last_checked = msg->last_modified;
}
else {
cbd->data->last_checked = msg->date;
}
}
else {
msg_info_pool ("cannot load map %s from %s: HTTP error %d",
map->uri, cbd->data->host, msg->code);
}
end:
REF_RELEASE (cbd);
return 0;
}
gboolean
rspamd_map_add (struct rspamd_config *cfg,
const gchar *map_line,
const gchar *description,
map_cb_t read_callback,
map_fin_cb_t fin_callback,
void **user_data)
{
struct rspamd_map *new_map;
enum fetch_proto proto;
const gchar *def, *p, *hostend;
struct file_map_data *fdata;
struct http_map_data *hdata;
gchar portbuf[6], *cksum_encoded, cksum[BLAKE2B_OUTBYTES];
gint i, s, r;
struct addrinfo hints, *res;
rspamd_mempool_t *pool;
/* First of all detect protocol line */
if (!rspamd_map_check_proto (map_line, (int *)&proto, &def)) {
return FALSE;
}
/* Constant pool */
if (cfg->map_pool == NULL) {
cfg->map_pool = rspamd_mempool_new (rspamd_mempool_suggest_size (),
"map");
memcpy (cfg->map_pool->tag.uid, cfg->cfg_pool->tag.uid,
sizeof (cfg->map_pool->tag.uid));
}
new_map = rspamd_mempool_alloc0 (cfg->map_pool, sizeof (struct rspamd_map));
new_map->read_callback = read_callback;
new_map->fin_callback = fin_callback;
new_map->user_data = user_data;
new_map->protocol = proto;
new_map->cfg = cfg;
new_map->id = g_random_int ();
new_map->locked =
rspamd_mempool_alloc0_shared (cfg->cfg_pool, sizeof (gint));
if (proto == MAP_PROTO_FILE) {
new_map->uri = rspamd_mempool_strdup (cfg->cfg_pool, def);
def = new_map->uri;
}
else {
new_map->uri = rspamd_mempool_strdup (cfg->cfg_pool, map_line);
}
if (description != NULL) {
new_map->description =
rspamd_mempool_strdup (cfg->cfg_pool, description);
}
/* Now check for each proto separately */
if (proto == MAP_PROTO_FILE) {
fdata =
rspamd_mempool_alloc0 (cfg->map_pool,
sizeof (struct file_map_data));
if (access (def, R_OK) == -1) {
if (errno != ENOENT) {
msg_err_config ("cannot open file '%s': %s", def, strerror
(errno));
return FALSE;
}
msg_info_config (
"map '%s' is not found, but it can be loaded automatically later",
def);
/* We still can add this file */
fdata->st.st_mtime = -1;
}
else {
stat (def, &fdata->st);
}
fdata->filename = rspamd_mempool_strdup (cfg->map_pool, def);
new_map->map_data = fdata;
}
else if (proto == MAP_PROTO_HTTP) {
hdata =
rspamd_mempool_alloc0 (cfg->map_pool,
sizeof (struct http_map_data));
/* Try to search port */
if ((p = strchr (def, ':')) != NULL) {
hostend = p;
i = 0;
p++;
while (g_ascii_isdigit (*p) && i < (gint)sizeof (portbuf) - 1) {
portbuf[i++] = *p++;
}
if (*p != '/') {
msg_info_config ("bad http map definition: %s", def);
return FALSE;
}
portbuf[i] = '\0';
hdata->port = atoi (portbuf);
}
else {
/* Default http port */
rspamd_snprintf (portbuf, sizeof (portbuf), "80");
hdata->port = 80;
/* Now separate host from path */
if ((p = strchr (def, '/')) == NULL) {
msg_info_config ("bad http map definition: %s", def);
return FALSE;
}
hostend = p;
}
hdata->host = rspamd_mempool_alloc (cfg->map_pool, hostend - def + 1);
rspamd_strlcpy (hdata->host, def, hostend - def + 1);
hdata->path = rspamd_mempool_strdup (cfg->map_pool, p);
/* Now try to resolve */
memset (&hints, 0, sizeof (hints));
hints.ai_family = AF_UNSPEC; /* Allow IPv4 or IPv6 */
hints.ai_socktype = SOCK_STREAM; /* Stream socket */
hints.ai_flags = 0;
hints.ai_protocol = 0; /* Any protocol */
hints.ai_canonname = NULL;
hints.ai_addr = NULL;
hints.ai_next = NULL;
if ((r = getaddrinfo (hdata->host, portbuf, &hints, &res)) == 0) {
hdata->addr = res;
rspamd_mempool_add_destructor (cfg->cfg_pool,
(rspamd_mempool_destruct_t)freeaddrinfo, hdata->addr);
}
else {
msg_err_config ("address resolution for %s failed: %s",
hdata->host,
gai_strerror (r));
return FALSE;
}
/* Now try to connect */
if ((s = rspamd_socket_tcp (hdata->addr, FALSE, FALSE)) == -1) {
msg_info_config ("cannot connect to http server %s: %d, %s",
hdata->host,
errno,
strerror (errno));
return FALSE;
}
close (s);
hdata->conn = rspamd_http_connection_new (http_map_read, http_map_error,
http_map_finish,
RSPAMD_HTTP_BODY_PARTIAL | RSPAMD_HTTP_CLIENT_SIMPLE,
RSPAMD_HTTP_CLIENT, NULL);
new_map->map_data = hdata;
}
/* Temp pool */
blake2b (cksum, new_map->uri, NULL, sizeof (cksum), strlen (new_map->uri), 0);
cksum_encoded = rspamd_encode_base32 (cksum, sizeof (cksum));
new_map->pool = rspamd_mempool_new (rspamd_mempool_suggest_size (), "map");
memcpy (new_map->pool->tag.uid, cksum_encoded,
sizeof (new_map->pool->tag.uid));
g_free (cksum_encoded);
pool = new_map->pool;
msg_info_pool ("added map %s", new_map->uri);
cfg->maps = g_list_prepend (cfg->maps, new_map);
return TRUE;
}
