struct rspamd_map *
rspamd_map_add (struct rspamd_config *cfg,
const gchar *map_line,
const gchar *description,
map_cb_t read_callback,
map_fin_cb_t fin_callback,
void **user_data)
{
struct rspamd_map *map;
struct rspamd_map_backend *bk;
bk = rspamd_map_parse_backend (cfg, map_line);
if (bk == NULL) {
return NULL;
}
map = g_slice_alloc0 (sizeof (struct rspamd_map));
map->read_callback = read_callback;
map->fin_callback = fin_callback;
map->user_data = user_data;
map->cfg = cfg;
map->id = g_random_int ();
map->locked =
rspamd_mempool_alloc0_shared (cfg->cfg_pool, sizeof (gint));
map->cache =
rspamd_mempool_alloc0_shared (cfg->cfg_pool, sizeof (*map->cache));
map->backends = g_ptr_array_sized_new (1);
g_ptr_array_add (map->backends, bk);
map->name = g_strdup (map_line);
map->poll_timeout = cfg->map_timeout;
if (description != NULL) {
map->description = g_strdup (description);
}
rspamd_map_calculate_hash (map);
msg_info_map ("added map %s", bk->uri);
cfg->maps = g_list_prepend (cfg->maps, map);
return map;
}
struct rspamd_map *
rspamd_map_add (struct rspamd_config *cfg,
const gchar *map_line,
const gchar *description,
map_cb_t read_callback,
map_fin_cb_t fin_callback,
void **user_data)
{
struct rspamd_map *new_map;
const gchar *def;
struct file_map_data *fdata;
struct http_map_data *hdata;
gchar *cksum_encoded, cksum[rspamd_cryptobox_HASHBYTES];
rspamd_mempool_t *pool;
struct http_parser_url up;
rspamd_ftok_t tok;
if (cfg->map_pool == NULL) {
cfg->map_pool = rspamd_mempool_new (rspamd_mempool_suggest_size (),
"map");
memcpy (cfg->map_pool->tag.uid, cfg->cfg_pool->tag.uid,
sizeof (cfg->map_pool->tag.uid));
}
new_map = rspamd_mempool_alloc0 (cfg->map_pool, sizeof (struct rspamd_map));
/* First of all detect protocol line */
if (rspamd_map_check_proto (cfg, map_line, new_map) == NULL) {
return NULL;
}
new_map->read_callback = read_callback;
new_map->fin_callback = fin_callback;
new_map->user_data = user_data;
new_map->cfg = cfg;
new_map->id = g_random_int ();
new_map->locked =
rspamd_mempool_alloc0_shared (cfg->cfg_pool, sizeof (gint));
def = new_map->uri;
if (description != NULL) {
new_map->description =
rspamd_mempool_strdup (cfg->cfg_pool, description);
}
/* Now check for each proto separately */
if (new_map->protocol == MAP_PROTO_FILE) {
fdata =
rspamd_mempool_alloc0 (cfg->map_pool,
sizeof (struct file_map_data));
if (access (def, R_OK) == -1) {
if (errno != ENOENT) {
msg_err_config ("cannot open file '%s': %s", def, strerror
(errno));
return NULL;
}
msg_info_config (
"map '%s' is not found, but it can be loaded automatically later",
def);
/* We still can add this file */
fdata->st.st_mtime = -1;
}
else {
stat (def, &fdata->st);
}
fdata->filename = rspamd_mempool_strdup (cfg->map_pool, def);
new_map->map_data = fdata;
}
else if (new_map->protocol == MAP_PROTO_HTTP) {
hdata =
rspamd_mempool_alloc0 (cfg->map_pool,
sizeof (struct http_map_data));
memset (&up, 0, sizeof (up));
if (http_parser_parse_url (new_map->uri, strlen (new_map->uri), FALSE,
&up) != 0) {
msg_err_config ("cannot parse HTTP url: %s", new_map->uri);
return NULL;
}
else {
if (!(up.field_set & 1 << UF_HOST)) {
msg_err_config ("cannot parse HTTP url: %s: no host", new_map->uri);
return NULL;
}
tok.begin = new_map->uri + up.field_data[UF_HOST].off;
tok.len = up.field_data[UF_HOST].len;
hdata->host = rspamd_mempool_ftokdup (cfg->map_pool, &tok);
if (up.field_set & 1 << UF_PORT) {
hdata->port = up.port;
}
else {
hdata->port = 80;
}
if (up.field_set & 1 << UF_PATH) {
tok.begin = new_map->uri + up.field_data[UF_PATH].off;
tok.len = strlen (tok.begin);
hdata->path = rspamd_mempool_ftokdup (cfg->map_pool, &tok);
}
}
new_map->map_data = hdata;
}
/* Temp pool */
rspamd_cryptobox_hash (cksum, new_map->uri, strlen (new_map->uri), NULL, 0);
cksum_encoded = rspamd_encode_base32 (cksum, sizeof (cksum));
new_map->pool = rspamd_mempool_new (rspamd_mempool_suggest_size (), "map");
rspamd_strlcpy (new_map->pool->tag.uid, cksum_encoded,
sizeof (new_map->pool->tag.uid));
g_free (cksum_encoded);
pool = new_map->pool;
msg_info_pool ("added map %s", new_map->uri);
cfg->maps = g_list_prepend (cfg->maps, new_map);
return new_map;
}
gboolean
rspamd_map_add (struct rspamd_config *cfg,
const gchar *map_line,
const gchar *description,
map_cb_t read_callback,
map_fin_cb_t fin_callback,
void **user_data)
{
struct rspamd_map *new_map;
enum fetch_proto proto;
const gchar *def, *p, *hostend;
struct file_map_data *fdata;
struct http_map_data *hdata;
gchar portbuf[6], *cksum_encoded, cksum[BLAKE2B_OUTBYTES];
gint i, s, r;
struct addrinfo hints, *res;
rspamd_mempool_t *pool;
/* First of all detect protocol line */
if (!rspamd_map_check_proto (map_line, (int *)&proto, &def)) {
return FALSE;
}
/* Constant pool */
if (cfg->map_pool == NULL) {
cfg->map_pool = rspamd_mempool_new (rspamd_mempool_suggest_size (),
"map");
memcpy (cfg->map_pool->tag.uid, cfg->cfg_pool->tag.uid,
sizeof (cfg->map_pool->tag.uid));
}
new_map = rspamd_mempool_alloc0 (cfg->map_pool, sizeof (struct rspamd_map));
new_map->read_callback = read_callback;
new_map->fin_callback = fin_callback;
new_map->user_data = user_data;
new_map->protocol = proto;
new_map->cfg = cfg;
new_map->id = g_random_int ();
new_map->locked =
rspamd_mempool_alloc0_shared (cfg->cfg_pool, sizeof (gint));
if (proto == MAP_PROTO_FILE) {
new_map->uri = rspamd_mempool_strdup (cfg->cfg_pool, def);
def = new_map->uri;
}
else {
new_map->uri = rspamd_mempool_strdup (cfg->cfg_pool, map_line);
}
if (description != NULL) {
new_map->description =
rspamd_mempool_strdup (cfg->cfg_pool, description);
}
/* Now check for each proto separately */
if (proto == MAP_PROTO_FILE) {
fdata =
rspamd_mempool_alloc0 (cfg->map_pool,
sizeof (struct file_map_data));
if (access (def, R_OK) == -1) {
if (errno != ENOENT) {
msg_err_config ("cannot open file '%s': %s", def, strerror
(errno));
return FALSE;
}
msg_info_config (
"map '%s' is not found, but it can be loaded automatically later",
def);
/* We still can add this file */
fdata->st.st_mtime = -1;
}
else {
stat (def, &fdata->st);
}
fdata->filename = rspamd_mempool_strdup (cfg->map_pool, def);
new_map->map_data = fdata;
}
else if (proto == MAP_PROTO_HTTP) {
hdata =
rspamd_mempool_alloc0 (cfg->map_pool,
sizeof (struct http_map_data));
/* Try to search port */
if ((p = strchr (def, ':')) != NULL) {
hostend = p;
i = 0;
p++;
while (g_ascii_isdigit (*p) && i < (gint)sizeof (portbuf) - 1) {
portbuf[i++] = *p++;
}
if (*p != '/') {
msg_info_config ("bad http map definition: %s", def);
return FALSE;
}
portbuf[i] = '\0';
hdata->port = atoi (portbuf);
}
else {
/* Default http port */
rspamd_snprintf (portbuf, sizeof (portbuf), "80");
hdata->port = 80;
/* Now separate host from path */
if ((p = strchr (def, '/')) == NULL) {
msg_info_config ("bad http map definition: %s", def);
return FALSE;
}
hostend = p;
}
hdata->host = rspamd_mempool_alloc (cfg->map_pool, hostend - def + 1);
rspamd_strlcpy (hdata->host, def, hostend - def + 1);
hdata->path = rspamd_mempool_strdup (cfg->map_pool, p);
/* Now try to resolve */
memset (&hints, 0, sizeof (hints));
hints.ai_family = AF_UNSPEC; /* Allow IPv4 or IPv6 */
hints.ai_socktype = SOCK_STREAM; /* Stream socket */
hints.ai_flags = 0;
hints.ai_protocol = 0; /* Any protocol */
hints.ai_canonname = NULL;
hints.ai_addr = NULL;
hints.ai_next = NULL;
if ((r = getaddrinfo (hdata->host, portbuf, &hints, &res)) == 0) {
hdata->addr = res;
rspamd_mempool_add_destructor (cfg->cfg_pool,
(rspamd_mempool_destruct_t)freeaddrinfo, hdata->addr);
}
else {
msg_err_config ("address resolution for %s failed: %s",
hdata->host,
gai_strerror (r));
return FALSE;
}
/* Now try to connect */
if ((s = rspamd_socket_tcp (hdata->addr, FALSE, FALSE)) == -1) {
msg_info_config ("cannot connect to http server %s: %d, %s",
hdata->host,
errno,
strerror (errno));
return FALSE;
}
close (s);
hdata->conn = rspamd_http_connection_new (http_map_read, http_map_error,
http_map_finish,
RSPAMD_HTTP_BODY_PARTIAL | RSPAMD_HTTP_CLIENT_SIMPLE,
RSPAMD_HTTP_CLIENT, NULL);
new_map->map_data = hdata;
}
/* Temp pool */
blake2b (cksum, new_map->uri, NULL, sizeof (cksum), strlen (new_map->uri), 0);
cksum_encoded = rspamd_encode_base32 (cksum, sizeof (cksum));
new_map->pool = rspamd_mempool_new (rspamd_mempool_suggest_size (), "map");
memcpy (new_map->pool->tag.uid, cksum_encoded,
sizeof (new_map->pool->tag.uid));
g_free (cksum_encoded);
pool = new_map->pool;
msg_info_pool ("added map %s", new_map->uri);
cfg->maps = g_list_prepend (cfg->maps, new_map);
return TRUE;
}
struct rspamd_map*
rspamd_map_add_from_ucl (struct rspamd_config *cfg,
const ucl_object_t *obj,
const gchar *description,
map_cb_t read_callback,
map_fin_cb_t fin_callback,
void **user_data)
{
ucl_object_iter_t it = NULL;
const ucl_object_t *cur, *elt;
struct rspamd_map *map;
struct rspamd_map_backend *bk;
g_assert (obj != NULL);
if (ucl_object_type (obj) == UCL_STRING) {
/* Just a plain string */
return rspamd_map_add (cfg, ucl_object_tostring (obj), NULL,
read_callback, fin_callback, user_data);
}
map = g_slice_alloc0 (sizeof (struct rspamd_map));
map->read_callback = read_callback;
map->fin_callback = fin_callback;
map->user_data = user_data;
map->cfg = cfg;
map->id = g_random_int ();
map->locked =
rspamd_mempool_alloc0_shared (cfg->cfg_pool, sizeof (gint));
map->cache =
rspamd_mempool_alloc0_shared (cfg->cfg_pool, sizeof (*map->cache));
map->backends = g_ptr_array_new ();
map->poll_timeout = cfg->map_timeout;
if (description) {
map->description = g_strdup (description);
}
if (ucl_object_type (obj) == UCL_ARRAY) {
/* Add array of maps as multiple backends */
while ((cur = ucl_object_iterate (obj, &it, true)) != NULL) {
if (ucl_object_type (cur) == UCL_STRING) {
bk = rspamd_map_parse_backend (cfg, ucl_object_tostring (cur));
if (bk != NULL) {
g_ptr_array_add (map->backends, bk);
if (!map->name) {
map->name = g_strdup (ucl_object_tostring (cur));
}
}
}
else {
msg_err_config ("bad map element type: %s",
ucl_object_type_to_string (ucl_object_type (cur)));
}
}
if (map->backends->len == 0) {
msg_err_config ("map has no urls to be loaded");
goto err;
}
}
else if (ucl_object_type (obj) == UCL_OBJECT) {
elt = ucl_object_lookup (obj, "name");
if (elt && ucl_object_type (elt) == UCL_STRING) {
map->name = g_strdup (ucl_object_tostring (elt));
}
elt = ucl_object_lookup (obj, "description");
if (elt && ucl_object_type (elt) == UCL_STRING) {
if (map->description) {
g_free (map->description);
}
map->description = g_strdup (ucl_object_tostring (elt));
}
elt = ucl_object_lookup_any (obj, "timeout", "poll", "poll_time",
"watch_interval", NULL);
if (elt) {
map->poll_timeout = ucl_object_todouble (elt);
}
elt = ucl_object_lookup_any (obj, "upstreams", "url", "urls", NULL);
if (elt == NULL) {
msg_err_config ("map has no urls to be loaded");
goto err;
}
if (ucl_object_type (obj) == UCL_ARRAY) {
/* Add array of maps as multiple backends */
while ((cur = ucl_object_iterate (elt, &it, true)) != NULL) {
if (ucl_object_type (cur) == UCL_STRING) {
bk = rspamd_map_parse_backend (cfg, ucl_object_tostring (cur));
if (bk != NULL) {
g_ptr_array_add (map->backends, bk);
if (!map->name) {
map->name = g_strdup (ucl_object_tostring (cur));
}
}
}
else {
msg_err_config ("bad map element type: %s",
ucl_object_type_to_string (ucl_object_type (cur)));
goto err;
}
}
if (map->backends->len == 0) {
msg_err_config ("map has no urls to be loaded");
goto err;
}
}
else if (ucl_object_type (elt) == UCL_STRING) {
bk = rspamd_map_parse_backend (cfg, ucl_object_tostring (elt));
if (bk != NULL) {
g_ptr_array_add (map->backends, bk);
if (!map->name) {
map->name = g_strdup (ucl_object_tostring (cur));
}
}
}
if (map->backends->len == 0) {
msg_err_config ("map has no urls to be loaded");
goto err;
}
}
rspamd_map_calculate_hash (map);
msg_info_map ("added map from ucl");
cfg->maps = g_list_prepend (cfg->maps, map);
return map;
err:
g_ptr_array_free (map->backends, TRUE);
g_free (map->name);
g_free (map->description);
g_slice_free1 (sizeof (*map), map);
return NULL;
}
/*
* Setup logger
*/
void
rspamd_set_logger (struct rspamd_config *cfg,
GQuark ptype,
rspamd_logger_t **plogger,
rspamd_mempool_t *pool)
{
rspamd_logger_t *logger;
if (plogger == NULL || *plogger == NULL) {
logger = g_slice_alloc0 (sizeof (rspamd_logger_t));
if (cfg->log_error_elts > 0 && pool) {
logger->errlog = rspamd_mempool_alloc0_shared (pool,
sizeof (*logger->errlog));
logger->errlog->pool = pool;
logger->errlog->max_elts = cfg->log_error_elts;
logger->errlog->elt_len = cfg->log_error_elt_maxlen;
logger->errlog->elts = rspamd_mempool_alloc0_shared (pool,
sizeof (struct rspamd_logger_error_elt) * cfg->log_error_elts +
cfg->log_error_elt_maxlen * cfg->log_error_elts);
}
if (pool) {
logger->mtx = rspamd_mempool_get_mutex (pool);
}
if (plogger) {
*plogger = logger;
}
}
else {
logger = *plogger;
}
logger->type = cfg->log_type;
logger->pid = getpid ();
logger->process_type = ptype;
switch (cfg->log_type) {
case RSPAMD_LOG_CONSOLE:
logger->log_func = file_log_function;
logger->fd = STDERR_FILENO;
break;
case RSPAMD_LOG_SYSLOG:
logger->log_func = syslog_log_function;
break;
case RSPAMD_LOG_FILE:
logger->log_func = file_log_function;
break;
}
logger->cfg = cfg;
/* Set up buffer */
if (cfg->log_buffered) {
if (cfg->log_buf_size != 0) {
logger->io_buf.size = cfg->log_buf_size;
}
else {
logger->io_buf.size = BUFSIZ;
}
logger->is_buffered = TRUE;
logger->io_buf.buf = g_malloc (logger->io_buf.size);
}
/* Set up conditional logging */
if (cfg->debug_ip_map != NULL) {
/* Try to add it as map first of all */
if (logger->debug_ip) {
radix_destroy_compressed (logger->debug_ip);
}
logger->debug_ip = NULL;
rspamd_config_radix_from_ucl (cfg,
cfg->debug_ip_map,
"IP addresses for which debug logs are enabled",
&logger->debug_ip, NULL);
}
else if (logger->debug_ip) {
radix_destroy_compressed (logger->debug_ip);
logger->debug_ip = NULL;
}
if (logger->pk) {
rspamd_pubkey_unref (logger->pk);
}
logger->pk = NULL;
if (logger->keypair) {
rspamd_keypair_unref (logger->keypair);
}
logger->keypair = NULL;
if (cfg->log_encryption_key) {
logger->pk = rspamd_pubkey_ref (cfg->log_encryption_key);
logger->keypair = rspamd_keypair_new (RSPAMD_KEYPAIR_KEX,
RSPAMD_CRYPTOBOX_MODE_25519);
rspamd_pubkey_calculate_nm (logger->pk, logger->keypair);
}
default_logger = logger;
}
gint
rspamd_symbols_cache_add_symbol (struct symbols_cache *cache,
const gchar *name,
double weight,
gint priority,
symbol_func_t func,
gpointer user_data,
enum rspamd_symbol_type type,
gint parent)
{
struct cache_item *item = NULL;
g_assert (cache != NULL);
if (name == NULL && type != SYMBOL_TYPE_CALLBACK) {
msg_warn ("no name for non-callback symbol!");
}
else if (type == SYMBOL_TYPE_VIRTUAL && parent == -1) {
msg_warn ("no parent symbol is associated with virtual symbol %s",
name);
}
if (name != NULL) {
if (g_hash_table_lookup (cache->items_by_symbol, name) != NULL) {
msg_err ("skip duplicate symbol registration for %s", name);
return -1;
}
}
item = rspamd_mempool_alloc0_shared (cache->static_pool,
sizeof (struct cache_item));
/*
* We do not share cd to skip locking, instead we'll just calculate it on
* save or accumulate
*/
item->cd = rspamd_mempool_alloc0 (cache->static_pool,
sizeof (struct counter_data));
if (name != NULL) {
item->symbol = rspamd_mempool_strdup (cache->static_pool, name);
}
item->func = func;
item->user_data = user_data;
item->priority = priority;
item->type = type;
item->weight = weight;
if (item->weight < 0 && item->priority == 0) {
/* Make priority for negative weighted symbols */
item->priority = 1;
}
item->id = cache->used_items;
item->parent = parent;
cache->used_items ++;
msg_debug ("used items: %d, added symbol: %s", cache->used_items, name);
rspamd_set_counter (item, 0);
g_ptr_array_add (cache->items_by_id, item);
g_ptr_array_add (cache->items_by_order, item);
item->deps = g_ptr_array_new ();
item->rdeps = g_ptr_array_new ();
rspamd_mempool_add_destructor (cache->static_pool,
rspamd_ptr_array_free_hard, item->deps);
rspamd_mempool_add_destructor (cache->static_pool,
rspamd_ptr_array_free_hard, item->rdeps);
if (name != NULL) {
g_hash_table_insert (cache->items_by_symbol, item->symbol, item);
}
return item->id;
}
struct rspamd_symbol_def *s;
gboolean skipped;
if (*cache == NULL) {
pcache = g_new0 (struct symbols_cache, 1);
*cache = pcache;
pcache->static_pool =
rspamd_mempool_new (rspamd_mempool_suggest_size ());
pcache->items_by_symbol = g_hash_table_new (rspamd_str_hash,
rspamd_str_equal);
}
item = rspamd_mempool_alloc0 (pcache->static_pool,
sizeof (struct cache_item));
item->s =
rspamd_mempool_alloc0_shared (pcache->static_pool,
sizeof (struct saved_cache_item));
item->cd = rspamd_mempool_alloc0_shared (pcache->static_pool,
sizeof (struct counter_data));
item->mtx = rspamd_mempool_get_mutex (pcache->static_pool);
rspamd_strlcpy (item->s->symbol, name, sizeof (item->s->symbol));
item->func = func;
item->user_data = user_data;
item->priority = priority;
switch (type) {
case SYMBOL_TYPE_NORMAL:
break;
case SYMBOL_TYPE_VIRTUAL:
item->is_virtual = TRUE;
