static gboolean
rspamd_map_check_file_sig (const char *fname,
struct rspamd_map *map,
struct rspamd_map_backend *bk,
const guchar *input,
gsize inlen)
{
gchar fpath[PATH_MAX];
guchar *data;
struct rspamd_cryptobox_pubkey *pk = NULL;
GString *b32_key;
gboolean ret;
gsize len = 0;
if (bk->trusted_pubkey == NULL) {
/* Try to load and check pubkey */
rspamd_snprintf (fpath, sizeof (fpath), "%s.pub", fname);
data = rspamd_file_xmap (fpath, PROT_READ, &len);
if (data == NULL) {
msg_err_map ("can't open pubkey %s: %s", fpath, strerror (errno));
return FALSE;
}
pk = rspamd_pubkey_from_base32 (data, len, RSPAMD_KEYPAIR_SIGN,
RSPAMD_CRYPTOBOX_MODE_25519);
munmap (data, len);
if (pk == NULL) {
msg_err_map ("can't load pubkey %s", fpath);
return FALSE;
}
/* We just check pk against the trusted db of keys */
b32_key = rspamd_pubkey_print (pk,
RSPAMD_KEYPAIR_BASE32|RSPAMD_KEYPAIR_PUBKEY);
g_assert (b32_key != NULL);
if (g_hash_table_lookup (map->cfg->trusted_keys, b32_key->str) == NULL) {
msg_err_map ("pubkey loaded from %s is untrusted: %v", fpath,
b32_key);
g_string_free (b32_key, TRUE);
rspamd_pubkey_unref (pk);
return FALSE;
}
g_string_free (b32_key, TRUE);
}
else {
pk = rspamd_pubkey_ref (bk->trusted_pubkey);
}
ret = rspamd_map_check_sig_pk (fname, map, input, inlen, pk);
rspamd_pubkey_unref (pk);
return ret;
}
static void
rspamd_http_client_func (const gchar *path, rspamd_inet_addr_t *addr,
struct rspamd_cryptobox_keypair *kp,
struct rspamd_cryptobox_pubkey *peer_kp,
struct rspamd_keypair_cache *c,
struct event_base *ev_base, double *latency)
{
struct rspamd_http_message *msg;
struct rspamd_http_connection *conn;
gchar urlbuf[PATH_MAX];
struct client_cbdata *cb;
gint fd;
g_assert ((fd = rspamd_inet_address_connect (addr, SOCK_STREAM, TRUE)) != -1);
conn = rspamd_http_connection_new (rspamd_client_body,
rspamd_client_err,
rspamd_client_finish,
RSPAMD_HTTP_CLIENT_SIMPLE,
RSPAMD_HTTP_CLIENT,
c,
NULL);
rspamd_snprintf (urlbuf, sizeof (urlbuf), "http://127.0.0.1/%s", path);
msg = rspamd_http_message_from_url (urlbuf);
g_assert (conn != NULL && msg != NULL);
if (kp != NULL) {
g_assert (peer_kp != NULL);
rspamd_http_connection_set_key (conn, kp);
msg->peer_key = rspamd_pubkey_ref (peer_kp);
}
cb = g_malloc (sizeof (*cb));
cb->ts = rspamd_get_ticks () * 1000.;
cb->lat = latency;
rspamd_http_connection_write_message (conn, msg, NULL, NULL, cb,
fd, NULL, ev_base);
}
gboolean
rspamd_client_command (struct rspamd_client_connection *conn,
const gchar *command, GQueue *attrs,
FILE *in, rspamd_client_callback cb,
gpointer ud, GError **err)
{
struct rspamd_client_request *req;
struct rspamd_http_client_header *nh;
gchar *p;
gsize remain, old_len;
GList *cur;
GString *input = NULL;
rspamd_fstring_t *body;
req = g_slice_alloc0 (sizeof (struct rspamd_client_request));
req->conn = conn;
req->cb = cb;
req->ud = ud;
req->msg = rspamd_http_new_message (HTTP_REQUEST);
if (conn->key) {
req->msg->peer_key = rspamd_pubkey_ref (conn->key);
}
if (in != NULL) {
/* Read input stream */
input = g_string_sized_new (BUFSIZ);
while (!feof (in)) {
p = input->str + input->len;
remain = input->allocated_len - input->len - 1;
if (remain == 0) {
old_len = input->len;
g_string_set_size (input, old_len * 2);
input->len = old_len;
continue;
}
remain = fread (p, 1, remain, in);
if (remain > 0) {
input->len += remain;
input->str[input->len] = '\0';
}
}
if (ferror (in) != 0) {
g_set_error (err, RCLIENT_ERROR, ferror (
in), "input IO error: %s", strerror (ferror (in)));
g_slice_free1 (sizeof (struct rspamd_client_request), req);
g_string_free (input, TRUE);
return FALSE;
}
body = rspamd_fstring_new_init (input->str, input->len);
rspamd_http_message_set_body_from_fstring_steal (req->msg, body);
req->input = input;
}
else {
req->input = NULL;
}
/* Convert headers */
cur = attrs->head;
while (cur != NULL) {
nh = cur->data;
rspamd_http_message_add_header (req->msg, nh->name, nh->value);
cur = g_list_next (cur);
}
req->msg->url = rspamd_fstring_append (req->msg->url, "/", 1);
req->msg->url = rspamd_fstring_append (req->msg->url, command, strlen (command));
conn->req = req;
rspamd_http_connection_write_message (conn->http_conn, req->msg, NULL,
"text/plain", req, conn->fd, &conn->timeout, conn->ev_base);
return TRUE;
}
static int
http_map_finish (struct rspamd_http_connection *conn,
struct rspamd_http_message *msg)
{
struct http_callback_data *cbd = conn->ud;
struct rspamd_map *map;
rspamd_mempool_t *pool;
char fpath[PATH_MAX];
guchar *aux_data, *in = NULL;
gsize inlen = 0;
struct stat st;
map = cbd->map;
pool = cbd->map->pool;
if (msg->code == 200) {
if (cbd->stage == map_load_file) {
if (msg->last_modified) {
cbd->data->last_checked = msg->last_modified;
}
else {
cbd->data->last_checked = msg->date;
}
/* Maybe we need to check signature ? */
if (map->is_signed) {
close (cbd->out_fd);
if (map->trusted_pubkey) {
/* No need to load key */
cbd->stage = map_load_signature;
cbd->pk = rspamd_pubkey_ref (map->trusted_pubkey);
rspamd_snprintf (fpath, sizeof (fpath), "%s.sig",
cbd->tmpfile);
}
else {
rspamd_snprintf (fpath, sizeof (fpath), "%s.pub",
cbd->tmpfile);
cbd->stage = map_load_pubkey;
}
cbd->out_fd = rspamd_file_xopen (fpath, O_RDWR|O_CREAT, 00644);
if (cbd->out_fd == -1) {
msg_err_pool ("cannot open pubkey file %s for writing: %s",
fpath, strerror (errno));
goto end;
}
rspamd_http_connection_reset (cbd->conn);
write_http_request (cbd);
goto end;
}
else {
/* Unsinged version - just open file */
in = rspamd_file_xmap (cbd->tmpfile, PROT_READ, &inlen);
if (in == NULL) {
msg_err_pool ("cannot read tempfile %s: %s", cbd->tmpfile,
strerror (errno));
goto end;
}
}
}
else if (cbd->stage == map_load_pubkey) {
/* We now can load pubkey */
(void)lseek (cbd->out_fd, 0, SEEK_SET);
if (fstat (cbd->out_fd, &st) == -1) {
msg_err_pool ("cannot stat pubkey file %s: %s",
fpath, strerror (errno));
goto end;
}
aux_data = mmap (NULL, st.st_size, PROT_READ, MAP_SHARED,
cbd->out_fd, 0);
close (cbd->out_fd);
cbd->out_fd = -1;
if (aux_data == MAP_FAILED) {
msg_err_pool ("cannot map pubkey file %s: %s",
fpath, strerror (errno));
goto end;
}
cbd->pk = rspamd_pubkey_from_base32 (aux_data, st.st_size,
RSPAMD_KEYPAIR_SIGN, RSPAMD_CRYPTOBOX_MODE_25519);
munmap (aux_data, st.st_size);
if (cbd->pk == NULL) {
msg_err_pool ("cannot load pubkey file %s: bad pubkey",
fpath);
goto end;
}
rspamd_snprintf (fpath, sizeof (fpath), "%s.sig", cbd->tmpfile);
cbd->out_fd = rspamd_file_xopen (fpath, O_RDWR|O_CREAT, 00644);
if (cbd->out_fd == -1) {
msg_err_pool ("cannot open signature file %s for writing: %s",
fpath, strerror (errno));
goto end;
}
cbd->stage = map_load_signature;
rspamd_http_connection_reset (cbd->conn);
write_http_request (cbd);
goto end;
}
else if (cbd->stage == map_load_signature) {
/* We can now check signature */
close (cbd->out_fd);
cbd->out_fd = -1;
in = rspamd_file_xmap (cbd->tmpfile, PROT_READ, &inlen);
if (in == NULL) {
msg_err_pool ("cannot read tempfile %s: %s", cbd->tmpfile,
strerror (errno));
goto end;
}
if (!rspamd_map_check_sig_pk (cbd->tmpfile, map, in, inlen, cbd->pk)) {
goto end;
}
}
g_assert (in != NULL);
map->read_callback (map->pool, in, inlen, &cbd->cbdata, TRUE);
map->fin_callback (map->pool, &cbd->cbdata);
*map->user_data = cbd->cbdata.cur_data;
msg_info_pool ("read map data from %s", cbd->data->host);
}
else if (msg->code == 304 && cbd->stage == map_load_file) {
msg_debug_pool ("data is not modified for server %s",
cbd->data->host);
if (msg->last_modified) {
cbd->data->last_checked = msg->last_modified;
}
else {
cbd->data->last_checked = msg->date;
}
}
else {
msg_info_pool ("cannot load map %s from %s: HTTP error %d",
map->uri, cbd->data->host, msg->code);
}
end:
REF_RELEASE (cbd);
return 0;
}
/*
* Setup logger
*/
void
rspamd_set_logger (struct rspamd_config *cfg,
GQuark ptype,
struct rspamd_main *rspamd)
{
rspamd_logger_t *logger;
if (rspamd->logger == NULL) {
rspamd->logger = g_slice_alloc0 (sizeof (rspamd_logger_t));
}
logger = rspamd->logger;
logger->type = cfg->log_type;
logger->pid = getpid ();
logger->process_type = ptype;
switch (cfg->log_type) {
case RSPAMD_LOG_CONSOLE:
logger->log_func = file_log_function;
logger->fd = STDERR_FILENO;
break;
case RSPAMD_LOG_SYSLOG:
logger->log_func = syslog_log_function;
break;
case RSPAMD_LOG_FILE:
logger->log_func = file_log_function;
break;
}
logger->cfg = cfg;
/* Set up buffer */
if (cfg->log_buffered) {
if (cfg->log_buf_size != 0) {
logger->io_buf.size = cfg->log_buf_size;
}
else {
logger->io_buf.size = BUFSIZ;
}
logger->is_buffered = TRUE;
logger->io_buf.buf = g_malloc (logger->io_buf.size);
}
/* Set up conditional logging */
if (cfg->debug_ip_map != NULL) {
/* Try to add it as map first of all */
if (logger->debug_ip) {
radix_destroy_compressed (logger->debug_ip);
}
logger->debug_ip = NULL;
rspamd_config_radix_from_ucl (cfg,
cfg->debug_ip_map,
"IP addresses for which debug logs are enabled",
&logger->debug_ip, NULL);
}
else if (logger->debug_ip) {
radix_destroy_compressed (logger->debug_ip);
logger->debug_ip = NULL;
}
if (logger->pk) {
rspamd_pubkey_unref (logger->pk);
}
logger->pk = NULL;
if (logger->keypair) {
rspamd_keypair_unref (logger->keypair);
}
logger->keypair = NULL;
if (cfg->log_encryption_key) {
logger->pk = rspamd_pubkey_ref (cfg->log_encryption_key);
logger->keypair = rspamd_keypair_new (RSPAMD_KEYPAIR_KEX,
RSPAMD_CRYPTOBOX_MODE_25519);
rspamd_pubkey_calculate_nm (logger->pk, logger->keypair);
}
default_logger = logger;
}
static int
http_map_finish (struct rspamd_http_connection *conn,
struct rspamd_http_message *msg)
{
struct http_callback_data *cbd = conn->ud;
struct rspamd_map *map;
struct rspamd_map_backend *bk;
guchar *aux_data, *in = NULL;
gsize inlen = 0, dlen = 0;
map = cbd->map;
bk = cbd->bk;
if (msg->code == 200) {
if (cbd->check) {
cbd->periodic->need_modify = TRUE;
/* Reset the whole chain */
cbd->periodic->cur_backend = 0;
rspamd_map_periodic_callback (-1, EV_TIMEOUT, cbd->periodic);
MAP_RELEASE (cbd, "http_callback_data");
return 0;
}
if (cbd->stage == map_load_file) {
if (msg->last_modified) {
cbd->data->last_checked = msg->last_modified;
}
else {
cbd->data->last_checked = msg->date;
}
/* Maybe we need to check signature ? */
if (bk->is_signed) {
if (bk->trusted_pubkey) {
/* No need to load key */
cbd->stage = map_load_signature;
cbd->pk = rspamd_pubkey_ref (bk->trusted_pubkey);
}
else {
cbd->stage = map_load_pubkey;
}
cbd->shmem_data = rspamd_http_message_shmem_ref (msg);
cbd->data_len = msg->body_buf.len;
rspamd_http_connection_reset (cbd->conn);
write_http_request (cbd);
MAP_RELEASE (cbd, "http_callback_data");
return 0;
}
else {
/* Unsinged version - just open file */
cbd->shmem_data = rspamd_http_message_shmem_ref (msg);
cbd->data_len = msg->body_buf.len;
goto read_data;
}
}
else if (cbd->stage == map_load_pubkey) {
/* We now can load pubkey */
cbd->shmem_pubkey = rspamd_http_message_shmem_ref (msg);
cbd->pubkey_len = msg->body_buf.len;
aux_data = rspamd_shmem_xmap (cbd->shmem_pubkey->shm_name,
PROT_READ, &inlen);
if (aux_data == NULL) {
msg_err_map ("cannot map pubkey file %s: %s",
cbd->shmem_pubkey->shm_name, strerror (errno));
goto err;
}
if (inlen < cbd->pubkey_len) {
msg_err_map ("cannot map pubkey file %s: %s",
cbd->shmem_pubkey->shm_name, strerror (errno));
munmap (aux_data, inlen);
goto err;
}
cbd->pk = rspamd_pubkey_from_base32 (aux_data, cbd->pubkey_len,
RSPAMD_KEYPAIR_SIGN, RSPAMD_CRYPTOBOX_MODE_25519);
munmap (aux_data, inlen);
if (cbd->pk == NULL) {
msg_err_map ("cannot load pubkey file %s: bad pubkey",
cbd->shmem_pubkey->shm_name);
goto err;
}
cbd->stage = map_load_signature;
rspamd_http_connection_reset (cbd->conn);
write_http_request (cbd);
MAP_RELEASE (cbd, "http_callback_data");
return 0;
}
else if (cbd->stage == map_load_signature) {
/* We can now check signature */
cbd->shmem_sig = rspamd_http_message_shmem_ref (msg);
cbd->sig_len = msg->body_buf.len;
aux_data = rspamd_shmem_xmap (cbd->shmem_sig->shm_name,
PROT_READ, &inlen);
if (aux_data == NULL) {
msg_err_map ("cannot map signature file %s: %s",
cbd->shmem_sig->shm_name, strerror (errno));
goto err;
}
if (inlen < cbd->sig_len) {
msg_err_map ("cannot map pubkey file %s: %s",
cbd->shmem_pubkey->shm_name, strerror (errno));
munmap (aux_data, inlen);
goto err;
}
in = rspamd_shmem_xmap (cbd->shmem_data->shm_name, PROT_READ, &dlen);
if (in == NULL) {
msg_err_map ("cannot read tempfile %s: %s",
cbd->shmem_data->shm_name,
strerror (errno));
munmap (aux_data, inlen);
goto err;
}
if (!rspamd_map_check_sig_pk_mem (aux_data, cbd->sig_len, map, in,
cbd->data_len, cbd->pk)) {
munmap (aux_data, inlen);
munmap (in, dlen);
goto err;
}
munmap (in, dlen);
}
read_data:
g_assert (cbd->shmem_data != NULL);
in = rspamd_shmem_xmap (cbd->shmem_data->shm_name, PROT_READ, &dlen);
if (in == NULL) {
msg_err_map ("cannot read tempfile %s: %s",
cbd->shmem_data->shm_name,
strerror (errno));
goto err;
}
map->read_callback (in, cbd->data_len, &cbd->periodic->cbdata, TRUE);
msg_info_map ("read map data from %s", cbd->data->host);
/*
* We know that a map is in the locked state
*/
if (g_atomic_int_compare_and_exchange (&map->cache->available, 0, 1)) {
/* Store cached data */
struct rspamd_http_map_cached_cbdata *cache_cbd;
struct timeval tv;
rspamd_strlcpy (map->cache->shmem_name, cbd->shmem_data->shm_name,
sizeof (map->cache->shmem_name));
map->cache->len = cbd->data_len;
map->cache->last_checked = cbd->data->last_checked;
cache_cbd = g_slice_alloc0 (sizeof (*cache_cbd));
cache_cbd->shm = cbd->shmem_data;
cache_cbd->map = map;
MAP_RETAIN (cache_cbd->shm, "shmem_data");
event_set (&cache_cbd->timeout, -1, EV_TIMEOUT, rspamd_map_cache_cb,
cache_cbd);
event_base_set (cbd->ev_base, &cache_cbd->timeout);
double_to_tv (map->poll_timeout, &tv);
event_add (&cache_cbd->timeout, &tv);
}
cbd->periodic->cur_backend ++;
munmap (in, dlen);
rspamd_map_periodic_callback (-1, EV_TIMEOUT, cbd->periodic);
}
else if (msg->code == 304 && (cbd->check && cbd->stage == map_load_file)) {
msg_debug_map ("data is not modified for server %s",
cbd->data->host);
if (msg->last_modified) {
cbd->data->last_checked = msg->last_modified;
}
else {
cbd->data->last_checked = msg->date;
}
cbd->periodic->cur_backend ++;
rspamd_map_periodic_callback (-1, EV_TIMEOUT, cbd->periodic);
}
else {
msg_info_map ("cannot load map %s from %s: HTTP error %d",
bk->uri, cbd->data->host, msg->code);
}
MAP_RELEASE (cbd, "http_callback_data");
return 0;
err:
cbd->periodic->errored = 1;
rspamd_map_periodic_callback (-1, EV_TIMEOUT, cbd->periodic);
MAP_RELEASE (cbd, "http_callback_data");
return 0;
}
gboolean
rspamd_client_command (struct rspamd_client_connection *conn,
const gchar *command, GQueue *attrs,
FILE *in, rspamd_client_callback cb,
gpointer ud, gboolean compressed,
const gchar *comp_dictionary,
GError **err)
{
struct rspamd_client_request *req;
struct rspamd_http_client_header *nh;
gchar *p;
gsize remain, old_len;
GList *cur;
GString *input = NULL;
rspamd_fstring_t *body;
guint dict_id = 0;
gsize dict_len = 0;
void *dict = NULL;
ZSTD_CCtx *zctx;
req = g_slice_alloc0 (sizeof (struct rspamd_client_request));
req->conn = conn;
req->cb = cb;
req->ud = ud;
req->msg = rspamd_http_new_message (HTTP_REQUEST);
if (conn->key) {
req->msg->peer_key = rspamd_pubkey_ref (conn->key);
}
if (in != NULL) {
/* Read input stream */
input = g_string_sized_new (BUFSIZ);
while (!feof (in)) {
p = input->str + input->len;
remain = input->allocated_len - input->len - 1;
if (remain == 0) {
old_len = input->len;
g_string_set_size (input, old_len * 2);
input->len = old_len;
continue;
}
remain = fread (p, 1, remain, in);
if (remain > 0) {
input->len += remain;
input->str[input->len] = '\0';
}
}
if (ferror (in) != 0) {
g_set_error (err, RCLIENT_ERROR, ferror (
in), "input IO error: %s", strerror (ferror (in)));
g_slice_free1 (sizeof (struct rspamd_client_request), req);
g_string_free (input, TRUE);
return FALSE;
}
if (!compressed) {
body = rspamd_fstring_new_init (input->str, input->len);
}
else {
if (comp_dictionary) {
dict = rspamd_file_xmap (comp_dictionary, PROT_READ, &dict_len);
if (dict == NULL) {
g_set_error (err, RCLIENT_ERROR, errno,
"cannot open dictionary %s: %s",
comp_dictionary,
strerror (errno));
g_slice_free1 (sizeof (struct rspamd_client_request), req);
g_string_free (input, TRUE);
return FALSE;
}
dict_id = ZDICT_getDictID (comp_dictionary, dict_len);
if (dict_id == 0) {
g_set_error (err, RCLIENT_ERROR, errno,
"cannot open dictionary %s: %s",
comp_dictionary,
strerror (errno));
g_slice_free1 (sizeof (struct rspamd_client_request), req);
g_string_free (input, TRUE);
munmap (dict, dict_len);
return FALSE;
}
}
body = rspamd_fstring_sized_new (ZSTD_compressBound (input->len));
zctx = ZSTD_createCCtx ();
body->len = ZSTD_compress_usingDict (zctx, body->str, body->allocated,
input->str, input->len,
dict, dict_len,
1);
munmap (dict, dict_len);
if (ZSTD_isError (body->len)) {
g_set_error (err, RCLIENT_ERROR, ferror (
in), "compression error");
g_slice_free1 (sizeof (struct rspamd_client_request), req);
g_string_free (input, TRUE);
rspamd_fstring_free (body);
ZSTD_freeCCtx (zctx);
return FALSE;
}
ZSTD_freeCCtx (zctx);
}
rspamd_http_message_set_body_from_fstring_steal (req->msg, body);
req->input = input;
}
else {
req->input = NULL;
}
/* Convert headers */
cur = attrs->head;
while (cur != NULL) {
nh = cur->data;
rspamd_http_message_add_header (req->msg, nh->name, nh->value);
cur = g_list_next (cur);
}
if (compressed) {
rspamd_http_message_add_header (req->msg, "Compression", "zstd");
if (dict_id != 0) {
gchar dict_str[32];
rspamd_snprintf (dict_str, sizeof (dict_str), "%ud", dict_id);
rspamd_http_message_add_header (req->msg, "Dictionary", dict_str);
}
}
req->msg->url = rspamd_fstring_append (req->msg->url, "/", 1);
req->msg->url = rspamd_fstring_append (req->msg->url, command, strlen (command));
conn->req = req;
if (compressed) {
rspamd_http_connection_write_message (conn->http_conn, req->msg, NULL,
"application/x-compressed", req, conn->fd,
&conn->timeout, conn->ev_base);
}
else {
rspamd_http_connection_write_message (conn->http_conn, req->msg, NULL,
"text/plain", req, conn->fd, &conn->timeout, conn->ev_base);
}
return TRUE;
}
