static void rspamd_map_file_check_callback (gint fd, short what, void *ud) { struct rspamd_map *map; struct map_periodic_cbdata *periodic = ud; struct file_map_data *data; struct rspamd_map_backend *bk; struct stat st; map = periodic->map; bk = g_ptr_array_index (map->backends, periodic->cur_backend); data = bk->data.fd; if (stat (data->filename, &st) != -1 && (st.st_mtime > data->st.st_mtime || data->st.st_mtime == -1)) { /* File was modified since last check */ msg_info_map ("old mtime is %t, new mtime is %t for map file %s", data->st.st_mtime, st.st_mtime, data->filename); memcpy (&data->st, &st, sizeof (struct stat)); periodic->need_modify = TRUE; periodic->cur_backend = 0; rspamd_map_periodic_callback (-1, EV_TIMEOUT, periodic); return; } /* Switch to the next backend */ periodic->cur_backend ++; rspamd_map_periodic_callback (-1, EV_TIMEOUT, periodic); }
static gboolean rspamd_map_check_sig_pk_mem (const guchar *sig, gsize siglen, struct rspamd_map *map, const guchar *input, gsize inlen, struct rspamd_cryptobox_pubkey *pk) { GString *b32_key; gboolean ret = TRUE; if (siglen != rspamd_cryptobox_signature_bytes (RSPAMD_CRYPTOBOX_MODE_25519)) { msg_err_map ("can't open signature for %s: invalid size: %z", map->name, siglen); ret = FALSE; } if (ret && !rspamd_cryptobox_verify (sig, input, inlen, rspamd_pubkey_get_pk (pk, NULL), RSPAMD_CRYPTOBOX_MODE_25519)) { msg_err_map ("can't verify signature for %s: incorrect signature", map->name); ret = FALSE; } if (ret) { b32_key = rspamd_pubkey_print (pk, RSPAMD_KEYPAIR_BASE32|RSPAMD_KEYPAIR_PUBKEY); msg_info_map ("verified signature for %s using trusted key %v", map->name, b32_key); g_string_free (b32_key, TRUE); } return ret; }
static gboolean rspamd_map_check_sig_pk (const char *fname, struct rspamd_map *map, const guchar *input, gsize inlen, struct rspamd_cryptobox_pubkey *pk) { gchar fpath[PATH_MAX]; guchar *data; GString *b32_key; gsize len = 0; /* Now load signature */ rspamd_snprintf (fpath, sizeof (fpath), "%s.sig", fname); data = rspamd_file_xmap (fpath, PROT_READ, &len); if (data == NULL) { msg_err_map ("can't open signature %s: %s", fpath, strerror (errno)); return FALSE; } if (len != rspamd_cryptobox_signature_bytes (RSPAMD_CRYPTOBOX_MODE_25519)) { msg_err_map ("can't open signature %s: invalid signature", fpath); munmap (data, len); return FALSE; } if (!rspamd_cryptobox_verify (data, input, inlen, rspamd_pubkey_get_pk (pk, NULL), RSPAMD_CRYPTOBOX_MODE_25519)) { msg_err_map ("can't verify signature %s: incorrect signature", fpath); munmap (data, len); return FALSE; } b32_key = rspamd_pubkey_print (pk, RSPAMD_KEYPAIR_BASE32|RSPAMD_KEYPAIR_PUBKEY); msg_info_map ("verified signature in file %s using trusted key %v", fpath, b32_key); g_string_free (b32_key, TRUE); munmap (data, len); return TRUE; }
struct rspamd_map * rspamd_map_add (struct rspamd_config *cfg, const gchar *map_line, const gchar *description, map_cb_t read_callback, map_fin_cb_t fin_callback, void **user_data) { struct rspamd_map *map; struct rspamd_map_backend *bk; bk = rspamd_map_parse_backend (cfg, map_line); if (bk == NULL) { return NULL; } map = g_slice_alloc0 (sizeof (struct rspamd_map)); map->read_callback = read_callback; map->fin_callback = fin_callback; map->user_data = user_data; map->cfg = cfg; map->id = g_random_int (); map->locked = rspamd_mempool_alloc0_shared (cfg->cfg_pool, sizeof (gint)); map->cache = rspamd_mempool_alloc0_shared (cfg->cfg_pool, sizeof (*map->cache)); map->backends = g_ptr_array_sized_new (1); g_ptr_array_add (map->backends, bk); map->name = g_strdup (map_line); map->poll_timeout = cfg->map_timeout; if (description != NULL) { map->description = g_strdup (description); } rspamd_map_calculate_hash (map); msg_info_map ("added map %s", bk->uri); cfg->maps = g_list_prepend (cfg->maps, map); return map; }
static void rspamd_map_file_read_callback (gint fd, short what, void *ud) { struct rspamd_map *map; struct map_periodic_cbdata *periodic = ud; struct file_map_data *data; struct rspamd_map_backend *bk; map = periodic->map; bk = g_ptr_array_index (map->backends, periodic->cur_backend); data = bk->data.fd; msg_info_map ("rereading map file %s", data->filename); if (!read_map_file (map, data, bk, periodic)) { periodic->errored = TRUE; } /* Switch to the next backend */ periodic->cur_backend ++; rspamd_map_periodic_callback (-1, EV_TIMEOUT, periodic); }
static gboolean rspamd_map_read_cached (struct rspamd_map *map, struct map_periodic_cbdata *periodic, const gchar *host) { gsize len; gpointer in; in = rspamd_shmem_xmap (map->cache->shmem_name, PROT_READ, &len); if (in == NULL) { return FALSE; } if (len < map->cache->len) { munmap (in, len); return FALSE; } map->read_callback (in, map->cache->len, &periodic->cbdata, TRUE); msg_info_map ("read map data from %s (cached)", host); munmap (in, len); return TRUE; }
static int http_map_finish (struct rspamd_http_connection *conn, struct rspamd_http_message *msg) { struct http_callback_data *cbd = conn->ud; struct rspamd_map *map; struct rspamd_map_backend *bk; guchar *aux_data, *in = NULL; gsize inlen = 0, dlen = 0; map = cbd->map; bk = cbd->bk; if (msg->code == 200) { if (cbd->check) { cbd->periodic->need_modify = TRUE; /* Reset the whole chain */ cbd->periodic->cur_backend = 0; rspamd_map_periodic_callback (-1, EV_TIMEOUT, cbd->periodic); MAP_RELEASE (cbd, "http_callback_data"); return 0; } if (cbd->stage == map_load_file) { if (msg->last_modified) { cbd->data->last_checked = msg->last_modified; } else { cbd->data->last_checked = msg->date; } /* Maybe we need to check signature ? */ if (bk->is_signed) { if (bk->trusted_pubkey) { /* No need to load key */ cbd->stage = map_load_signature; cbd->pk = rspamd_pubkey_ref (bk->trusted_pubkey); } else { cbd->stage = map_load_pubkey; } cbd->shmem_data = rspamd_http_message_shmem_ref (msg); cbd->data_len = msg->body_buf.len; rspamd_http_connection_reset (cbd->conn); write_http_request (cbd); MAP_RELEASE (cbd, "http_callback_data"); return 0; } else { /* Unsinged version - just open file */ cbd->shmem_data = rspamd_http_message_shmem_ref (msg); cbd->data_len = msg->body_buf.len; goto read_data; } } else if (cbd->stage == map_load_pubkey) { /* We now can load pubkey */ cbd->shmem_pubkey = rspamd_http_message_shmem_ref (msg); cbd->pubkey_len = msg->body_buf.len; aux_data = rspamd_shmem_xmap (cbd->shmem_pubkey->shm_name, PROT_READ, &inlen); if (aux_data == NULL) { msg_err_map ("cannot map pubkey file %s: %s", cbd->shmem_pubkey->shm_name, strerror (errno)); goto err; } if (inlen < cbd->pubkey_len) { msg_err_map ("cannot map pubkey file %s: %s", cbd->shmem_pubkey->shm_name, strerror (errno)); munmap (aux_data, inlen); goto err; } cbd->pk = rspamd_pubkey_from_base32 (aux_data, cbd->pubkey_len, RSPAMD_KEYPAIR_SIGN, RSPAMD_CRYPTOBOX_MODE_25519); munmap (aux_data, inlen); if (cbd->pk == NULL) { msg_err_map ("cannot load pubkey file %s: bad pubkey", cbd->shmem_pubkey->shm_name); goto err; } cbd->stage = map_load_signature; rspamd_http_connection_reset (cbd->conn); write_http_request (cbd); MAP_RELEASE (cbd, "http_callback_data"); return 0; } else if (cbd->stage == map_load_signature) { /* We can now check signature */ cbd->shmem_sig = rspamd_http_message_shmem_ref (msg); cbd->sig_len = msg->body_buf.len; aux_data = rspamd_shmem_xmap (cbd->shmem_sig->shm_name, PROT_READ, &inlen); if (aux_data == NULL) { msg_err_map ("cannot map signature file %s: %s", cbd->shmem_sig->shm_name, strerror (errno)); goto err; } if (inlen < cbd->sig_len) { msg_err_map ("cannot map pubkey file %s: %s", cbd->shmem_pubkey->shm_name, strerror (errno)); munmap (aux_data, inlen); goto err; } in = rspamd_shmem_xmap (cbd->shmem_data->shm_name, PROT_READ, &dlen); if (in == NULL) { msg_err_map ("cannot read tempfile %s: %s", cbd->shmem_data->shm_name, strerror (errno)); munmap (aux_data, inlen); goto err; } if (!rspamd_map_check_sig_pk_mem (aux_data, cbd->sig_len, map, in, cbd->data_len, cbd->pk)) { munmap (aux_data, inlen); munmap (in, dlen); goto err; } munmap (in, dlen); } read_data: g_assert (cbd->shmem_data != NULL); in = rspamd_shmem_xmap (cbd->shmem_data->shm_name, PROT_READ, &dlen); if (in == NULL) { msg_err_map ("cannot read tempfile %s: %s", cbd->shmem_data->shm_name, strerror (errno)); goto err; } map->read_callback (in, cbd->data_len, &cbd->periodic->cbdata, TRUE); msg_info_map ("read map data from %s", cbd->data->host); /* * We know that a map is in the locked state */ if (g_atomic_int_compare_and_exchange (&map->cache->available, 0, 1)) { /* Store cached data */ struct rspamd_http_map_cached_cbdata *cache_cbd; struct timeval tv; rspamd_strlcpy (map->cache->shmem_name, cbd->shmem_data->shm_name, sizeof (map->cache->shmem_name)); map->cache->len = cbd->data_len; map->cache->last_checked = cbd->data->last_checked; cache_cbd = g_slice_alloc0 (sizeof (*cache_cbd)); cache_cbd->shm = cbd->shmem_data; cache_cbd->map = map; MAP_RETAIN (cache_cbd->shm, "shmem_data"); event_set (&cache_cbd->timeout, -1, EV_TIMEOUT, rspamd_map_cache_cb, cache_cbd); event_base_set (cbd->ev_base, &cache_cbd->timeout); double_to_tv (map->poll_timeout, &tv); event_add (&cache_cbd->timeout, &tv); } cbd->periodic->cur_backend ++; munmap (in, dlen); rspamd_map_periodic_callback (-1, EV_TIMEOUT, cbd->periodic); } else if (msg->code == 304 && (cbd->check && cbd->stage == map_load_file)) { msg_debug_map ("data is not modified for server %s", cbd->data->host); if (msg->last_modified) { cbd->data->last_checked = msg->last_modified; } else { cbd->data->last_checked = msg->date; } cbd->periodic->cur_backend ++; rspamd_map_periodic_callback (-1, EV_TIMEOUT, cbd->periodic); } else { msg_info_map ("cannot load map %s from %s: HTTP error %d", bk->uri, cbd->data->host, msg->code); } MAP_RELEASE (cbd, "http_callback_data"); return 0; err: cbd->periodic->errored = 1; rspamd_map_periodic_callback (-1, EV_TIMEOUT, cbd->periodic); MAP_RELEASE (cbd, "http_callback_data"); return 0; }
struct rspamd_map* rspamd_map_add_from_ucl (struct rspamd_config *cfg, const ucl_object_t *obj, const gchar *description, map_cb_t read_callback, map_fin_cb_t fin_callback, void **user_data) { ucl_object_iter_t it = NULL; const ucl_object_t *cur, *elt; struct rspamd_map *map; struct rspamd_map_backend *bk; g_assert (obj != NULL); if (ucl_object_type (obj) == UCL_STRING) { /* Just a plain string */ return rspamd_map_add (cfg, ucl_object_tostring (obj), NULL, read_callback, fin_callback, user_data); } map = g_slice_alloc0 (sizeof (struct rspamd_map)); map->read_callback = read_callback; map->fin_callback = fin_callback; map->user_data = user_data; map->cfg = cfg; map->id = g_random_int (); map->locked = rspamd_mempool_alloc0_shared (cfg->cfg_pool, sizeof (gint)); map->cache = rspamd_mempool_alloc0_shared (cfg->cfg_pool, sizeof (*map->cache)); map->backends = g_ptr_array_new (); map->poll_timeout = cfg->map_timeout; if (description) { map->description = g_strdup (description); } if (ucl_object_type (obj) == UCL_ARRAY) { /* Add array of maps as multiple backends */ while ((cur = ucl_object_iterate (obj, &it, true)) != NULL) { if (ucl_object_type (cur) == UCL_STRING) { bk = rspamd_map_parse_backend (cfg, ucl_object_tostring (cur)); if (bk != NULL) { g_ptr_array_add (map->backends, bk); if (!map->name) { map->name = g_strdup (ucl_object_tostring (cur)); } } } else { msg_err_config ("bad map element type: %s", ucl_object_type_to_string (ucl_object_type (cur))); } } if (map->backends->len == 0) { msg_err_config ("map has no urls to be loaded"); goto err; } } else if (ucl_object_type (obj) == UCL_OBJECT) { elt = ucl_object_lookup (obj, "name"); if (elt && ucl_object_type (elt) == UCL_STRING) { map->name = g_strdup (ucl_object_tostring (elt)); } elt = ucl_object_lookup (obj, "description"); if (elt && ucl_object_type (elt) == UCL_STRING) { if (map->description) { g_free (map->description); } map->description = g_strdup (ucl_object_tostring (elt)); } elt = ucl_object_lookup_any (obj, "timeout", "poll", "poll_time", "watch_interval", NULL); if (elt) { map->poll_timeout = ucl_object_todouble (elt); } elt = ucl_object_lookup_any (obj, "upstreams", "url", "urls", NULL); if (elt == NULL) { msg_err_config ("map has no urls to be loaded"); goto err; } if (ucl_object_type (obj) == UCL_ARRAY) { /* Add array of maps as multiple backends */ while ((cur = ucl_object_iterate (elt, &it, true)) != NULL) { if (ucl_object_type (cur) == UCL_STRING) { bk = rspamd_map_parse_backend (cfg, ucl_object_tostring (cur)); if (bk != NULL) { g_ptr_array_add (map->backends, bk); if (!map->name) { map->name = g_strdup (ucl_object_tostring (cur)); } } } else { msg_err_config ("bad map element type: %s", ucl_object_type_to_string (ucl_object_type (cur))); goto err; } } if (map->backends->len == 0) { msg_err_config ("map has no urls to be loaded"); goto err; } } else if (ucl_object_type (elt) == UCL_STRING) { bk = rspamd_map_parse_backend (cfg, ucl_object_tostring (elt)); if (bk != NULL) { g_ptr_array_add (map->backends, bk); if (!map->name) { map->name = g_strdup (ucl_object_tostring (cur)); } } } if (map->backends->len == 0) { msg_err_config ("map has no urls to be loaded"); goto err; } } rspamd_map_calculate_hash (map); msg_info_map ("added map from ucl"); cfg->maps = g_list_prepend (cfg->maps, map); return map; err: g_ptr_array_free (map->backends, TRUE); g_free (map->name); g_free (map->description); g_slice_free1 (sizeof (*map), map); return NULL; }
static int http_map_finish (struct rspamd_http_connection *conn, struct rspamd_http_message *msg) { struct http_callback_data *cbd = conn->ud; struct rspamd_map *map; struct rspamd_map_backend *bk; char fpath[PATH_MAX]; guchar *aux_data, *in = NULL; gsize inlen = 0; struct stat st; map = cbd->map; bk = cbd->bk; if (msg->code == 200) { if (cbd->check) { cbd->periodic->need_modify = TRUE; /* Reset the whole chain */ cbd->periodic->cur_backend = 0; rspamd_map_periodic_callback (-1, EV_TIMEOUT, cbd->periodic); MAP_RELEASE (cbd); return 0; } if (cbd->stage == map_load_file) { if (msg->last_modified) { cbd->data->last_checked = msg->last_modified; } else { cbd->data->last_checked = msg->date; } /* Maybe we need to check signature ? */ if (bk->is_signed) { close (cbd->out_fd); if (bk->trusted_pubkey) { /* No need to load key */ cbd->stage = map_load_signature; cbd->pk = rspamd_pubkey_ref (bk->trusted_pubkey); rspamd_snprintf (fpath, sizeof (fpath), "%s.sig", cbd->tmpfile); } else { rspamd_snprintf (fpath, sizeof (fpath), "%s.pub", cbd->tmpfile); cbd->stage = map_load_pubkey; } cbd->out_fd = rspamd_file_xopen (fpath, O_RDWR|O_CREAT, 00644); if (cbd->out_fd == -1) { msg_err_map ("cannot open pubkey file %s for writing: %s", fpath, strerror (errno)); goto err; } rspamd_http_connection_reset (cbd->conn); write_http_request (cbd); MAP_RELEASE (cbd); return 0; } else { /* Unsinged version - just open file */ in = rspamd_file_xmap (cbd->tmpfile, PROT_READ, &inlen); if (in == NULL) { msg_err_map ("cannot read tempfile %s: %s", cbd->tmpfile, strerror (errno)); goto err; } } } else if (cbd->stage == map_load_pubkey) { /* We now can load pubkey */ (void)lseek (cbd->out_fd, 0, SEEK_SET); if (fstat (cbd->out_fd, &st) == -1) { msg_err_map ("cannot stat pubkey file %s: %s", fpath, strerror (errno)); goto err; } aux_data = mmap (NULL, st.st_size, PROT_READ, MAP_SHARED, cbd->out_fd, 0); close (cbd->out_fd); cbd->out_fd = -1; if (aux_data == MAP_FAILED) { msg_err_map ("cannot map pubkey file %s: %s", fpath, strerror (errno)); goto err; } cbd->pk = rspamd_pubkey_from_base32 (aux_data, st.st_size, RSPAMD_KEYPAIR_SIGN, RSPAMD_CRYPTOBOX_MODE_25519); munmap (aux_data, st.st_size); if (cbd->pk == NULL) { msg_err_map ("cannot load pubkey file %s: bad pubkey", fpath); goto err; } rspamd_snprintf (fpath, sizeof (fpath), "%s.sig", cbd->tmpfile); cbd->out_fd = rspamd_file_xopen (fpath, O_RDWR|O_CREAT, 00644); if (cbd->out_fd == -1) { msg_err_map ("cannot open signature file %s for writing: %s", fpath, strerror (errno)); goto err; } cbd->stage = map_load_signature; rspamd_http_connection_reset (cbd->conn); write_http_request (cbd); MAP_RELEASE (cbd); return 0; } else if (cbd->stage == map_load_signature) { /* We can now check signature */ close (cbd->out_fd); cbd->out_fd = -1; in = rspamd_file_xmap (cbd->tmpfile, PROT_READ, &inlen); if (in == NULL) { msg_err_map ("cannot read tempfile %s: %s", cbd->tmpfile, strerror (errno)); goto err; } if (!rspamd_map_check_sig_pk (cbd->tmpfile, map, in, inlen, cbd->pk)) { goto err; } } g_assert (in != NULL); map->read_callback (in, inlen, &cbd->periodic->cbdata, TRUE); msg_info_map ("read map data from %s", cbd->data->host); cbd->periodic->cur_backend ++; rspamd_map_periodic_callback (-1, EV_TIMEOUT, cbd->periodic); } else if (msg->code == 304 && (cbd->check && cbd->stage == map_load_file)) { msg_debug_map ("data is not modified for server %s", cbd->data->host); if (msg->last_modified) { cbd->data->last_checked = msg->last_modified; } else { cbd->data->last_checked = msg->date; } } else { msg_info_map ("cannot load map %s from %s: HTTP error %d", bk->uri, cbd->data->host, msg->code); } MAP_RELEASE (cbd); return 0; err: cbd->periodic->errored = 1; rspamd_map_periodic_callback (-1, EV_TIMEOUT, cbd->periodic); MAP_RELEASE (cbd); return 0; }