struct symbols_cache*
rspamd_symbols_cache_new (void)
{
struct symbols_cache *cache;
cache = g_slice_alloc0 (sizeof (struct symbols_cache));
cache->static_pool =
rspamd_mempool_new (rspamd_mempool_suggest_size ());
cache->items_by_symbol = g_hash_table_new (rspamd_str_hash,
rspamd_str_equal);
cache->items_by_order = g_ptr_array_new ();
cache->items_by_id = g_ptr_array_new ();
cache->mtx = rspamd_mempool_get_mutex (cache->static_pool);
cache->reload_time = CACHE_RELOAD_TIME;
cache->total_freq = 1;
cache->max_weight = 1.0;
return cache;
}
void
rspamd_http_test_func (void)
{
struct event_base *ev_base = event_init ();
rspamd_mempool_t *pool = rspamd_mempool_new (rspamd_mempool_suggest_size ());
gpointer serv_key, client_key, peer_key;
struct rspamd_keypair_cache *c;
rspamd_mempool_mutex_t *mtx;
rspamd_inet_addr_t addr;
struct timespec ts1, ts2;
gchar filepath[PATH_MAX], buf[512];
gint fd, i, j;
pid_t sfd;
GString *b32_key;
double diff, total_diff = 0.0, latency[pconns * ntests], mean, std;
rspamd_cryptobox_init ();
rspamd_snprintf (filepath, sizeof (filepath), "/tmp/http-test-XXXXXX");
g_assert ((fd = mkstemp (filepath)) != -1);
for (i = 0; i < file_blocks; i ++) {
memset (buf, 0, sizeof (buf));
g_assert (write (fd, buf, sizeof (buf)) == sizeof (buf));
}
mtx = rspamd_mempool_get_mutex (pool);
rspamd_parse_inet_address (&addr, "127.0.0.1");
rspamd_inet_address_set_port (&addr, ottery_rand_range (30000) + 32768);
serv_key = rspamd_http_connection_gen_key ();
client_key = rspamd_http_connection_gen_key ();
c = rspamd_keypair_cache_new (16);
rspamd_mempool_lock_mutex (mtx);
sfd = fork ();
g_assert (sfd != -1);
if (sfd == 0) {
rspamd_http_server_func ("/tmp/", &addr, mtx, serv_key, c);
exit (EXIT_SUCCESS);
}
rspamd_mempool_lock_mutex (mtx);
/* Do client stuff */
for (i = 0; i < ntests; i ++) {
for (j = 0; j < pconns; j ++) {
rspamd_http_client_func (filepath + sizeof ("/tmp") - 1, &addr,
NULL, NULL, c, ev_base, &latency[i * pconns + j]);
}
clock_gettime (CLOCK_MONOTONIC, &ts1);
event_base_loop (ev_base, 0);
clock_gettime (CLOCK_MONOTONIC, &ts2);
diff = (ts2.tv_sec - ts1.tv_sec) * 1000. + /* Seconds */
(ts2.tv_nsec - ts1.tv_nsec) / 1000000.; /* Nanoseconds */
total_diff += diff;
}
msg_info ("Made %d connections of size %d in %.6f ms, %.6f cps",
ntests * pconns,
sizeof (buf) * file_blocks,
total_diff, ntests * pconns / total_diff * 1000.);
mean = rspamd_http_calculate_mean (latency, &std);
msg_info ("Latency: %.6f ms mean, %.6f dev",
mean, std);
/* Now test encrypted */
b32_key = rspamd_http_connection_print_key (serv_key,
RSPAMD_KEYPAIR_PUBKEY|RSPAMD_KEYPAIR_BASE32);
g_assert (b32_key != NULL);
peer_key = rspamd_http_connection_make_peer_key (b32_key->str);
g_assert (peer_key != NULL);
total_diff = 0.0;
for (i = 0; i < ntests; i ++) {
for (j = 0; j < pconns; j ++) {
rspamd_http_client_func (filepath + sizeof ("/tmp") - 1, &addr,
client_key, peer_key, c, ev_base, &latency[i * pconns + j]);
}
clock_gettime (CLOCK_MONOTONIC, &ts1);
event_base_loop (ev_base, 0);
clock_gettime (CLOCK_MONOTONIC, &ts2);
diff = (ts2.tv_sec - ts1.tv_sec) * 1000. + /* Seconds */
(ts2.tv_nsec - ts1.tv_nsec) / 1000000.; /* Nanoseconds */
total_diff += diff;
}
msg_info ("Made %d encrypted connections of size %d in %.6f ms, %.6f cps",
ntests * pconns,
sizeof (buf) * file_blocks,
total_diff, ntests * pconns / total_diff * 1000.);
mean = rspamd_http_calculate_mean (latency, &std);
msg_info ("Latency: %.6f ms mean, %.6f dev",
mean, std);
/* Restart server */
kill (sfd, SIGTERM);
wait (&i);
sfd = fork ();
g_assert (sfd != -1);
if (sfd == 0) {
rspamd_http_server_func ("/tmp/", &addr, mtx, serv_key, NULL);
exit (EXIT_SUCCESS);
}
rspamd_mempool_lock_mutex (mtx);
total_diff = 0.0;
for (i = 0; i < ntests; i ++) {
for (j = 0; j < pconns; j ++) {
rspamd_http_client_func (filepath + sizeof ("/tmp") - 1, &addr,
client_key, peer_key, c, ev_base, &latency[i * pconns + j]);
}
clock_gettime (CLOCK_MONOTONIC, &ts1);
event_base_loop (ev_base, 0);
clock_gettime (CLOCK_MONOTONIC, &ts2);
diff = (ts2.tv_sec - ts1.tv_sec) * 1000. + /* Seconds */
(ts2.tv_nsec - ts1.tv_nsec) / 1000000.; /* Nanoseconds */
total_diff += diff;
}
msg_info ("Made %d uncached encrypted connections of size %d in %.6f ms, %.6f cps",
ntests * pconns,
sizeof (buf) * file_blocks,
total_diff, ntests * pconns / total_diff * 1000.);
mean = rspamd_http_calculate_mean (latency, &std);
msg_info ("Latency: %.6f ms mean, %.6f dev",
mean, std);
close (fd);
unlink (filepath);
kill (sfd, SIGTERM);
}
/*
* Setup logger
*/
void
rspamd_set_logger (struct rspamd_config *cfg,
GQuark ptype,
rspamd_logger_t **plogger,
rspamd_mempool_t *pool)
{
rspamd_logger_t *logger;
if (plogger == NULL || *plogger == NULL) {
logger = g_slice_alloc0 (sizeof (rspamd_logger_t));
if (cfg->log_error_elts > 0 && pool) {
logger->errlog = rspamd_mempool_alloc0_shared (pool,
sizeof (*logger->errlog));
logger->errlog->pool = pool;
logger->errlog->max_elts = cfg->log_error_elts;
logger->errlog->elt_len = cfg->log_error_elt_maxlen;
logger->errlog->elts = rspamd_mempool_alloc0_shared (pool,
sizeof (struct rspamd_logger_error_elt) * cfg->log_error_elts +
cfg->log_error_elt_maxlen * cfg->log_error_elts);
}
if (pool) {
logger->mtx = rspamd_mempool_get_mutex (pool);
}
if (plogger) {
*plogger = logger;
}
}
else {
logger = *plogger;
}
logger->type = cfg->log_type;
logger->pid = getpid ();
logger->process_type = ptype;
switch (cfg->log_type) {
case RSPAMD_LOG_CONSOLE:
logger->log_func = file_log_function;
logger->fd = STDERR_FILENO;
break;
case RSPAMD_LOG_SYSLOG:
logger->log_func = syslog_log_function;
break;
case RSPAMD_LOG_FILE:
logger->log_func = file_log_function;
break;
}
logger->cfg = cfg;
/* Set up buffer */
if (cfg->log_buffered) {
if (cfg->log_buf_size != 0) {
logger->io_buf.size = cfg->log_buf_size;
}
else {
logger->io_buf.size = BUFSIZ;
}
logger->is_buffered = TRUE;
logger->io_buf.buf = g_malloc (logger->io_buf.size);
}
/* Set up conditional logging */
if (cfg->debug_ip_map != NULL) {
/* Try to add it as map first of all */
if (logger->debug_ip) {
radix_destroy_compressed (logger->debug_ip);
}
logger->debug_ip = NULL;
rspamd_config_radix_from_ucl (cfg,
cfg->debug_ip_map,
"IP addresses for which debug logs are enabled",
&logger->debug_ip, NULL);
}
else if (logger->debug_ip) {
radix_destroy_compressed (logger->debug_ip);
logger->debug_ip = NULL;
}
if (logger->pk) {
rspamd_pubkey_unref (logger->pk);
}
logger->pk = NULL;
if (logger->keypair) {
rspamd_keypair_unref (logger->keypair);
}
logger->keypair = NULL;
if (cfg->log_encryption_key) {
logger->pk = rspamd_pubkey_ref (cfg->log_encryption_key);
logger->keypair = rspamd_keypair_new (RSPAMD_KEYPAIR_KEX,
RSPAMD_CRYPTOBOX_MODE_25519);
rspamd_pubkey_calculate_nm (logger->pk, logger->keypair);
}
default_logger = logger;
}
*cache = pcache;
pcache->static_pool =
rspamd_mempool_new (rspamd_mempool_suggest_size ());
pcache->items_by_symbol = g_hash_table_new (rspamd_str_hash,
rspamd_str_equal);
}
item = rspamd_mempool_alloc0 (pcache->static_pool,
sizeof (struct cache_item));
item->s =
rspamd_mempool_alloc0_shared (pcache->static_pool,
sizeof (struct saved_cache_item));
item->cd = rspamd_mempool_alloc0_shared (pcache->static_pool,
sizeof (struct counter_data));
item->mtx = rspamd_mempool_get_mutex (pcache->static_pool);
rspamd_strlcpy (item->s->symbol, name, sizeof (item->s->symbol));
item->func = func;
item->user_data = user_data;
item->priority = priority;
switch (type) {
case SYMBOL_TYPE_NORMAL:
break;
case SYMBOL_TYPE_VIRTUAL:
item->is_virtual = TRUE;
break;
case SYMBOL_TYPE_CALLBACK:
item->is_callback = TRUE;
break;
void
rspamd_http_test_func (void)
{
struct event_base *ev_base = event_init ();
rspamd_mempool_t *pool = rspamd_mempool_new (rspamd_mempool_suggest_size (), NULL);
struct rspamd_cryptobox_keypair *serv_key, *client_key;
struct rspamd_cryptobox_pubkey *peer_key;
struct rspamd_keypair_cache *c;
rspamd_mempool_mutex_t *mtx;
rspamd_inet_addr_t *addr;
gdouble ts1, ts2;
gchar filepath[PATH_MAX], *buf;
gchar *env;
gint fd;
guint i, j;
pid_t *sfd;
GString *b32_key;
double diff, total_diff = 0.0, *latency, mean, std;
/* Read environment */
if ((env = getenv ("RSPAMD_HTTP_CONNS")) != NULL) {
pconns = strtoul (env, NULL, 10);
}
else {
return;
}
if ((env = getenv ("RSPAMD_HTTP_TESTS")) != NULL) {
ntests = strtoul (env, NULL, 10);
}
if ((env = getenv ("RSPAMD_HTTP_SIZE")) != NULL) {
file_size = strtoul (env, NULL, 10);
}
if ((env = getenv ("RSPAMD_HTTP_SERVERS")) != NULL) {
nservers = strtoul (env, NULL, 10);
}
rspamd_cryptobox_init ();
rspamd_snprintf (filepath, sizeof (filepath), "/tmp/http-test-XXXXXX");
g_assert ((fd = mkstemp (filepath)) != -1);
sfd = g_alloca (sizeof (*sfd) * nservers);
latency = g_malloc0 (pconns * ntests * sizeof (gdouble));
buf = g_malloc (file_size);
memset (buf, 0, file_size);
g_assert (write (fd, buf, file_size) == file_size);
g_free (buf);
mtx = rspamd_mempool_get_mutex (pool);
rspamd_parse_inet_address (&addr, "127.0.0.1", 0);
rspamd_inet_address_set_port (addr, 43898);
serv_key = rspamd_keypair_new (RSPAMD_KEYPAIR_KEX,
RSPAMD_CRYPTOBOX_MODE_25519);
client_key = rspamd_keypair_new (RSPAMD_KEYPAIR_KEX,
RSPAMD_CRYPTOBOX_MODE_25519);
c = rspamd_keypair_cache_new (16);
rspamd_http_start_servers (sfd, addr, serv_key, NULL);
usleep (100000);
/* Do client stuff */
gperf_profiler_init (NULL, "plain-http-client");
for (i = 0; i < ntests; i ++) {
for (j = 0; j < pconns; j ++) {
rspamd_http_client_func (filepath + sizeof ("/tmp") - 1, addr,
NULL, NULL, c, ev_base, &latency[i * pconns + j]);
}
ts1 = rspamd_get_ticks ();
event_base_loop (ev_base, 0);
ts2 = rspamd_get_ticks ();
diff = (ts2 - ts1) * 1000.0;
total_diff += diff;
}
gperf_profiler_stop ();
msg_info ("Made %d connections of size %d in %.6f ms, %.6f cps",
ntests * pconns,
file_size,
total_diff, ntests * pconns / total_diff * 1000.);
mean = rspamd_http_calculate_mean (latency, &std);
msg_info ("Latency: %.6f ms mean, %.6f dev",
mean, std);
rspamd_http_stop_servers (sfd);
rspamd_http_start_servers (sfd, addr, serv_key, c);
//rspamd_mempool_lock_mutex (mtx);
usleep (100000);
b32_key = rspamd_keypair_print (serv_key,
RSPAMD_KEYPAIR_PUBKEY|RSPAMD_KEYPAIR_BASE32);
g_assert (b32_key != NULL);
peer_key = rspamd_pubkey_from_base32 (b32_key->str, b32_key->len,
RSPAMD_KEYPAIR_KEX, RSPAMD_CRYPTOBOX_MODE_25519);
g_assert (peer_key != NULL);
total_diff = 0.0;
gperf_profiler_init (NULL, "cached-http-client");
for (i = 0; i < ntests; i ++) {
for (j = 0; j < pconns; j ++) {
rspamd_http_client_func (filepath + sizeof ("/tmp") - 1, addr,
client_key, peer_key, c, ev_base, &latency[i * pconns + j]);
}
ts1 = rspamd_get_ticks ();
event_base_loop (ev_base, 0);
ts2 = rspamd_get_ticks ();
diff = (ts2 - ts1) * 1000.0;
total_diff += diff;
}
gperf_profiler_stop ();
msg_info ("Made %d encrypted connections of size %d in %.6f ms, %.6f cps",
ntests * pconns,
file_size,
total_diff, ntests * pconns / total_diff * 1000.);
mean = rspamd_http_calculate_mean (latency, &std);
msg_info ("Latency: %.6f ms mean, %.6f dev",
mean, std);
/* Restart server */
rspamd_http_stop_servers (sfd);
/* No keypairs cache */
rspamd_http_start_servers (sfd, addr, serv_key, NULL);
usleep (100000);
total_diff = 0.0;
gperf_profiler_init (NULL, "fair-http-client");
for (i = 0; i < ntests; i ++) {
for (j = 0; j < pconns; j ++) {
rspamd_http_client_func (filepath + sizeof ("/tmp") - 1, addr,
client_key, peer_key, c, ev_base, &latency[i * pconns + j]);
}
ts1 = rspamd_get_ticks ();
event_base_loop (ev_base, 0);
ts2 = rspamd_get_ticks ();
diff = (ts2 - ts1) * 1000.0;
total_diff += diff;
}
gperf_profiler_stop ();
msg_info ("Made %d uncached encrypted connections of size %d in %.6f ms, %.6f cps",
ntests * pconns,
file_size,
total_diff, ntests * pconns / total_diff * 1000.);
mean = rspamd_http_calculate_mean (latency, &std);
msg_info ("Latency: %.6f ms mean, %.6f dev",
mean, std);
/* AES mode */
serv_key = rspamd_keypair_new (RSPAMD_KEYPAIR_KEX,
RSPAMD_CRYPTOBOX_MODE_NIST);
client_key = rspamd_keypair_new (RSPAMD_KEYPAIR_KEX,
RSPAMD_CRYPTOBOX_MODE_NIST);
c = rspamd_keypair_cache_new (16);
/* Restart server */
rspamd_http_stop_servers (sfd);
/* No keypairs cache */
rspamd_http_start_servers (sfd, addr, serv_key, c);
//rspamd_mempool_lock_mutex (mtx);
usleep (100000);
b32_key = rspamd_keypair_print (serv_key,
RSPAMD_KEYPAIR_PUBKEY | RSPAMD_KEYPAIR_BASE32);
g_assert (b32_key != NULL);
peer_key = rspamd_pubkey_from_base32 (b32_key->str, b32_key->len,
RSPAMD_KEYPAIR_KEX, RSPAMD_CRYPTOBOX_MODE_NIST);
g_assert (peer_key != NULL);
total_diff = 0.0;
gperf_profiler_init (NULL, "cached-http-client-aes");
for (i = 0; i < ntests; i++) {
for (j = 0; j < pconns; j++) {
rspamd_http_client_func (filepath + sizeof ("/tmp") - 1,
addr,
client_key,
peer_key,
NULL,
ev_base,
&latency[i * pconns + j]);
}
ts1 = rspamd_get_ticks ();
event_base_loop (ev_base, 0);
ts2 = rspamd_get_ticks ();
diff = (ts2 - ts1) * 1000.0;
total_diff += diff;
}
gperf_profiler_stop ();
msg_info (
"Made %d aes encrypted connections of size %d in %.6f ms, %.6f cps",
ntests * pconns,
file_size,
total_diff,
ntests * pconns / total_diff * 1000.);
mean = rspamd_http_calculate_mean (latency, &std);
msg_info ("Latency: %.6f ms mean, %.6f dev",
mean, std);
/* Restart server */
rspamd_http_stop_servers (sfd);
/* No keypairs cache */
rspamd_http_start_servers (sfd, addr, serv_key, NULL);
//rspamd_mempool_lock_mutex (mtx);
usleep (100000);
total_diff = 0.0;
gperf_profiler_init (NULL, "fair-http-client-aes");
for (i = 0; i < ntests; i++) {
for (j = 0; j < pconns; j++) {
rspamd_http_client_func (filepath + sizeof ("/tmp") - 1,
addr,
client_key,
peer_key,
c,
ev_base,
&latency[i * pconns + j]);
}
ts1 = rspamd_get_ticks ();
event_base_loop (ev_base, 0);
ts2 = rspamd_get_ticks ();
diff = (ts2 - ts1) * 1000.0;
total_diff += diff;
}
gperf_profiler_stop ();
msg_info (
"Made %d uncached aes encrypted connections of size %d in %.6f ms, %.6f cps",
ntests * pconns,
file_size,
total_diff,
ntests * pconns / total_diff * 1000.);
mean = rspamd_http_calculate_mean (latency, &std);
msg_info ("Latency: %.6f ms mean, %.6f dev",
mean, std);
close (fd);
unlink (filepath);
rspamd_http_stop_servers (sfd);
}
/*
* Setup logger
*/
void
rspamd_set_logger (struct rspamd_config *cfg,
GQuark ptype,
struct rspamd_main *rspamd)
{
if (rspamd->logger == NULL) {
rspamd->logger = g_malloc (sizeof (rspamd_logger_t));
memset (rspamd->logger, 0, sizeof (rspamd_logger_t));
/* Small pool for interlocking */
rspamd->logger->pool = rspamd_mempool_new (512, NULL);
rspamd->logger->mtx = rspamd_mempool_get_mutex (rspamd->logger->pool);
}
rspamd->logger->type = cfg->log_type;
rspamd->logger->pid = getpid ();
rspamd->logger->process_type = ptype;
switch (cfg->log_type) {
case RSPAMD_LOG_CONSOLE:
rspamd->logger->log_func = file_log_function;
rspamd->logger->fd = STDERR_FILENO;
break;
case RSPAMD_LOG_SYSLOG:
rspamd->logger->log_func = syslog_log_function;
break;
case RSPAMD_LOG_FILE:
rspamd->logger->log_func = file_log_function;
break;
}
rspamd->logger->cfg = cfg;
/* Set up buffer */
if (rspamd->cfg->log_buffered) {
if (rspamd->cfg->log_buf_size != 0) {
rspamd->logger->io_buf.size = rspamd->cfg->log_buf_size;
}
else {
rspamd->logger->io_buf.size = BUFSIZ;
}
rspamd->logger->is_buffered = TRUE;
rspamd->logger->io_buf.buf = g_malloc (rspamd->logger->io_buf.size);
}
/* Set up conditional logging */
if (rspamd->cfg->debug_ip_map != NULL) {
/* Try to add it as map first of all */
if (rspamd->logger->debug_ip) {
radix_destroy_compressed (rspamd->logger->debug_ip);
}
rspamd->logger->debug_ip = radix_create_compressed ();
if (!rspamd_map_add (rspamd->cfg, rspamd->cfg->debug_ip_map,
"IP addresses for which debug logs are enabled",
rspamd_radix_read, rspamd_radix_fin,
(void **) &rspamd->logger->debug_ip)) {
radix_add_generic_iplist (rspamd->cfg->debug_ip_map,
&rspamd->logger->debug_ip);
}
}
else if (rspamd->logger->debug_ip) {
radix_destroy_compressed (rspamd->logger->debug_ip);
rspamd->logger->debug_ip = NULL;
}
default_logger = rspamd->logger;
}
