static guint
rspamd_re_cache_process_pcre (struct rspamd_re_runtime *rt,
rspamd_regexp_t *re, rspamd_mempool_t *pool,
const guchar *in, gsize len,
gboolean is_raw)
{
guint r = 0;
const gchar *start = NULL, *end = NULL;
guint max_hits = rspamd_regexp_get_maxhits (re);
guint64 id = rspamd_regexp_get_cache_id (re);
gdouble t1, t2;
const gdouble slow_time = 0.1;
if (len == 0) {
len = strlen (in);
}
if (rt->cache->max_re_data > 0 && len > rt->cache->max_re_data) {
len = rt->cache->max_re_data;
}
r = rt->results[id];
t1 = rspamd_get_ticks ();
if (max_hits == 0 || r < max_hits) {
while (rspamd_regexp_search (re,
in,
len,
&start,
&end,
is_raw,
NULL)) {
r++;
if (max_hits > 0 && r >= max_hits) {
break;
}
}
rt->stat.regexp_checked++;
rt->stat.bytes_scanned_pcre += len;
rt->stat.bytes_scanned += len;
if (r > 0) {
rt->stat.regexp_matched += r;
}
}
t2 = rspamd_get_ticks ();
if (t2 - t1 > slow_time) {
msg_info_pool ("regexp '%16s' took %.2f seconds to execute",
rspamd_regexp_get_pattern (re), t2 - t1);
}
return r;
}
static gboolean
rspamd_symbols_cache_check_symbol (struct rspamd_task *task,
struct symbols_cache *cache,
struct cache_item *item,
struct cache_savepoint *checkpoint)
{
guint pending_before, pending_after;
double t1, t2;
guint64 diff;
if (item->type == SYMBOL_TYPE_NORMAL || item->type == SYMBOL_TYPE_CALLBACK) {
g_assert (item->func != NULL);
/* Check has been started */
setbit (checkpoint->processed_bits, item->id * 2);
t1 = rspamd_get_ticks ();
pending_before = rspamd_session_events_pending (task->s);
/* Watch for events appeared */
rspamd_session_watch_start (task->s, rspamd_symbols_cache_watcher_cb,
item);
if (item->symbol != NULL &&
G_UNLIKELY (check_debug_symbol (task->cfg, item->symbol))) {
rspamd_log_debug (rspamd_main->logger);
item->func (task, item->user_data);
rspamd_log_nodebug (rspamd_main->logger);
}
else {
item->func (task, item->user_data);
}
t2 = rspamd_get_ticks ();
diff = (t2 - t1) * 1000000;
rspamd_set_counter (item, diff);
rspamd_session_watch_stop (task->s);
pending_after = rspamd_session_events_pending (task->s);
if (pending_before == pending_after) {
/* No new events registered */
setbit (checkpoint->processed_bits, item->id * 2 + 1);
return TRUE;
}
return FALSE;
}
else {
setbit (checkpoint->processed_bits, item->id * 2);
setbit (checkpoint->processed_bits, item->id * 2 + 1);
return TRUE;
}
}
static gint
rspamd_client_finish (struct rspamd_http_connection *conn,
struct rspamd_http_message *msg)
{
struct client_cbdata *cb = conn->ud;
*(cb->lat) = rspamd_get_ticks () * 1000. - cb->ts;
close (conn->fd);
rspamd_http_connection_unref (conn);
g_free (cb);
return 0;
}
static void
rspamd_http_client_func (const gchar *path, rspamd_inet_addr_t *addr,
struct rspamd_cryptobox_keypair *kp,
struct rspamd_cryptobox_pubkey *peer_kp,
struct rspamd_keypair_cache *c,
struct event_base *ev_base, double *latency)
{
struct rspamd_http_message *msg;
struct rspamd_http_connection *conn;
gchar urlbuf[PATH_MAX];
struct client_cbdata *cb;
gint fd;
g_assert ((fd = rspamd_inet_address_connect (addr, SOCK_STREAM, TRUE)) != -1);
conn = rspamd_http_connection_new (rspamd_client_body,
rspamd_client_err,
rspamd_client_finish,
RSPAMD_HTTP_CLIENT_SIMPLE,
RSPAMD_HTTP_CLIENT,
c,
NULL);
rspamd_snprintf (urlbuf, sizeof (urlbuf), "http://127.0.0.1/%s", path);
msg = rspamd_http_message_from_url (urlbuf);
g_assert (conn != NULL && msg != NULL);
if (kp != NULL) {
g_assert (peer_kp != NULL);
rspamd_http_connection_set_key (conn, kp);
msg->peer_key = rspamd_pubkey_ref (peer_kp);
}
cb = g_malloc (sizeof (*cb));
cb->ts = rspamd_get_ticks () * 1000.;
cb->lat = latency;
rspamd_http_connection_write_message (conn, msg, NULL, NULL, cb,
fd, NULL, ev_base);
}
void
rspamd_radix_test_func (void)
{
#if 0
radix_tree_t *tree = radix_tree_create ();
#endif
radix_compressed_t *comp_tree = radix_create_compressed ();
struct {
guint32 addr;
guint32 mask;
guint8 addr6[16];
guint32 mask6;
} *addrs;
gsize nelts, i;
gint lc;
gboolean all_good = TRUE;
gdouble ts1, ts2;
double diff;
/* Test suite for the compressed trie */
rspamd_radix_text_vec ();
nelts = max_elts;
/* First of all we generate many elements and push them to the array */
addrs = g_malloc (nelts * sizeof (addrs[0]));
for (i = 0; i < nelts; i ++) {
addrs[i].addr = ottery_rand_uint32 ();
addrs[i].mask = masks[ottery_rand_range(G_N_ELEMENTS (masks) - 1)];
ottery_rand_bytes (addrs[i].addr6, sizeof(addrs[i].addr6));
addrs[i].mask6 = ottery_rand_range(128);
}
#if 0
msg_info ("old radix performance (%z elts)", nelts);
ts1 = rspamd_get_ticks ();
for (i = 0; i < nelts; i ++) {
guint32 mask = G_MAXUINT32 << (32 - addrs[i].mask);
radix32tree_insert (tree, addrs[i].addr, mask, 1);
}
ts2 = rspamd_get_ticks ();
diff = (ts2 - ts1) * 1000.0;
msg_info ("Added %z elements in %.6f ms", nelts, diff);
ts1 = rspamd_get_ticks ();
for (lc = 0; lc < lookup_cycles; lc ++) {
for (i = 0; i < nelts; i ++) {
g_assert (radix32tree_find (tree, addrs[i].addr) != RADIX_NO_VALUE);
}
}
ts2 = rspamd_get_ticks ();
diff = (ts2 - ts1) * 1000.0;
msg_info ("Checked %z elements in %.6f ms", nelts, diff);
ts1 = rspamd_get_ticks ();
for (i = 0; i < nelts; i ++) {
radix32tree_delete (tree, addrs[i].addr, addrs[i].mask);
}
ts2 = rspamd_get_ticks ();
diff = (ts2 - ts1) * 1000.;
msg_info ("Deleted %z elements in %.6f ms", nelts, diff);
radix_tree_free (tree);
#endif
msg_info ("new radix performance (%z elts)", nelts);
ts1 = rspamd_get_ticks ();
for (i = 0; i < nelts; i ++) {
radix_insert_compressed (comp_tree, addrs[i].addr6, sizeof (addrs[i].addr6),
128 - addrs[i].mask6, i);
}
ts2 = rspamd_get_ticks ();
diff = (ts2 - ts1) * 1000.0;
msg_info ("Added %z elements in %.6f ms", nelts, diff);
ts1 = rspamd_get_ticks ();
for (lc = 0; lc < lookup_cycles; lc ++) {
for (i = 0; i < nelts; i ++) {
if (radix_find_compressed (comp_tree, addrs[i].addr6, sizeof (addrs[i].addr6))
== RADIX_NO_VALUE) {
all_good = FALSE;
}
}
}
#if 1
if (!all_good) {
for (i = 0; i < nelts; i ++) {
/* Used to write bad random vector */
char ipbuf[INET6_ADDRSTRLEN + 1];
inet_ntop(AF_INET6, addrs[i].addr6, ipbuf, sizeof(ipbuf));
msg_info("{\"%s\", NULL, \"%ud\", 0, 0, 0, 0},",
ipbuf,
addrs[i].mask6);
}
}
#endif
g_assert (all_good);
ts2 = rspamd_get_ticks ();
diff = (ts2 - ts1) * 1000.0;
msg_info ("Checked %z elements in %.6f ms", nelts, diff);
radix_destroy_compressed (comp_tree);
g_free (addrs);
}
void
rspamd_cryptobox_test_func (void)
{
void *map;
guchar *begin, *end;
rspamd_nm_t key;
rspamd_nonce_t nonce;
rspamd_sig_t mac;
struct rspamd_cryptobox_segment *seg;
double t1, t2;
gint i, cnt, ms;
map = create_mapping (mapping_size, &begin, &end);
ottery_rand_bytes (key, sizeof (key));
ottery_rand_bytes (nonce, sizeof (nonce));
memset (mac, 0, sizeof (mac));
seg = g_slice_alloc0 (sizeof (*seg) * max_seg * 10);
/* Test baseline */
t1 = rspamd_get_ticks ();
rspamd_cryptobox_encrypt_nm_inplace (begin, end - begin, nonce, key, mac);
t2 = rspamd_get_ticks ();
check_result (key, nonce, mac, begin, end);
msg_info ("baseline encryption: %.6f", t2 - t1);
/* A single chunk as vector */
seg[0].data = begin;
seg[0].len = end - begin;
t1 = rspamd_get_ticks ();
rspamd_cryptobox_encryptv_nm_inplace (seg, 1, nonce, key, mac);
t2 = rspamd_get_ticks ();
check_result (key, nonce, mac, begin, end);
msg_info ("bulk encryption: %.6f", t2 - t1);
/* Two chunks as vector */
seg[0].data = begin;
seg[0].len = (end - begin) / 2;
seg[1].data = begin + seg[0].len;
seg[1].len = (end - begin) - seg[0].len;
t1 = rspamd_get_ticks ();
rspamd_cryptobox_encryptv_nm_inplace (seg, 2, nonce, key, mac);
t2 = rspamd_get_ticks ();
check_result (key, nonce, mac, begin, end);
msg_info ("2 equal chunks encryption: %.6f", t2 - t1);
seg[0].data = begin;
seg[0].len = 1;
seg[1].data = begin + seg[0].len;
seg[1].len = (end - begin) - seg[0].len;
t1 = rspamd_get_ticks ();
rspamd_cryptobox_encryptv_nm_inplace (seg, 2, nonce, key, mac);
t2 = rspamd_get_ticks ();
check_result (key, nonce, mac, begin, end);
msg_info ("small and large chunks encryption: %.6f", t2 - t1);
seg[0].data = begin;
seg[0].len = (end - begin) - 3;
seg[1].data = begin + seg[0].len;
seg[1].len = (end - begin) - seg[0].len;
t1 = rspamd_get_ticks ();
rspamd_cryptobox_encryptv_nm_inplace (seg, 2, nonce, key, mac);
t2 = rspamd_get_ticks ();
check_result (key, nonce, mac, begin, end);
msg_info ("large and small chunks encryption: %.6f", t2 - t1);
/* Random two chunks as vector */
seg[0].data = begin;
seg[0].len = ottery_rand_range (end - begin - 1) + 1;
seg[1].data = begin + seg[0].len;
seg[1].len = (end - begin) - seg[0].len;
t1 = rspamd_get_ticks ();
rspamd_cryptobox_encryptv_nm_inplace (seg, 2, nonce, key, mac);
t2 = rspamd_get_ticks ();
check_result (key, nonce, mac, begin, end);
msg_info ("random 2 chunks encryption: %.6f", t2 - t1);
/* 3 specific chunks */
seg[0].data = begin;
seg[0].len = 2;
seg[1].data = begin + seg[0].len;
seg[1].len = 2049;
seg[2].data = begin + seg[0].len + seg[1].len;
seg[2].len = (end - begin) - seg[0].len - seg[1].len;
t1 = rspamd_get_ticks ();
rspamd_cryptobox_encryptv_nm_inplace (seg, 3, nonce, key, mac);
t2 = rspamd_get_ticks ();
check_result (key, nonce, mac, begin, end);
msg_info ("small, medium and large chunks encryption: %.6f", t2 - t1);
cnt = create_random_split (seg, max_seg, begin, end);
t1 = rspamd_get_ticks ();
rspamd_cryptobox_encryptv_nm_inplace (seg, cnt, nonce, key, mac);
t2 = rspamd_get_ticks ();
check_result (key, nonce, mac, begin, end);
msg_info ("random split of %d chunks encryption: %.6f", cnt, t2 - t1);
cnt = create_realistic_split (seg, max_seg, begin, end);
t1 = rspamd_get_ticks ();
rspamd_cryptobox_encryptv_nm_inplace (seg, cnt, nonce, key, mac);
t2 = rspamd_get_ticks ();
check_result (key, nonce, mac, begin, end);
msg_info ("realistic split of %d chunks encryption: %.6f", cnt, t2 - t1);
cnt = create_constrainted_split (seg, max_seg + 1, 32, begin, end);
t1 = rspamd_get_ticks ();
rspamd_cryptobox_encryptv_nm_inplace (seg, cnt, nonce, key, mac);
t2 = rspamd_get_ticks ();
check_result (key, nonce, mac, begin, end);
msg_info ("constrainted split of %d chunks encryption: %.6f", cnt, t2 - t1);
for (i = 0; i < random_fuzz_cnt; i ++) {
ms = ottery_rand_range (i % max_seg * 2) + 1;
cnt = create_random_split (seg, ms, begin, end);
t1 = rspamd_get_ticks ();
rspamd_cryptobox_encryptv_nm_inplace (seg, cnt, nonce, key, mac);
t2 = rspamd_get_ticks ();
check_result (key, nonce, mac, begin, end);
if (i % 1000 == 0) {
msg_info ("random fuzz iterations: %d", i);
}
}
for (i = 0; i < random_fuzz_cnt; i ++) {
ms = ottery_rand_range (i % max_seg * 2) + 1;
cnt = create_realistic_split (seg, ms, begin, end);
t1 = rspamd_get_ticks ();
rspamd_cryptobox_encryptv_nm_inplace (seg, cnt, nonce, key, mac);
t2 = rspamd_get_ticks ();
check_result (key, nonce, mac, begin, end);
if (i % 1000 == 0) {
msg_info ("realistic fuzz iterations: %d", i);
}
}
for (i = 0; i < random_fuzz_cnt; i ++) {
ms = ottery_rand_range (i % max_seg * 10) + 1;
cnt = create_constrainted_split (seg, ms, i, begin, end);
t1 = rspamd_get_ticks ();
rspamd_cryptobox_encryptv_nm_inplace (seg, cnt, nonce, key, mac);
t2 = rspamd_get_ticks ();
check_result (key, nonce, mac, begin, end);
if (i % 1000 == 0) {
msg_info ("constrainted fuzz iterations: %d", i);
}
}
}
/*
* Create new task
*/
struct rspamd_task *
rspamd_task_new (struct rspamd_worker *worker)
{
struct rspamd_task *new_task;
new_task = g_slice_alloc0 (sizeof (struct rspamd_task));
new_task->worker = worker;
if (worker) {
new_task->cfg = worker->srv->cfg;
if (new_task->cfg->check_all_filters) {
new_task->flags |= RSPAMD_TASK_FLAG_PASS_ALL;
}
}
gettimeofday (&new_task->tv, NULL);
new_task->time_real = rspamd_get_ticks ();
new_task->time_virtual = rspamd_get_virtual_ticks ();
new_task->task_pool = rspamd_mempool_new (rspamd_mempool_suggest_size (), "task");
new_task->results = g_hash_table_new (rspamd_str_hash, rspamd_str_equal);
rspamd_mempool_add_destructor (new_task->task_pool,
(rspamd_mempool_destruct_t) g_hash_table_unref,
new_task->results);
new_task->re_cache = g_hash_table_new (rspamd_str_hash, rspamd_str_equal);
rspamd_mempool_add_destructor (new_task->task_pool,
(rspamd_mempool_destruct_t) g_hash_table_unref,
new_task->re_cache);
new_task->raw_headers = g_hash_table_new (rspamd_strcase_hash,
rspamd_strcase_equal);
new_task->request_headers = g_hash_table_new_full (rspamd_gstring_icase_hash,
rspamd_gstring_icase_equal, rspamd_gstring_free_hard,
rspamd_gstring_free_hard);
rspamd_mempool_add_destructor (new_task->task_pool,
(rspamd_mempool_destruct_t) g_hash_table_unref,
new_task->request_headers);
new_task->reply_headers = g_hash_table_new_full (rspamd_gstring_icase_hash,
rspamd_gstring_icase_equal, rspamd_gstring_free_hard,
rspamd_gstring_free_hard);
rspamd_mempool_add_destructor (new_task->task_pool,
(rspamd_mempool_destruct_t) g_hash_table_unref,
new_task->reply_headers);
rspamd_mempool_add_destructor (new_task->task_pool,
(rspamd_mempool_destruct_t) g_hash_table_unref,
new_task->raw_headers);
new_task->emails = g_hash_table_new (rspamd_url_hash, rspamd_emails_cmp);
rspamd_mempool_add_destructor (new_task->task_pool,
(rspamd_mempool_destruct_t) g_hash_table_unref,
new_task->emails);
new_task->urls = g_hash_table_new (rspamd_url_hash, rspamd_urls_cmp);
rspamd_mempool_add_destructor (new_task->task_pool,
(rspamd_mempool_destruct_t) g_hash_table_unref,
new_task->urls);
new_task->parts = g_ptr_array_sized_new (4);
rspamd_mempool_add_destructor (new_task->task_pool,
rspamd_ptr_array_free_hard, new_task->parts);
new_task->text_parts = g_ptr_array_sized_new (2);
rspamd_mempool_add_destructor (new_task->task_pool,
rspamd_ptr_array_free_hard, new_task->text_parts);
new_task->received = g_ptr_array_sized_new (8);
rspamd_mempool_add_destructor (new_task->task_pool,
rspamd_ptr_array_free_hard, new_task->received);
new_task->sock = -1;
new_task->flags |= (RSPAMD_TASK_FLAG_MIME|RSPAMD_TASK_FLAG_JSON);
new_task->pre_result.action = METRIC_ACTION_NOACTION;
new_task->message_id = new_task->queue_id = "undef";
return new_task;
}
void
rspamd_http_test_func (void)
{
struct event_base *ev_base = event_init ();
rspamd_mempool_t *pool = rspamd_mempool_new (rspamd_mempool_suggest_size (), NULL);
struct rspamd_cryptobox_keypair *serv_key, *client_key;
struct rspamd_cryptobox_pubkey *peer_key;
struct rspamd_keypair_cache *c;
rspamd_mempool_mutex_t *mtx;
rspamd_inet_addr_t *addr;
gdouble ts1, ts2;
gchar filepath[PATH_MAX], *buf;
gchar *env;
gint fd;
guint i, j;
pid_t *sfd;
GString *b32_key;
double diff, total_diff = 0.0, *latency, mean, std;
/* Read environment */
if ((env = getenv ("RSPAMD_HTTP_CONNS")) != NULL) {
pconns = strtoul (env, NULL, 10);
}
else {
return;
}
if ((env = getenv ("RSPAMD_HTTP_TESTS")) != NULL) {
ntests = strtoul (env, NULL, 10);
}
if ((env = getenv ("RSPAMD_HTTP_SIZE")) != NULL) {
file_size = strtoul (env, NULL, 10);
}
if ((env = getenv ("RSPAMD_HTTP_SERVERS")) != NULL) {
nservers = strtoul (env, NULL, 10);
}
rspamd_cryptobox_init ();
rspamd_snprintf (filepath, sizeof (filepath), "/tmp/http-test-XXXXXX");
g_assert ((fd = mkstemp (filepath)) != -1);
sfd = g_alloca (sizeof (*sfd) * nservers);
latency = g_malloc0 (pconns * ntests * sizeof (gdouble));
buf = g_malloc (file_size);
memset (buf, 0, file_size);
g_assert (write (fd, buf, file_size) == file_size);
g_free (buf);
mtx = rspamd_mempool_get_mutex (pool);
rspamd_parse_inet_address (&addr, "127.0.0.1", 0);
rspamd_inet_address_set_port (addr, 43898);
serv_key = rspamd_keypair_new (RSPAMD_KEYPAIR_KEX,
RSPAMD_CRYPTOBOX_MODE_25519);
client_key = rspamd_keypair_new (RSPAMD_KEYPAIR_KEX,
RSPAMD_CRYPTOBOX_MODE_25519);
c = rspamd_keypair_cache_new (16);
rspamd_http_start_servers (sfd, addr, serv_key, NULL);
usleep (100000);
/* Do client stuff */
gperf_profiler_init (NULL, "plain-http-client");
for (i = 0; i < ntests; i ++) {
for (j = 0; j < pconns; j ++) {
rspamd_http_client_func (filepath + sizeof ("/tmp") - 1, addr,
NULL, NULL, c, ev_base, &latency[i * pconns + j]);
}
ts1 = rspamd_get_ticks ();
event_base_loop (ev_base, 0);
ts2 = rspamd_get_ticks ();
diff = (ts2 - ts1) * 1000.0;
total_diff += diff;
}
gperf_profiler_stop ();
msg_info ("Made %d connections of size %d in %.6f ms, %.6f cps",
ntests * pconns,
file_size,
total_diff, ntests * pconns / total_diff * 1000.);
mean = rspamd_http_calculate_mean (latency, &std);
msg_info ("Latency: %.6f ms mean, %.6f dev",
mean, std);
rspamd_http_stop_servers (sfd);
rspamd_http_start_servers (sfd, addr, serv_key, c);
//rspamd_mempool_lock_mutex (mtx);
usleep (100000);
b32_key = rspamd_keypair_print (serv_key,
RSPAMD_KEYPAIR_PUBKEY|RSPAMD_KEYPAIR_BASE32);
g_assert (b32_key != NULL);
peer_key = rspamd_pubkey_from_base32 (b32_key->str, b32_key->len,
RSPAMD_KEYPAIR_KEX, RSPAMD_CRYPTOBOX_MODE_25519);
g_assert (peer_key != NULL);
total_diff = 0.0;
gperf_profiler_init (NULL, "cached-http-client");
for (i = 0; i < ntests; i ++) {
for (j = 0; j < pconns; j ++) {
rspamd_http_client_func (filepath + sizeof ("/tmp") - 1, addr,
client_key, peer_key, c, ev_base, &latency[i * pconns + j]);
}
ts1 = rspamd_get_ticks ();
event_base_loop (ev_base, 0);
ts2 = rspamd_get_ticks ();
diff = (ts2 - ts1) * 1000.0;
total_diff += diff;
}
gperf_profiler_stop ();
msg_info ("Made %d encrypted connections of size %d in %.6f ms, %.6f cps",
ntests * pconns,
file_size,
total_diff, ntests * pconns / total_diff * 1000.);
mean = rspamd_http_calculate_mean (latency, &std);
msg_info ("Latency: %.6f ms mean, %.6f dev",
mean, std);
/* Restart server */
rspamd_http_stop_servers (sfd);
/* No keypairs cache */
rspamd_http_start_servers (sfd, addr, serv_key, NULL);
usleep (100000);
total_diff = 0.0;
gperf_profiler_init (NULL, "fair-http-client");
for (i = 0; i < ntests; i ++) {
for (j = 0; j < pconns; j ++) {
rspamd_http_client_func (filepath + sizeof ("/tmp") - 1, addr,
client_key, peer_key, c, ev_base, &latency[i * pconns + j]);
}
ts1 = rspamd_get_ticks ();
event_base_loop (ev_base, 0);
ts2 = rspamd_get_ticks ();
diff = (ts2 - ts1) * 1000.0;
total_diff += diff;
}
gperf_profiler_stop ();
msg_info ("Made %d uncached encrypted connections of size %d in %.6f ms, %.6f cps",
ntests * pconns,
file_size,
total_diff, ntests * pconns / total_diff * 1000.);
mean = rspamd_http_calculate_mean (latency, &std);
msg_info ("Latency: %.6f ms mean, %.6f dev",
mean, std);
/* AES mode */
serv_key = rspamd_keypair_new (RSPAMD_KEYPAIR_KEX,
RSPAMD_CRYPTOBOX_MODE_NIST);
client_key = rspamd_keypair_new (RSPAMD_KEYPAIR_KEX,
RSPAMD_CRYPTOBOX_MODE_NIST);
c = rspamd_keypair_cache_new (16);
/* Restart server */
rspamd_http_stop_servers (sfd);
/* No keypairs cache */
rspamd_http_start_servers (sfd, addr, serv_key, c);
//rspamd_mempool_lock_mutex (mtx);
usleep (100000);
b32_key = rspamd_keypair_print (serv_key,
RSPAMD_KEYPAIR_PUBKEY | RSPAMD_KEYPAIR_BASE32);
g_assert (b32_key != NULL);
peer_key = rspamd_pubkey_from_base32 (b32_key->str, b32_key->len,
RSPAMD_KEYPAIR_KEX, RSPAMD_CRYPTOBOX_MODE_NIST);
g_assert (peer_key != NULL);
total_diff = 0.0;
gperf_profiler_init (NULL, "cached-http-client-aes");
for (i = 0; i < ntests; i++) {
for (j = 0; j < pconns; j++) {
rspamd_http_client_func (filepath + sizeof ("/tmp") - 1,
addr,
client_key,
peer_key,
NULL,
ev_base,
&latency[i * pconns + j]);
}
ts1 = rspamd_get_ticks ();
event_base_loop (ev_base, 0);
ts2 = rspamd_get_ticks ();
diff = (ts2 - ts1) * 1000.0;
total_diff += diff;
}
gperf_profiler_stop ();
msg_info (
"Made %d aes encrypted connections of size %d in %.6f ms, %.6f cps",
ntests * pconns,
file_size,
total_diff,
ntests * pconns / total_diff * 1000.);
mean = rspamd_http_calculate_mean (latency, &std);
msg_info ("Latency: %.6f ms mean, %.6f dev",
mean, std);
/* Restart server */
rspamd_http_stop_servers (sfd);
/* No keypairs cache */
rspamd_http_start_servers (sfd, addr, serv_key, NULL);
//rspamd_mempool_lock_mutex (mtx);
usleep (100000);
total_diff = 0.0;
gperf_profiler_init (NULL, "fair-http-client-aes");
for (i = 0; i < ntests; i++) {
for (j = 0; j < pconns; j++) {
rspamd_http_client_func (filepath + sizeof ("/tmp") - 1,
addr,
client_key,
peer_key,
c,
ev_base,
&latency[i * pconns + j]);
}
ts1 = rspamd_get_ticks ();
event_base_loop (ev_base, 0);
ts2 = rspamd_get_ticks ();
diff = (ts2 - ts1) * 1000.0;
total_diff += diff;
}
gperf_profiler_stop ();
msg_info (
"Made %d uncached aes encrypted connections of size %d in %.6f ms, %.6f cps",
ntests * pconns,
file_size,
total_diff,
ntests * pconns / total_diff * 1000.);
mean = rspamd_http_calculate_mean (latency, &std);
msg_info ("Latency: %.6f ms mean, %.6f dev",
mean, std);
close (fd);
unlink (filepath);
rspamd_http_stop_servers (sfd);
}
static gboolean
rspamd_symbols_cache_check_symbol (struct rspamd_task *task,
struct symbols_cache *cache,
struct cache_item *item,
struct cache_savepoint *checkpoint,
gdouble *total_diff)
{
guint pending_before, pending_after;
double t1, t2;
gdouble diff;
struct rspamd_task **ptask;
lua_State *L;
gboolean check = TRUE;
const gdouble slow_diff_limit = 1e5;
if (item->type & (SYMBOL_TYPE_NORMAL|SYMBOL_TYPE_CALLBACK)) {
g_assert (item->func != NULL);
/* Check has been started */
setbit (checkpoint->processed_bits, item->id * 2);
if (RSPAMD_TASK_IS_EMPTY (task) && !(item->type & SYMBOL_TYPE_EMPTY)) {
check = FALSE;
}
else if (item->condition_cb != -1) {
/* We also executes condition callback to check if we need this symbol */
L = task->cfg->lua_state;
lua_rawgeti (L, LUA_REGISTRYINDEX, item->condition_cb);
ptask = lua_newuserdata (L, sizeof (struct rspamd_task *));
rspamd_lua_setclass (L, "rspamd{task}", -1);
*ptask = task;
if (lua_pcall (L, 1, 1, 0) != 0) {
msg_info_task ("call to condition for %s failed: %s",
item->symbol, lua_tostring (L, -1));
lua_pop (L, 1);
}
else {
check = lua_toboolean (L, -1);
lua_pop (L, 1);
}
}
if (check) {
t1 = rspamd_get_ticks ();
pending_before = rspamd_session_events_pending (task->s);
/* Watch for events appeared */
rspamd_session_watch_start (task->s, rspamd_symbols_cache_watcher_cb,
item);
msg_debug_task ("execute %s, %d", item->symbol, item->id);
item->func (task, item->user_data);
t2 = rspamd_get_ticks ();
diff = (t2 - t1) * 1e6;
if (total_diff) {
*total_diff += diff;
}
if (diff > slow_diff_limit) {
msg_info_task ("slow rule: %s: %d ms", item->symbol,
(gint)(diff / 1000.));
}
rspamd_set_counter (item, diff);
rspamd_session_watch_stop (task->s);
pending_after = rspamd_session_events_pending (task->s);
if (pending_before == pending_after) {
/* No new events registered */
setbit (checkpoint->processed_bits, item->id * 2 + 1);
return TRUE;
}
return FALSE;
}
else {
msg_debug_task ("skipping check of %s as its condition is false",
item->symbol);
setbit (checkpoint->processed_bits, item->id * 2 + 1);
return TRUE;
}
}
else {
setbit (checkpoint->processed_bits, item->id * 2);
setbit (checkpoint->processed_bits, item->id * 2 + 1);
return TRUE;
}
}
static void
rspamd_process_file (struct rspamd_config *cfg, const gchar *fname, gint mode)
{
struct rspamd_task *task;
gint fd;
gpointer map;
struct stat st;
GError *err = NULL;
#if 0
GMimeMessage *message;
GMimeParser *parser;
GMimeStream *stream;
GByteArray tmp;
#endif
struct rspamd_mime_part *part;
guint i;
gdouble ts1, ts2;
fd = open (fname, O_RDONLY);
if (fd == -1) {
rspamd_fprintf (stderr, "cannot open %s: %s\n", fname, strerror (errno));
exit (EXIT_FAILURE);
}
if (fstat (fd, &st) == -1) {
rspamd_fprintf (stderr, "cannot stat %s: %s\n", fname, strerror (errno));
exit (EXIT_FAILURE);
}
map = mmap (NULL, st.st_size, PROT_READ, MAP_SHARED, fd, 0);
close (fd);
if (map == MAP_FAILED) {
rspamd_fprintf (stderr, "cannot mmap %s: %s\n", fname, strerror (errno));
exit (EXIT_FAILURE);
}
task = rspamd_task_new (NULL, cfg);
task->msg.begin = map;
task->msg.len = st.st_size;
ts1 = rspamd_get_ticks ();
if (mode == MODE_NORMAL) {
if (!rspamd_mime_parse_task (task, &err)) {
rspamd_fprintf (stderr, "cannot parse %s: %e\n", fname, err);
g_error_free (err);
}
}
#if 0
else if (mode == MODE_GMIME) {
tmp.data = map;
tmp.len = st.st_size;
stream = g_mime_stream_mem_new_with_byte_array (&tmp);
g_mime_stream_mem_set_owner (GMIME_STREAM_MEM (stream), FALSE);
parser = g_mime_parser_new_with_stream (stream);
message = g_mime_parser_construct_message (parser);
}
#endif
ts2 = rspamd_get_ticks ();
total_time += ts2 - ts1;
if (mode == MODE_NORMAL) {
for (i = 0; i < task->parts->len; i ++) {
part = g_ptr_array_index (task->parts, i);
if (part->ct->flags & RSPAMD_CONTENT_TYPE_MULTIPART) {
rspamd_show_multipart (part);
}
else if (part->ct->flags & RSPAMD_CONTENT_TYPE_MESSAGE) {
rspamd_show_message (part);
}
else {
rspamd_show_normal (part);
}
}
}
#if 0
else if (mode == MODE_GMIME) {
g_mime_message_foreach (message, mime_foreach_callback, NULL);
}
#endif
rspamd_task_free (task);
munmap (map, st.st_size);
#if 0
if (mode == MODE_GMIME) {
g_object_unref (message);
}
#endif
}
gboolean
rspamd_client_command (struct rspamd_client_connection *conn,
const gchar *command, GQueue *attrs,
FILE *in, rspamd_client_callback cb,
gpointer ud, gboolean compressed,
const gchar *comp_dictionary,
const gchar *filename,
GError **err)
{
struct rspamd_client_request *req;
struct rspamd_http_client_header *nh;
gchar *p;
gsize remain, old_len;
GList *cur;
GString *input = NULL;
rspamd_fstring_t *body;
guint dict_id = 0;
gsize dict_len = 0;
void *dict = NULL;
ZSTD_CCtx *zctx;
req = g_malloc0 (sizeof (struct rspamd_client_request));
req->conn = conn;
req->cb = cb;
req->ud = ud;
req->msg = rspamd_http_new_message (HTTP_REQUEST);
if (conn->key) {
req->msg->peer_key = rspamd_pubkey_ref (conn->key);
}
if (in != NULL) {
/* Read input stream */
input = g_string_sized_new (BUFSIZ);
while (!feof (in)) {
p = input->str + input->len;
remain = input->allocated_len - input->len - 1;
if (remain == 0) {
old_len = input->len;
g_string_set_size (input, old_len * 2);
input->len = old_len;
continue;
}
remain = fread (p, 1, remain, in);
if (remain > 0) {
input->len += remain;
input->str[input->len] = '\0';
}
}
if (ferror (in) != 0) {
g_set_error (err, RCLIENT_ERROR, ferror (
in), "input IO error: %s", strerror (ferror (in)));
g_free (req);
g_string_free (input, TRUE);
return FALSE;
}
if (!compressed) {
body = rspamd_fstring_new_init (input->str, input->len);
}
else {
if (comp_dictionary) {
dict = rspamd_file_xmap (comp_dictionary, PROT_READ, &dict_len,
TRUE);
if (dict == NULL) {
g_set_error (err, RCLIENT_ERROR, errno,
"cannot open dictionary %s: %s",
comp_dictionary,
strerror (errno));
g_free (req);
g_string_free (input, TRUE);
return FALSE;
}
dict_id = ZDICT_getDictID (comp_dictionary, dict_len);
if (dict_id == 0) {
g_set_error (err, RCLIENT_ERROR, errno,
"cannot open dictionary %s: %s",
comp_dictionary,
strerror (errno));
g_free (req);
g_string_free (input, TRUE);
munmap (dict, dict_len);
return FALSE;
}
}
body = rspamd_fstring_sized_new (ZSTD_compressBound (input->len));
zctx = ZSTD_createCCtx ();
body->len = ZSTD_compress_usingDict (zctx, body->str, body->allocated,
input->str, input->len,
dict, dict_len,
1);
munmap (dict, dict_len);
if (ZSTD_isError (body->len)) {
g_set_error (err, RCLIENT_ERROR, ferror (
in), "compression error");
g_free (req);
g_string_free (input, TRUE);
rspamd_fstring_free (body);
ZSTD_freeCCtx (zctx);
return FALSE;
}
ZSTD_freeCCtx (zctx);
}
rspamd_http_message_set_body_from_fstring_steal (req->msg, body);
req->input = input;
}
else {
req->input = NULL;
}
/* Convert headers */
cur = attrs->head;
while (cur != NULL) {
nh = cur->data;
rspamd_http_message_add_header (req->msg, nh->name, nh->value);
cur = g_list_next (cur);
}
if (compressed) {
rspamd_http_message_add_header (req->msg, "Compression", "zstd");
if (dict_id != 0) {
gchar dict_str[32];
rspamd_snprintf (dict_str, sizeof (dict_str), "%ud", dict_id);
rspamd_http_message_add_header (req->msg, "Dictionary", dict_str);
}
}
if (filename) {
rspamd_http_message_add_header (req->msg, "Filename", filename);
}
req->msg->url = rspamd_fstring_append (req->msg->url, "/", 1);
req->msg->url = rspamd_fstring_append (req->msg->url, command, strlen (command));
conn->req = req;
conn->start_time = rspamd_get_ticks (FALSE);
if (compressed) {
rspamd_http_connection_write_message (conn->http_conn, req->msg, NULL,
"application/x-compressed", req,
&conn->timeout);
}
else {
rspamd_http_connection_write_message (conn->http_conn, req->msg, NULL,
"text/plain", req, &conn->timeout);
}
return TRUE;
}
static gint
rspamd_client_finish_handler (struct rspamd_http_connection *conn,
struct rspamd_http_message *msg)
{
struct rspamd_client_request *req =
(struct rspamd_client_request *)conn->ud;
struct rspamd_client_connection *c;
struct ucl_parser *parser;
GError *err;
const rspamd_ftok_t *tok;
c = req->conn;
if (!c->req_sent) {
c->req_sent = TRUE;
c->send_time = rspamd_get_ticks (FALSE);
rspamd_http_connection_reset (c->http_conn);
rspamd_http_connection_read_message (c->http_conn,
c->req,
&c->timeout);
return 0;
}
else {
if (rspamd_http_message_get_body (msg, NULL) == NULL || msg->code / 100 != 2) {
err = g_error_new (RCLIENT_ERROR, msg->code, "HTTP error: %d, %.*s",
msg->code,
(gint)msg->status->len, msg->status->str);
req->cb (c, msg, c->server_name->str, NULL, req->input, req->ud,
c->start_time, c->send_time, err);
g_error_free (err);
return 0;
}
tok = rspamd_http_message_find_header (msg, "compression");
if (tok) {
/* Need to uncompress */
rspamd_ftok_t t;
t.begin = "zstd";
t.len = 4;
if (rspamd_ftok_casecmp (tok, &t) == 0) {
ZSTD_DStream *zstream;
ZSTD_inBuffer zin;
ZSTD_outBuffer zout;
guchar *out;
gsize outlen, r;
zstream = ZSTD_createDStream ();
ZSTD_initDStream (zstream);
zin.pos = 0;
zin.src = msg->body_buf.begin;
zin.size = msg->body_buf.len;
if ((outlen = ZSTD_getDecompressedSize (zin.src, zin.size)) == 0) {
outlen = ZSTD_DStreamOutSize ();
}
out = g_malloc (outlen);
zout.dst = out;
zout.pos = 0;
zout.size = outlen;
while (zin.pos < zin.size) {
r = ZSTD_decompressStream (zstream, &zout, &zin);
if (ZSTD_isError (r)) {
err = g_error_new (RCLIENT_ERROR, 500,
"Decompression error: %s",
ZSTD_getErrorName (r));
req->cb (c, msg, c->server_name->str, NULL,
req->input, req->ud, c->start_time,
c->send_time, err);
g_error_free (err);
ZSTD_freeDStream (zstream);
g_free (out);
return 0;
}
if (zout.pos == zout.size) {
/* We need to extend output buffer */
zout.size = zout.size * 1.5 + 1.0;
zout.dst = g_realloc (zout.dst, zout.size);
}
}
ZSTD_freeDStream (zstream);
parser = ucl_parser_new (0);
if (!ucl_parser_add_chunk (parser, zout.dst, zout.pos)) {
err = g_error_new (RCLIENT_ERROR, msg->code, "Cannot parse UCL: %s",
ucl_parser_get_error (parser));
ucl_parser_free (parser);
req->cb (c, msg, c->server_name->str, NULL, req->input,
req->ud, c->start_time, c->send_time, err);
g_error_free (err);
g_free (zout.dst);
return 0;
}
g_free (zout.dst);
}
else {
err = g_error_new (RCLIENT_ERROR, 500,
"Invalid compression method");
req->cb (c, msg, c->server_name->str, NULL,
req->input, req->ud, c->start_time, c->send_time, err);
g_error_free (err);
return 0;
}
}
else {
parser = ucl_parser_new (0);
if (!ucl_parser_add_chunk (parser, msg->body_buf.begin, msg->body_buf.len)) {
err = g_error_new (RCLIENT_ERROR, msg->code, "Cannot parse UCL: %s",
ucl_parser_get_error (parser));
ucl_parser_free (parser);
req->cb (c, msg, c->server_name->str, NULL,
req->input, req->ud, c->start_time, c->send_time, err);
g_error_free (err);
return 0;
}
}
req->cb (c, msg, c->server_name->str, ucl_parser_get_object (
parser), req->input, req->ud, c->start_time, c->send_time, NULL);
ucl_parser_free (parser);
}
return 0;
}
