static guint rspamd_re_cache_process_pcre (struct rspamd_re_runtime *rt, rspamd_regexp_t *re, rspamd_mempool_t *pool, const guchar *in, gsize len, gboolean is_raw) { guint r = 0; const gchar *start = NULL, *end = NULL; guint max_hits = rspamd_regexp_get_maxhits (re); guint64 id = rspamd_regexp_get_cache_id (re); gdouble t1, t2; const gdouble slow_time = 0.1; if (len == 0) { len = strlen (in); } if (rt->cache->max_re_data > 0 && len > rt->cache->max_re_data) { len = rt->cache->max_re_data; } r = rt->results[id]; t1 = rspamd_get_ticks (); if (max_hits == 0 || r < max_hits) { while (rspamd_regexp_search (re, in, len, &start, &end, is_raw, NULL)) { r++; if (max_hits > 0 && r >= max_hits) { break; } } rt->stat.regexp_checked++; rt->stat.bytes_scanned_pcre += len; rt->stat.bytes_scanned += len; if (r > 0) { rt->stat.regexp_matched += r; } } t2 = rspamd_get_ticks (); if (t2 - t1 > slow_time) { msg_info_pool ("regexp '%16s' took %.2f seconds to execute", rspamd_regexp_get_pattern (re), t2 - t1); } return r; }
static gboolean rspamd_symbols_cache_check_symbol (struct rspamd_task *task, struct symbols_cache *cache, struct cache_item *item, struct cache_savepoint *checkpoint) { guint pending_before, pending_after; double t1, t2; guint64 diff; if (item->type == SYMBOL_TYPE_NORMAL || item->type == SYMBOL_TYPE_CALLBACK) { g_assert (item->func != NULL); /* Check has been started */ setbit (checkpoint->processed_bits, item->id * 2); t1 = rspamd_get_ticks (); pending_before = rspamd_session_events_pending (task->s); /* Watch for events appeared */ rspamd_session_watch_start (task->s, rspamd_symbols_cache_watcher_cb, item); if (item->symbol != NULL && G_UNLIKELY (check_debug_symbol (task->cfg, item->symbol))) { rspamd_log_debug (rspamd_main->logger); item->func (task, item->user_data); rspamd_log_nodebug (rspamd_main->logger); } else { item->func (task, item->user_data); } t2 = rspamd_get_ticks (); diff = (t2 - t1) * 1000000; rspamd_set_counter (item, diff); rspamd_session_watch_stop (task->s); pending_after = rspamd_session_events_pending (task->s); if (pending_before == pending_after) { /* No new events registered */ setbit (checkpoint->processed_bits, item->id * 2 + 1); return TRUE; } return FALSE; } else { setbit (checkpoint->processed_bits, item->id * 2); setbit (checkpoint->processed_bits, item->id * 2 + 1); return TRUE; } }
static gint rspamd_client_finish (struct rspamd_http_connection *conn, struct rspamd_http_message *msg) { struct client_cbdata *cb = conn->ud; *(cb->lat) = rspamd_get_ticks () * 1000. - cb->ts; close (conn->fd); rspamd_http_connection_unref (conn); g_free (cb); return 0; }
static void rspamd_http_client_func (const gchar *path, rspamd_inet_addr_t *addr, struct rspamd_cryptobox_keypair *kp, struct rspamd_cryptobox_pubkey *peer_kp, struct rspamd_keypair_cache *c, struct event_base *ev_base, double *latency) { struct rspamd_http_message *msg; struct rspamd_http_connection *conn; gchar urlbuf[PATH_MAX]; struct client_cbdata *cb; gint fd; g_assert ((fd = rspamd_inet_address_connect (addr, SOCK_STREAM, TRUE)) != -1); conn = rspamd_http_connection_new (rspamd_client_body, rspamd_client_err, rspamd_client_finish, RSPAMD_HTTP_CLIENT_SIMPLE, RSPAMD_HTTP_CLIENT, c, NULL); rspamd_snprintf (urlbuf, sizeof (urlbuf), "http://127.0.0.1/%s", path); msg = rspamd_http_message_from_url (urlbuf); g_assert (conn != NULL && msg != NULL); if (kp != NULL) { g_assert (peer_kp != NULL); rspamd_http_connection_set_key (conn, kp); msg->peer_key = rspamd_pubkey_ref (peer_kp); } cb = g_malloc (sizeof (*cb)); cb->ts = rspamd_get_ticks () * 1000.; cb->lat = latency; rspamd_http_connection_write_message (conn, msg, NULL, NULL, cb, fd, NULL, ev_base); }
void rspamd_radix_test_func (void) { #if 0 radix_tree_t *tree = radix_tree_create (); #endif radix_compressed_t *comp_tree = radix_create_compressed (); struct { guint32 addr; guint32 mask; guint8 addr6[16]; guint32 mask6; } *addrs; gsize nelts, i; gint lc; gboolean all_good = TRUE; gdouble ts1, ts2; double diff; /* Test suite for the compressed trie */ rspamd_radix_text_vec (); nelts = max_elts; /* First of all we generate many elements and push them to the array */ addrs = g_malloc (nelts * sizeof (addrs[0])); for (i = 0; i < nelts; i ++) { addrs[i].addr = ottery_rand_uint32 (); addrs[i].mask = masks[ottery_rand_range(G_N_ELEMENTS (masks) - 1)]; ottery_rand_bytes (addrs[i].addr6, sizeof(addrs[i].addr6)); addrs[i].mask6 = ottery_rand_range(128); } #if 0 msg_info ("old radix performance (%z elts)", nelts); ts1 = rspamd_get_ticks (); for (i = 0; i < nelts; i ++) { guint32 mask = G_MAXUINT32 << (32 - addrs[i].mask); radix32tree_insert (tree, addrs[i].addr, mask, 1); } ts2 = rspamd_get_ticks (); diff = (ts2 - ts1) * 1000.0; msg_info ("Added %z elements in %.6f ms", nelts, diff); ts1 = rspamd_get_ticks (); for (lc = 0; lc < lookup_cycles; lc ++) { for (i = 0; i < nelts; i ++) { g_assert (radix32tree_find (tree, addrs[i].addr) != RADIX_NO_VALUE); } } ts2 = rspamd_get_ticks (); diff = (ts2 - ts1) * 1000.0; msg_info ("Checked %z elements in %.6f ms", nelts, diff); ts1 = rspamd_get_ticks (); for (i = 0; i < nelts; i ++) { radix32tree_delete (tree, addrs[i].addr, addrs[i].mask); } ts2 = rspamd_get_ticks (); diff = (ts2 - ts1) * 1000.; msg_info ("Deleted %z elements in %.6f ms", nelts, diff); radix_tree_free (tree); #endif msg_info ("new radix performance (%z elts)", nelts); ts1 = rspamd_get_ticks (); for (i = 0; i < nelts; i ++) { radix_insert_compressed (comp_tree, addrs[i].addr6, sizeof (addrs[i].addr6), 128 - addrs[i].mask6, i); } ts2 = rspamd_get_ticks (); diff = (ts2 - ts1) * 1000.0; msg_info ("Added %z elements in %.6f ms", nelts, diff); ts1 = rspamd_get_ticks (); for (lc = 0; lc < lookup_cycles; lc ++) { for (i = 0; i < nelts; i ++) { if (radix_find_compressed (comp_tree, addrs[i].addr6, sizeof (addrs[i].addr6)) == RADIX_NO_VALUE) { all_good = FALSE; } } } #if 1 if (!all_good) { for (i = 0; i < nelts; i ++) { /* Used to write bad random vector */ char ipbuf[INET6_ADDRSTRLEN + 1]; inet_ntop(AF_INET6, addrs[i].addr6, ipbuf, sizeof(ipbuf)); msg_info("{\"%s\", NULL, \"%ud\", 0, 0, 0, 0},", ipbuf, addrs[i].mask6); } } #endif g_assert (all_good); ts2 = rspamd_get_ticks (); diff = (ts2 - ts1) * 1000.0; msg_info ("Checked %z elements in %.6f ms", nelts, diff); radix_destroy_compressed (comp_tree); g_free (addrs); }
void rspamd_cryptobox_test_func (void) { void *map; guchar *begin, *end; rspamd_nm_t key; rspamd_nonce_t nonce; rspamd_sig_t mac; struct rspamd_cryptobox_segment *seg; double t1, t2; gint i, cnt, ms; map = create_mapping (mapping_size, &begin, &end); ottery_rand_bytes (key, sizeof (key)); ottery_rand_bytes (nonce, sizeof (nonce)); memset (mac, 0, sizeof (mac)); seg = g_slice_alloc0 (sizeof (*seg) * max_seg * 10); /* Test baseline */ t1 = rspamd_get_ticks (); rspamd_cryptobox_encrypt_nm_inplace (begin, end - begin, nonce, key, mac); t2 = rspamd_get_ticks (); check_result (key, nonce, mac, begin, end); msg_info ("baseline encryption: %.6f", t2 - t1); /* A single chunk as vector */ seg[0].data = begin; seg[0].len = end - begin; t1 = rspamd_get_ticks (); rspamd_cryptobox_encryptv_nm_inplace (seg, 1, nonce, key, mac); t2 = rspamd_get_ticks (); check_result (key, nonce, mac, begin, end); msg_info ("bulk encryption: %.6f", t2 - t1); /* Two chunks as vector */ seg[0].data = begin; seg[0].len = (end - begin) / 2; seg[1].data = begin + seg[0].len; seg[1].len = (end - begin) - seg[0].len; t1 = rspamd_get_ticks (); rspamd_cryptobox_encryptv_nm_inplace (seg, 2, nonce, key, mac); t2 = rspamd_get_ticks (); check_result (key, nonce, mac, begin, end); msg_info ("2 equal chunks encryption: %.6f", t2 - t1); seg[0].data = begin; seg[0].len = 1; seg[1].data = begin + seg[0].len; seg[1].len = (end - begin) - seg[0].len; t1 = rspamd_get_ticks (); rspamd_cryptobox_encryptv_nm_inplace (seg, 2, nonce, key, mac); t2 = rspamd_get_ticks (); check_result (key, nonce, mac, begin, end); msg_info ("small and large chunks encryption: %.6f", t2 - t1); seg[0].data = begin; seg[0].len = (end - begin) - 3; seg[1].data = begin + seg[0].len; seg[1].len = (end - begin) - seg[0].len; t1 = rspamd_get_ticks (); rspamd_cryptobox_encryptv_nm_inplace (seg, 2, nonce, key, mac); t2 = rspamd_get_ticks (); check_result (key, nonce, mac, begin, end); msg_info ("large and small chunks encryption: %.6f", t2 - t1); /* Random two chunks as vector */ seg[0].data = begin; seg[0].len = ottery_rand_range (end - begin - 1) + 1; seg[1].data = begin + seg[0].len; seg[1].len = (end - begin) - seg[0].len; t1 = rspamd_get_ticks (); rspamd_cryptobox_encryptv_nm_inplace (seg, 2, nonce, key, mac); t2 = rspamd_get_ticks (); check_result (key, nonce, mac, begin, end); msg_info ("random 2 chunks encryption: %.6f", t2 - t1); /* 3 specific chunks */ seg[0].data = begin; seg[0].len = 2; seg[1].data = begin + seg[0].len; seg[1].len = 2049; seg[2].data = begin + seg[0].len + seg[1].len; seg[2].len = (end - begin) - seg[0].len - seg[1].len; t1 = rspamd_get_ticks (); rspamd_cryptobox_encryptv_nm_inplace (seg, 3, nonce, key, mac); t2 = rspamd_get_ticks (); check_result (key, nonce, mac, begin, end); msg_info ("small, medium and large chunks encryption: %.6f", t2 - t1); cnt = create_random_split (seg, max_seg, begin, end); t1 = rspamd_get_ticks (); rspamd_cryptobox_encryptv_nm_inplace (seg, cnt, nonce, key, mac); t2 = rspamd_get_ticks (); check_result (key, nonce, mac, begin, end); msg_info ("random split of %d chunks encryption: %.6f", cnt, t2 - t1); cnt = create_realistic_split (seg, max_seg, begin, end); t1 = rspamd_get_ticks (); rspamd_cryptobox_encryptv_nm_inplace (seg, cnt, nonce, key, mac); t2 = rspamd_get_ticks (); check_result (key, nonce, mac, begin, end); msg_info ("realistic split of %d chunks encryption: %.6f", cnt, t2 - t1); cnt = create_constrainted_split (seg, max_seg + 1, 32, begin, end); t1 = rspamd_get_ticks (); rspamd_cryptobox_encryptv_nm_inplace (seg, cnt, nonce, key, mac); t2 = rspamd_get_ticks (); check_result (key, nonce, mac, begin, end); msg_info ("constrainted split of %d chunks encryption: %.6f", cnt, t2 - t1); for (i = 0; i < random_fuzz_cnt; i ++) { ms = ottery_rand_range (i % max_seg * 2) + 1; cnt = create_random_split (seg, ms, begin, end); t1 = rspamd_get_ticks (); rspamd_cryptobox_encryptv_nm_inplace (seg, cnt, nonce, key, mac); t2 = rspamd_get_ticks (); check_result (key, nonce, mac, begin, end); if (i % 1000 == 0) { msg_info ("random fuzz iterations: %d", i); } } for (i = 0; i < random_fuzz_cnt; i ++) { ms = ottery_rand_range (i % max_seg * 2) + 1; cnt = create_realistic_split (seg, ms, begin, end); t1 = rspamd_get_ticks (); rspamd_cryptobox_encryptv_nm_inplace (seg, cnt, nonce, key, mac); t2 = rspamd_get_ticks (); check_result (key, nonce, mac, begin, end); if (i % 1000 == 0) { msg_info ("realistic fuzz iterations: %d", i); } } for (i = 0; i < random_fuzz_cnt; i ++) { ms = ottery_rand_range (i % max_seg * 10) + 1; cnt = create_constrainted_split (seg, ms, i, begin, end); t1 = rspamd_get_ticks (); rspamd_cryptobox_encryptv_nm_inplace (seg, cnt, nonce, key, mac); t2 = rspamd_get_ticks (); check_result (key, nonce, mac, begin, end); if (i % 1000 == 0) { msg_info ("constrainted fuzz iterations: %d", i); } } }
/* * Create new task */ struct rspamd_task * rspamd_task_new (struct rspamd_worker *worker) { struct rspamd_task *new_task; new_task = g_slice_alloc0 (sizeof (struct rspamd_task)); new_task->worker = worker; if (worker) { new_task->cfg = worker->srv->cfg; if (new_task->cfg->check_all_filters) { new_task->flags |= RSPAMD_TASK_FLAG_PASS_ALL; } } gettimeofday (&new_task->tv, NULL); new_task->time_real = rspamd_get_ticks (); new_task->time_virtual = rspamd_get_virtual_ticks (); new_task->task_pool = rspamd_mempool_new (rspamd_mempool_suggest_size (), "task"); new_task->results = g_hash_table_new (rspamd_str_hash, rspamd_str_equal); rspamd_mempool_add_destructor (new_task->task_pool, (rspamd_mempool_destruct_t) g_hash_table_unref, new_task->results); new_task->re_cache = g_hash_table_new (rspamd_str_hash, rspamd_str_equal); rspamd_mempool_add_destructor (new_task->task_pool, (rspamd_mempool_destruct_t) g_hash_table_unref, new_task->re_cache); new_task->raw_headers = g_hash_table_new (rspamd_strcase_hash, rspamd_strcase_equal); new_task->request_headers = g_hash_table_new_full (rspamd_gstring_icase_hash, rspamd_gstring_icase_equal, rspamd_gstring_free_hard, rspamd_gstring_free_hard); rspamd_mempool_add_destructor (new_task->task_pool, (rspamd_mempool_destruct_t) g_hash_table_unref, new_task->request_headers); new_task->reply_headers = g_hash_table_new_full (rspamd_gstring_icase_hash, rspamd_gstring_icase_equal, rspamd_gstring_free_hard, rspamd_gstring_free_hard); rspamd_mempool_add_destructor (new_task->task_pool, (rspamd_mempool_destruct_t) g_hash_table_unref, new_task->reply_headers); rspamd_mempool_add_destructor (new_task->task_pool, (rspamd_mempool_destruct_t) g_hash_table_unref, new_task->raw_headers); new_task->emails = g_hash_table_new (rspamd_url_hash, rspamd_emails_cmp); rspamd_mempool_add_destructor (new_task->task_pool, (rspamd_mempool_destruct_t) g_hash_table_unref, new_task->emails); new_task->urls = g_hash_table_new (rspamd_url_hash, rspamd_urls_cmp); rspamd_mempool_add_destructor (new_task->task_pool, (rspamd_mempool_destruct_t) g_hash_table_unref, new_task->urls); new_task->parts = g_ptr_array_sized_new (4); rspamd_mempool_add_destructor (new_task->task_pool, rspamd_ptr_array_free_hard, new_task->parts); new_task->text_parts = g_ptr_array_sized_new (2); rspamd_mempool_add_destructor (new_task->task_pool, rspamd_ptr_array_free_hard, new_task->text_parts); new_task->received = g_ptr_array_sized_new (8); rspamd_mempool_add_destructor (new_task->task_pool, rspamd_ptr_array_free_hard, new_task->received); new_task->sock = -1; new_task->flags |= (RSPAMD_TASK_FLAG_MIME|RSPAMD_TASK_FLAG_JSON); new_task->pre_result.action = METRIC_ACTION_NOACTION; new_task->message_id = new_task->queue_id = "undef"; return new_task; }
void rspamd_http_test_func (void) { struct event_base *ev_base = event_init (); rspamd_mempool_t *pool = rspamd_mempool_new (rspamd_mempool_suggest_size (), NULL); struct rspamd_cryptobox_keypair *serv_key, *client_key; struct rspamd_cryptobox_pubkey *peer_key; struct rspamd_keypair_cache *c; rspamd_mempool_mutex_t *mtx; rspamd_inet_addr_t *addr; gdouble ts1, ts2; gchar filepath[PATH_MAX], *buf; gchar *env; gint fd; guint i, j; pid_t *sfd; GString *b32_key; double diff, total_diff = 0.0, *latency, mean, std; /* Read environment */ if ((env = getenv ("RSPAMD_HTTP_CONNS")) != NULL) { pconns = strtoul (env, NULL, 10); } else { return; } if ((env = getenv ("RSPAMD_HTTP_TESTS")) != NULL) { ntests = strtoul (env, NULL, 10); } if ((env = getenv ("RSPAMD_HTTP_SIZE")) != NULL) { file_size = strtoul (env, NULL, 10); } if ((env = getenv ("RSPAMD_HTTP_SERVERS")) != NULL) { nservers = strtoul (env, NULL, 10); } rspamd_cryptobox_init (); rspamd_snprintf (filepath, sizeof (filepath), "/tmp/http-test-XXXXXX"); g_assert ((fd = mkstemp (filepath)) != -1); sfd = g_alloca (sizeof (*sfd) * nservers); latency = g_malloc0 (pconns * ntests * sizeof (gdouble)); buf = g_malloc (file_size); memset (buf, 0, file_size); g_assert (write (fd, buf, file_size) == file_size); g_free (buf); mtx = rspamd_mempool_get_mutex (pool); rspamd_parse_inet_address (&addr, "127.0.0.1", 0); rspamd_inet_address_set_port (addr, 43898); serv_key = rspamd_keypair_new (RSPAMD_KEYPAIR_KEX, RSPAMD_CRYPTOBOX_MODE_25519); client_key = rspamd_keypair_new (RSPAMD_KEYPAIR_KEX, RSPAMD_CRYPTOBOX_MODE_25519); c = rspamd_keypair_cache_new (16); rspamd_http_start_servers (sfd, addr, serv_key, NULL); usleep (100000); /* Do client stuff */ gperf_profiler_init (NULL, "plain-http-client"); for (i = 0; i < ntests; i ++) { for (j = 0; j < pconns; j ++) { rspamd_http_client_func (filepath + sizeof ("/tmp") - 1, addr, NULL, NULL, c, ev_base, &latency[i * pconns + j]); } ts1 = rspamd_get_ticks (); event_base_loop (ev_base, 0); ts2 = rspamd_get_ticks (); diff = (ts2 - ts1) * 1000.0; total_diff += diff; } gperf_profiler_stop (); msg_info ("Made %d connections of size %d in %.6f ms, %.6f cps", ntests * pconns, file_size, total_diff, ntests * pconns / total_diff * 1000.); mean = rspamd_http_calculate_mean (latency, &std); msg_info ("Latency: %.6f ms mean, %.6f dev", mean, std); rspamd_http_stop_servers (sfd); rspamd_http_start_servers (sfd, addr, serv_key, c); //rspamd_mempool_lock_mutex (mtx); usleep (100000); b32_key = rspamd_keypair_print (serv_key, RSPAMD_KEYPAIR_PUBKEY|RSPAMD_KEYPAIR_BASE32); g_assert (b32_key != NULL); peer_key = rspamd_pubkey_from_base32 (b32_key->str, b32_key->len, RSPAMD_KEYPAIR_KEX, RSPAMD_CRYPTOBOX_MODE_25519); g_assert (peer_key != NULL); total_diff = 0.0; gperf_profiler_init (NULL, "cached-http-client"); for (i = 0; i < ntests; i ++) { for (j = 0; j < pconns; j ++) { rspamd_http_client_func (filepath + sizeof ("/tmp") - 1, addr, client_key, peer_key, c, ev_base, &latency[i * pconns + j]); } ts1 = rspamd_get_ticks (); event_base_loop (ev_base, 0); ts2 = rspamd_get_ticks (); diff = (ts2 - ts1) * 1000.0; total_diff += diff; } gperf_profiler_stop (); msg_info ("Made %d encrypted connections of size %d in %.6f ms, %.6f cps", ntests * pconns, file_size, total_diff, ntests * pconns / total_diff * 1000.); mean = rspamd_http_calculate_mean (latency, &std); msg_info ("Latency: %.6f ms mean, %.6f dev", mean, std); /* Restart server */ rspamd_http_stop_servers (sfd); /* No keypairs cache */ rspamd_http_start_servers (sfd, addr, serv_key, NULL); usleep (100000); total_diff = 0.0; gperf_profiler_init (NULL, "fair-http-client"); for (i = 0; i < ntests; i ++) { for (j = 0; j < pconns; j ++) { rspamd_http_client_func (filepath + sizeof ("/tmp") - 1, addr, client_key, peer_key, c, ev_base, &latency[i * pconns + j]); } ts1 = rspamd_get_ticks (); event_base_loop (ev_base, 0); ts2 = rspamd_get_ticks (); diff = (ts2 - ts1) * 1000.0; total_diff += diff; } gperf_profiler_stop (); msg_info ("Made %d uncached encrypted connections of size %d in %.6f ms, %.6f cps", ntests * pconns, file_size, total_diff, ntests * pconns / total_diff * 1000.); mean = rspamd_http_calculate_mean (latency, &std); msg_info ("Latency: %.6f ms mean, %.6f dev", mean, std); /* AES mode */ serv_key = rspamd_keypair_new (RSPAMD_KEYPAIR_KEX, RSPAMD_CRYPTOBOX_MODE_NIST); client_key = rspamd_keypair_new (RSPAMD_KEYPAIR_KEX, RSPAMD_CRYPTOBOX_MODE_NIST); c = rspamd_keypair_cache_new (16); /* Restart server */ rspamd_http_stop_servers (sfd); /* No keypairs cache */ rspamd_http_start_servers (sfd, addr, serv_key, c); //rspamd_mempool_lock_mutex (mtx); usleep (100000); b32_key = rspamd_keypair_print (serv_key, RSPAMD_KEYPAIR_PUBKEY | RSPAMD_KEYPAIR_BASE32); g_assert (b32_key != NULL); peer_key = rspamd_pubkey_from_base32 (b32_key->str, b32_key->len, RSPAMD_KEYPAIR_KEX, RSPAMD_CRYPTOBOX_MODE_NIST); g_assert (peer_key != NULL); total_diff = 0.0; gperf_profiler_init (NULL, "cached-http-client-aes"); for (i = 0; i < ntests; i++) { for (j = 0; j < pconns; j++) { rspamd_http_client_func (filepath + sizeof ("/tmp") - 1, addr, client_key, peer_key, NULL, ev_base, &latency[i * pconns + j]); } ts1 = rspamd_get_ticks (); event_base_loop (ev_base, 0); ts2 = rspamd_get_ticks (); diff = (ts2 - ts1) * 1000.0; total_diff += diff; } gperf_profiler_stop (); msg_info ( "Made %d aes encrypted connections of size %d in %.6f ms, %.6f cps", ntests * pconns, file_size, total_diff, ntests * pconns / total_diff * 1000.); mean = rspamd_http_calculate_mean (latency, &std); msg_info ("Latency: %.6f ms mean, %.6f dev", mean, std); /* Restart server */ rspamd_http_stop_servers (sfd); /* No keypairs cache */ rspamd_http_start_servers (sfd, addr, serv_key, NULL); //rspamd_mempool_lock_mutex (mtx); usleep (100000); total_diff = 0.0; gperf_profiler_init (NULL, "fair-http-client-aes"); for (i = 0; i < ntests; i++) { for (j = 0; j < pconns; j++) { rspamd_http_client_func (filepath + sizeof ("/tmp") - 1, addr, client_key, peer_key, c, ev_base, &latency[i * pconns + j]); } ts1 = rspamd_get_ticks (); event_base_loop (ev_base, 0); ts2 = rspamd_get_ticks (); diff = (ts2 - ts1) * 1000.0; total_diff += diff; } gperf_profiler_stop (); msg_info ( "Made %d uncached aes encrypted connections of size %d in %.6f ms, %.6f cps", ntests * pconns, file_size, total_diff, ntests * pconns / total_diff * 1000.); mean = rspamd_http_calculate_mean (latency, &std); msg_info ("Latency: %.6f ms mean, %.6f dev", mean, std); close (fd); unlink (filepath); rspamd_http_stop_servers (sfd); }
static gboolean rspamd_symbols_cache_check_symbol (struct rspamd_task *task, struct symbols_cache *cache, struct cache_item *item, struct cache_savepoint *checkpoint, gdouble *total_diff) { guint pending_before, pending_after; double t1, t2; gdouble diff; struct rspamd_task **ptask; lua_State *L; gboolean check = TRUE; const gdouble slow_diff_limit = 1e5; if (item->type & (SYMBOL_TYPE_NORMAL|SYMBOL_TYPE_CALLBACK)) { g_assert (item->func != NULL); /* Check has been started */ setbit (checkpoint->processed_bits, item->id * 2); if (RSPAMD_TASK_IS_EMPTY (task) && !(item->type & SYMBOL_TYPE_EMPTY)) { check = FALSE; } else if (item->condition_cb != -1) { /* We also executes condition callback to check if we need this symbol */ L = task->cfg->lua_state; lua_rawgeti (L, LUA_REGISTRYINDEX, item->condition_cb); ptask = lua_newuserdata (L, sizeof (struct rspamd_task *)); rspamd_lua_setclass (L, "rspamd{task}", -1); *ptask = task; if (lua_pcall (L, 1, 1, 0) != 0) { msg_info_task ("call to condition for %s failed: %s", item->symbol, lua_tostring (L, -1)); lua_pop (L, 1); } else { check = lua_toboolean (L, -1); lua_pop (L, 1); } } if (check) { t1 = rspamd_get_ticks (); pending_before = rspamd_session_events_pending (task->s); /* Watch for events appeared */ rspamd_session_watch_start (task->s, rspamd_symbols_cache_watcher_cb, item); msg_debug_task ("execute %s, %d", item->symbol, item->id); item->func (task, item->user_data); t2 = rspamd_get_ticks (); diff = (t2 - t1) * 1e6; if (total_diff) { *total_diff += diff; } if (diff > slow_diff_limit) { msg_info_task ("slow rule: %s: %d ms", item->symbol, (gint)(diff / 1000.)); } rspamd_set_counter (item, diff); rspamd_session_watch_stop (task->s); pending_after = rspamd_session_events_pending (task->s); if (pending_before == pending_after) { /* No new events registered */ setbit (checkpoint->processed_bits, item->id * 2 + 1); return TRUE; } return FALSE; } else { msg_debug_task ("skipping check of %s as its condition is false", item->symbol); setbit (checkpoint->processed_bits, item->id * 2 + 1); return TRUE; } } else { setbit (checkpoint->processed_bits, item->id * 2); setbit (checkpoint->processed_bits, item->id * 2 + 1); return TRUE; } }
static void rspamd_process_file (struct rspamd_config *cfg, const gchar *fname, gint mode) { struct rspamd_task *task; gint fd; gpointer map; struct stat st; GError *err = NULL; #if 0 GMimeMessage *message; GMimeParser *parser; GMimeStream *stream; GByteArray tmp; #endif struct rspamd_mime_part *part; guint i; gdouble ts1, ts2; fd = open (fname, O_RDONLY); if (fd == -1) { rspamd_fprintf (stderr, "cannot open %s: %s\n", fname, strerror (errno)); exit (EXIT_FAILURE); } if (fstat (fd, &st) == -1) { rspamd_fprintf (stderr, "cannot stat %s: %s\n", fname, strerror (errno)); exit (EXIT_FAILURE); } map = mmap (NULL, st.st_size, PROT_READ, MAP_SHARED, fd, 0); close (fd); if (map == MAP_FAILED) { rspamd_fprintf (stderr, "cannot mmap %s: %s\n", fname, strerror (errno)); exit (EXIT_FAILURE); } task = rspamd_task_new (NULL, cfg); task->msg.begin = map; task->msg.len = st.st_size; ts1 = rspamd_get_ticks (); if (mode == MODE_NORMAL) { if (!rspamd_mime_parse_task (task, &err)) { rspamd_fprintf (stderr, "cannot parse %s: %e\n", fname, err); g_error_free (err); } } #if 0 else if (mode == MODE_GMIME) { tmp.data = map; tmp.len = st.st_size; stream = g_mime_stream_mem_new_with_byte_array (&tmp); g_mime_stream_mem_set_owner (GMIME_STREAM_MEM (stream), FALSE); parser = g_mime_parser_new_with_stream (stream); message = g_mime_parser_construct_message (parser); } #endif ts2 = rspamd_get_ticks (); total_time += ts2 - ts1; if (mode == MODE_NORMAL) { for (i = 0; i < task->parts->len; i ++) { part = g_ptr_array_index (task->parts, i); if (part->ct->flags & RSPAMD_CONTENT_TYPE_MULTIPART) { rspamd_show_multipart (part); } else if (part->ct->flags & RSPAMD_CONTENT_TYPE_MESSAGE) { rspamd_show_message (part); } else { rspamd_show_normal (part); } } } #if 0 else if (mode == MODE_GMIME) { g_mime_message_foreach (message, mime_foreach_callback, NULL); } #endif rspamd_task_free (task); munmap (map, st.st_size); #if 0 if (mode == MODE_GMIME) { g_object_unref (message); } #endif }
gboolean rspamd_client_command (struct rspamd_client_connection *conn, const gchar *command, GQueue *attrs, FILE *in, rspamd_client_callback cb, gpointer ud, gboolean compressed, const gchar *comp_dictionary, const gchar *filename, GError **err) { struct rspamd_client_request *req; struct rspamd_http_client_header *nh; gchar *p; gsize remain, old_len; GList *cur; GString *input = NULL; rspamd_fstring_t *body; guint dict_id = 0; gsize dict_len = 0; void *dict = NULL; ZSTD_CCtx *zctx; req = g_malloc0 (sizeof (struct rspamd_client_request)); req->conn = conn; req->cb = cb; req->ud = ud; req->msg = rspamd_http_new_message (HTTP_REQUEST); if (conn->key) { req->msg->peer_key = rspamd_pubkey_ref (conn->key); } if (in != NULL) { /* Read input stream */ input = g_string_sized_new (BUFSIZ); while (!feof (in)) { p = input->str + input->len; remain = input->allocated_len - input->len - 1; if (remain == 0) { old_len = input->len; g_string_set_size (input, old_len * 2); input->len = old_len; continue; } remain = fread (p, 1, remain, in); if (remain > 0) { input->len += remain; input->str[input->len] = '\0'; } } if (ferror (in) != 0) { g_set_error (err, RCLIENT_ERROR, ferror ( in), "input IO error: %s", strerror (ferror (in))); g_free (req); g_string_free (input, TRUE); return FALSE; } if (!compressed) { body = rspamd_fstring_new_init (input->str, input->len); } else { if (comp_dictionary) { dict = rspamd_file_xmap (comp_dictionary, PROT_READ, &dict_len, TRUE); if (dict == NULL) { g_set_error (err, RCLIENT_ERROR, errno, "cannot open dictionary %s: %s", comp_dictionary, strerror (errno)); g_free (req); g_string_free (input, TRUE); return FALSE; } dict_id = ZDICT_getDictID (comp_dictionary, dict_len); if (dict_id == 0) { g_set_error (err, RCLIENT_ERROR, errno, "cannot open dictionary %s: %s", comp_dictionary, strerror (errno)); g_free (req); g_string_free (input, TRUE); munmap (dict, dict_len); return FALSE; } } body = rspamd_fstring_sized_new (ZSTD_compressBound (input->len)); zctx = ZSTD_createCCtx (); body->len = ZSTD_compress_usingDict (zctx, body->str, body->allocated, input->str, input->len, dict, dict_len, 1); munmap (dict, dict_len); if (ZSTD_isError (body->len)) { g_set_error (err, RCLIENT_ERROR, ferror ( in), "compression error"); g_free (req); g_string_free (input, TRUE); rspamd_fstring_free (body); ZSTD_freeCCtx (zctx); return FALSE; } ZSTD_freeCCtx (zctx); } rspamd_http_message_set_body_from_fstring_steal (req->msg, body); req->input = input; } else { req->input = NULL; } /* Convert headers */ cur = attrs->head; while (cur != NULL) { nh = cur->data; rspamd_http_message_add_header (req->msg, nh->name, nh->value); cur = g_list_next (cur); } if (compressed) { rspamd_http_message_add_header (req->msg, "Compression", "zstd"); if (dict_id != 0) { gchar dict_str[32]; rspamd_snprintf (dict_str, sizeof (dict_str), "%ud", dict_id); rspamd_http_message_add_header (req->msg, "Dictionary", dict_str); } } if (filename) { rspamd_http_message_add_header (req->msg, "Filename", filename); } req->msg->url = rspamd_fstring_append (req->msg->url, "/", 1); req->msg->url = rspamd_fstring_append (req->msg->url, command, strlen (command)); conn->req = req; conn->start_time = rspamd_get_ticks (FALSE); if (compressed) { rspamd_http_connection_write_message (conn->http_conn, req->msg, NULL, "application/x-compressed", req, &conn->timeout); } else { rspamd_http_connection_write_message (conn->http_conn, req->msg, NULL, "text/plain", req, &conn->timeout); } return TRUE; }
static gint rspamd_client_finish_handler (struct rspamd_http_connection *conn, struct rspamd_http_message *msg) { struct rspamd_client_request *req = (struct rspamd_client_request *)conn->ud; struct rspamd_client_connection *c; struct ucl_parser *parser; GError *err; const rspamd_ftok_t *tok; c = req->conn; if (!c->req_sent) { c->req_sent = TRUE; c->send_time = rspamd_get_ticks (FALSE); rspamd_http_connection_reset (c->http_conn); rspamd_http_connection_read_message (c->http_conn, c->req, &c->timeout); return 0; } else { if (rspamd_http_message_get_body (msg, NULL) == NULL || msg->code / 100 != 2) { err = g_error_new (RCLIENT_ERROR, msg->code, "HTTP error: %d, %.*s", msg->code, (gint)msg->status->len, msg->status->str); req->cb (c, msg, c->server_name->str, NULL, req->input, req->ud, c->start_time, c->send_time, err); g_error_free (err); return 0; } tok = rspamd_http_message_find_header (msg, "compression"); if (tok) { /* Need to uncompress */ rspamd_ftok_t t; t.begin = "zstd"; t.len = 4; if (rspamd_ftok_casecmp (tok, &t) == 0) { ZSTD_DStream *zstream; ZSTD_inBuffer zin; ZSTD_outBuffer zout; guchar *out; gsize outlen, r; zstream = ZSTD_createDStream (); ZSTD_initDStream (zstream); zin.pos = 0; zin.src = msg->body_buf.begin; zin.size = msg->body_buf.len; if ((outlen = ZSTD_getDecompressedSize (zin.src, zin.size)) == 0) { outlen = ZSTD_DStreamOutSize (); } out = g_malloc (outlen); zout.dst = out; zout.pos = 0; zout.size = outlen; while (zin.pos < zin.size) { r = ZSTD_decompressStream (zstream, &zout, &zin); if (ZSTD_isError (r)) { err = g_error_new (RCLIENT_ERROR, 500, "Decompression error: %s", ZSTD_getErrorName (r)); req->cb (c, msg, c->server_name->str, NULL, req->input, req->ud, c->start_time, c->send_time, err); g_error_free (err); ZSTD_freeDStream (zstream); g_free (out); return 0; } if (zout.pos == zout.size) { /* We need to extend output buffer */ zout.size = zout.size * 1.5 + 1.0; zout.dst = g_realloc (zout.dst, zout.size); } } ZSTD_freeDStream (zstream); parser = ucl_parser_new (0); if (!ucl_parser_add_chunk (parser, zout.dst, zout.pos)) { err = g_error_new (RCLIENT_ERROR, msg->code, "Cannot parse UCL: %s", ucl_parser_get_error (parser)); ucl_parser_free (parser); req->cb (c, msg, c->server_name->str, NULL, req->input, req->ud, c->start_time, c->send_time, err); g_error_free (err); g_free (zout.dst); return 0; } g_free (zout.dst); } else { err = g_error_new (RCLIENT_ERROR, 500, "Invalid compression method"); req->cb (c, msg, c->server_name->str, NULL, req->input, req->ud, c->start_time, c->send_time, err); g_error_free (err); return 0; } } else { parser = ucl_parser_new (0); if (!ucl_parser_add_chunk (parser, msg->body_buf.begin, msg->body_buf.len)) { err = g_error_new (RCLIENT_ERROR, msg->code, "Cannot parse UCL: %s", ucl_parser_get_error (parser)); ucl_parser_free (parser); req->cb (c, msg, c->server_name->str, NULL, req->input, req->ud, c->start_time, c->send_time, err); g_error_free (err); return 0; } } req->cb (c, msg, c->server_name->str, ucl_parser_get_object ( parser), req->input, req->ud, c->start_time, c->send_time, NULL); ucl_parser_free (parser); } return 0; }