int
main (int argc, char **argv)
{
struct rspamd_config *cfg;
rspamd_main = (struct rspamd_main *)g_malloc (sizeof (struct rspamd_main));
memset (rspamd_main, 0, sizeof (struct rspamd_main));
rspamd_main->server_pool = rspamd_mempool_new (rspamd_mempool_suggest_size (), NULL);
cfg = rspamd_config_new ();
rspamd_main->cfg = cfg;
cfg->cfg_pool = rspamd_mempool_new (rspamd_mempool_suggest_size (), NULL);
cfg->log_type = RSPAMD_LOG_CONSOLE;
cfg->log_level = G_LOG_LEVEL_INFO;
rspamd_set_logger (cfg, g_quark_from_static_string("rspamd-test"),
&rspamd_main->logger, rspamd_main->server_pool);
(void)rspamd_log_open (rspamd_main->logger);
g_test_init (&argc, &argv, NULL);
cfg->libs_ctx = rspamd_init_libs ();
base = event_init ();
rspamd_stat_init (cfg, base);
if (g_test_verbose ()) {
cfg->log_level = G_LOG_LEVEL_DEBUG;
rspamd_set_logger (cfg, g_quark_from_static_string("rspamd-test"),
&rspamd_main->logger, rspamd_main->server_pool);
(void)rspamd_log_reopen (rspamd_main->logger);
}
g_log_set_default_handler (rspamd_glib_log_function, rspamd_main->logger);
g_test_add_func ("/rspamd/mem_pool", rspamd_mem_pool_test_func);
g_test_add_func ("/rspamd/radix", rspamd_radix_test_func);
g_test_add_func ("/rspamd/dns", rspamd_dns_test_func);
g_test_add_func ("/rspamd/dkim", rspamd_dkim_test_func);
g_test_add_func ("/rspamd/rrd", rspamd_rrd_test_func);
g_test_add_func ("/rspamd/upstream", rspamd_upstream_test_func);
g_test_add_func ("/rspamd/shingles", rspamd_shingles_test_func);
g_test_add_func ("/rspamd/http", rspamd_http_test_func);
g_test_add_func ("/rspamd/lua", rspamd_lua_test_func);
g_test_add_func ("/rspamd/cryptobox", rspamd_cryptobox_test_func);
g_test_add_func ("/rspamd/heap", rspamd_heap_test_func);
#if 0
g_test_add_func ("/rspamd/url", rspamd_url_test_func);
g_test_add_func ("/rspamd/statfile", rspamd_statfile_test_func);
g_test_add_func ("/rspamd/aio", rspamd_async_test_func);
#endif
g_test_run ();
g_mime_shutdown ();
rspamd_regexp_library_finalize ();
return 0;
}
gint
dkim_module_reconfig (struct rspamd_config *cfg)
{
struct module_ctx saved_ctx;
saved_ctx = dkim_module_ctx->ctx;
rspamd_mempool_delete (dkim_module_ctx->dkim_pool);
radix_destroy_compressed (dkim_module_ctx->whitelist_ip);
if (dkim_module_ctx->dkim_domains) {
g_hash_table_destroy (dkim_module_ctx->dkim_domains);
}
memset (dkim_module_ctx, 0, sizeof (*dkim_module_ctx));
dkim_module_ctx->ctx = saved_ctx;
dkim_module_ctx->dkim_pool = rspamd_mempool_new (rspamd_mempool_suggest_size (), "dkim");
dkim_module_ctx->sign_headers = "from:sender:reply-to:subject:date:message-id:"
"to:cc:mime-version:content-type:content-transfer-encoding:"
"resent-to:resent-cc:resent-from:resent-sender:resent-message-id:"
"in-reply-to:references:list-id:list-owner:list-unsubscribe:"
"list-subscribe:list-post";
dkim_module_ctx->sign_condition_ref = -1;
dkim_module_ctx->max_sigs = DEFAULT_MAX_SIGS;
return dkim_module_config (cfg);
}
gint
chartable_module_reconfig (struct rspamd_config *cfg)
{
rspamd_mempool_delete (chartable_module_ctx->chartable_pool);
chartable_module_ctx->chartable_pool = rspamd_mempool_new (1024);
return chartable_module_config (cfg);
}
/* Init function */
gint
regexp_module_init (struct rspamd_config *cfg, struct module_ctx **ctx)
{
regexp_module_ctx = g_malloc (sizeof (struct regexp_ctx));
regexp_module_ctx->regexp_pool = rspamd_mempool_new (rspamd_mempool_suggest_size (), NULL);
*ctx = (struct module_ctx *)regexp_module_ctx;
return 0;
}
gint
chartable_module_init (struct rspamd_config *cfg, struct module_ctx **ctx)
{
chartable_module_ctx = g_malloc (sizeof (struct chartable_ctx));
chartable_module_ctx->chartable_pool = rspamd_mempool_new (rspamd_mempool_suggest_size (), NULL);
*ctx = (struct module_ctx *)chartable_module_ctx;
return 0;
}
void
rspamd_dns_test_func ()
{
struct rspamd_config *cfg;
rspamd_mempool_t *pool;
struct rspamd_async_session *s;
cfg = (struct rspamd_config *)g_malloc (sizeof (struct rspamd_config));
bzero (cfg, sizeof (struct rspamd_config));
cfg->cfg_pool = rspamd_mempool_new (rspamd_mempool_suggest_size ());
cfg->dns_retransmits = 2;
cfg->dns_timeout = 0.5;
pool = rspamd_mempool_new (rspamd_mempool_suggest_size ());
s = new_async_session (pool, session_fin, NULL, NULL, NULL);
resolver = dns_resolver_init (NULL, base, cfg);
requests ++;
g_assert (make_dns_request (resolver, s, pool, test_dns_cb, NULL, RDNS_REQUEST_A, "google.com"));
requests ++;
g_assert (make_dns_request (resolver, s, pool, test_dns_cb, NULL, RDNS_REQUEST_PTR, "81.19.70.3"));
requests ++;
g_assert (make_dns_request (resolver, s, pool, test_dns_cb, NULL, RDNS_REQUEST_MX, "rambler.ru"));
requests ++;
g_assert (make_dns_request (resolver, s, pool, test_dns_cb, NULL, RDNS_REQUEST_TXT, "rambler.ru"));
requests ++;
g_assert (make_dns_request (resolver, s, pool, test_dns_cb, NULL, RDNS_REQUEST_TXT, "google.com"));
requests ++;
g_assert (make_dns_request (resolver, s, pool, test_dns_cb, NULL, RDNS_REQUEST_SPF, "rambler.ru"));
requests ++;
g_assert (make_dns_request (resolver, s, pool, test_dns_cb, NULL, RDNS_REQUEST_SRV, "_xmpp-server._tcp.jabber.org"));
requests ++;
g_assert (make_dns_request (resolver, s, pool, test_dns_cb, NULL, RDNS_REQUEST_TXT, "non-existent.arpa"));
g_assert (resolver != NULL);
event_loop (0);
}
gint
dkim_module_init (struct rspamd_config *cfg, struct module_ctx **ctx)
{
dkim_module_ctx = g_malloc0 (sizeof (struct dkim_ctx));
dkim_module_ctx->dkim_pool = rspamd_mempool_new (
rspamd_mempool_suggest_size ());
*ctx = (struct module_ctx *)dkim_module_ctx;
return 0;
}
void
rspamd_init_cfg (struct rspamd_config *cfg, gboolean init_lua)
{
cfg->cfg_pool = rspamd_mempool_new (rspamd_mempool_suggest_size (), "cfg");
rspamd_config_defaults (cfg);
if (init_lua) {
cfg->lua_state = rspamd_lua_init (cfg);
rspamd_mempool_add_destructor (cfg->cfg_pool,
(rspamd_mempool_destruct_t)lua_close, cfg->lua_state);
}
}
gint
regexp_module_reconfig (struct rspamd_config *cfg)
{
struct module_ctx saved_ctx;
saved_ctx = regexp_module_ctx->ctx;
rspamd_mempool_delete (regexp_module_ctx->regexp_pool);
memset (regexp_module_ctx, 0, sizeof (*regexp_module_ctx));
regexp_module_ctx->ctx = saved_ctx;
regexp_module_ctx->regexp_pool = rspamd_mempool_new (rspamd_mempool_suggest_size (), NULL);
return regexp_module_config (cfg);
}
gint
fuzzy_check_module_init (struct rspamd_config *cfg, struct module_ctx **ctx)
{
fuzzy_module_ctx = g_malloc0 (sizeof (struct fuzzy_ctx));
fuzzy_module_ctx->fuzzy_pool = rspamd_mempool_new (
rspamd_mempool_suggest_size ());
fuzzy_module_ctx->cfg = cfg;
*ctx = (struct module_ctx *)fuzzy_module_ctx;
return 0;
}
gint
fuzzy_check_module_init (struct rspamd_config *cfg, struct module_ctx **ctx)
{
fuzzy_module_ctx = g_malloc0 (sizeof (struct fuzzy_ctx));
fuzzy_module_ctx->fuzzy_pool = rspamd_mempool_new (rspamd_mempool_suggest_size (), NULL);
fuzzy_module_ctx->cfg = cfg;
/* TODO: this should match rules count actually */
fuzzy_module_ctx->keypairs_cache = rspamd_keypair_cache_new (32);
*ctx = (struct module_ctx *)fuzzy_module_ctx;
return 0;
}
gint
dkim_module_reconfig (struct rspamd_config *cfg)
{
rspamd_mempool_delete (dkim_module_ctx->dkim_pool);
radix_destroy_compressed (dkim_module_ctx->whitelist_ip);
if (dkim_module_ctx->dkim_domains) {
g_hash_table_destroy (dkim_module_ctx->dkim_domains);
}
memset (dkim_module_ctx, 0, sizeof (*dkim_module_ctx));
dkim_module_ctx->dkim_pool = rspamd_mempool_new (
rspamd_mempool_suggest_size ());
return dkim_module_config (cfg);
}
void
rspamd_dkim_test_func ()
{
#if 0
rspamd_dkim_context_t *ctx;
rspamd_dkim_key_t *key;
rspamd_mempool_t *pool;
struct rspamd_dns_resolver *resolver;
struct rspamd_config *cfg;
GError *err = NULL;
struct rspamd_async_session *s;
cfg = (struct rspamd_config *)g_malloc (sizeof (struct rspamd_config));
bzero (cfg, sizeof (struct rspamd_config));
cfg->cfg_pool = rspamd_mempool_new (rspamd_mempool_suggest_size ());
cfg->dns_retransmits = 2;
cfg->dns_timeout = 0.5;
pool = rspamd_mempool_new (rspamd_mempool_suggest_size ());
resolver = dns_resolver_init (NULL, base, cfg);
g_assert (resolver != NULL);
ctx = rspamd_create_dkim_context (test_dkim_sig, pool, 0, &err);
g_assert (ctx != NULL);
/* Key part */
s = rspamd_session_create (pool, session_fin, NULL, NULL, NULL);
g_assert (rspamd_get_dkim_key (ctx, resolver, s, test_key_handler, s));
event_base_loop (base, 0);
#endif
}
radix_compressed_t *
radix_create_compressed (void)
{
radix_compressed_t *tree;
tree = g_slice_alloc (sizeof (*tree));
if (tree == NULL) {
return NULL;
}
tree->pool = rspamd_mempool_new (rspamd_mempool_suggest_size (), NULL);
tree->size = 0;
tree->tree = btrie_init (tree->pool);
return tree;
}
struct symbols_cache*
rspamd_symbols_cache_new (void)
{
struct symbols_cache *cache;
cache = g_slice_alloc0 (sizeof (struct symbols_cache));
cache->static_pool =
rspamd_mempool_new (rspamd_mempool_suggest_size ());
cache->items_by_symbol = g_hash_table_new (rspamd_str_hash,
rspamd_str_equal);
cache->items_by_order = g_ptr_array_new ();
cache->items_by_id = g_ptr_array_new ();
cache->mtx = rspamd_mempool_get_mutex (cache->static_pool);
cache->reload_time = CACHE_RELOAD_TIME;
cache->total_freq = 1;
cache->max_weight = 1.0;
return cache;
}
void
rspamd_fuzzy_test_func ()
{
rspamd_mempool_t *pool;
rspamd_fuzzy_t *h1, *h2, *h3, *h4, *h5;
rspamd_fstring_t f1, f2, f3, f4, f5;
int diff2;
pool = rspamd_mempool_new (1024);
f1.begin = s1;
f1.len = strlen (s1);
f2.begin = s2;
f2.len = strlen (s2);
f3.begin = s3;
f3.len = strlen (s3);
f4.begin = s4;
f4.len = strlen (s4);
f5.begin = s5;
f5.len = strlen (s5);
h1 = rspamd_fuzzy_init (&f1, pool);
h2 = rspamd_fuzzy_init (&f2, pool);
h3 = rspamd_fuzzy_init (&f3, pool);
h4 = rspamd_fuzzy_init (&f4, pool);
h5 = rspamd_fuzzy_init (&f5, pool);
diff2 = rspamd_fuzzy_compare (h2, h5);
msg_debug ("rspamd_fuzzy_test_func: s1, s2 difference between strings is %d", rspamd_fuzzy_compare (h1, h2));
msg_debug ("rspamd_fuzzy_test_func: s1, s3 difference between strings is %d", rspamd_fuzzy_compare (h1, h3));
msg_debug ("rspamd_fuzzy_test_func: s3, s4 difference between strings is %d", rspamd_fuzzy_compare (h3, h4));
msg_debug ("rspamd_fuzzy_test_func: s2, s4 difference between strings is %d", rspamd_fuzzy_compare (h2, h4));
msg_debug ("rspamd_fuzzy_test_func: s2, s5 difference between strings is %d", diff2);
/* Identical strings */
if (diff2 != 100) {
msg_err ("hash difference is %d", diff2);
g_assert (diff2 == 100);
}
rspamd_mempool_delete (pool);
}
/***
* @function url.create([mempool,] str)
* @param {rspamd_mempool} memory pool for URL, e.g. `task:get_mempool()`
* @param {string} text that contains URL (can also contain other stuff)
* @return {url} new url object that exists as long as the corresponding mempool exists
*/
static gint
lua_url_create (lua_State *L)
{
rspamd_mempool_t *pool;
const gchar *text;
size_t length;
gboolean own_pool = FALSE;
if (lua_type (L, 1) == LUA_TUSERDATA) {
pool = rspamd_lua_check_mempool (L, 1);
text = luaL_checklstring (L, 2, &length);
}
else {
own_pool = TRUE;
pool = rspamd_mempool_new (rspamd_mempool_suggest_size (), "url");
text = luaL_checklstring (L, 1, &length);
}
if (pool == NULL || text == NULL) {
if (own_pool && pool) {
rspamd_mempool_delete (pool);
}
return luaL_error (L, "invalid arguments");
}
else {
rspamd_url_find_single (pool, text, length, FALSE,
lua_url_single_inserter, L);
if (lua_type (L, -1) != LUA_TUSERDATA) {
/* URL is actually not found */
lua_pushnil (L);
}
}
if (own_pool && pool) {
rspamd_mempool_delete (pool);
}
return 1;
}
void
rspamd_statfile_test_func ()
{
/*
* XXX: broken, old, need to be rewritten
*/
#if 0
statfile_pool_t *pool;
rspamd_mempool_t *p;
stat_file_t *st;
uint32_t random_hashes[HASHES_NUM], i, v;
time_t now = time (NULL);
p = rspamd_mempool_new (rspamd_mempool_suggest_size ());
umask (S_IWGRP | S_IWOTH);
pool = statfile_pool_new (p, TRUE);
for (i = 0; i < HASHES_NUM; i ++) {
random_hashes[i] = ottery_rand_uint32 ();
}
/* Create new file */
g_assert (rspamd_mmaped_file_create (pool, TEST_FILENAME, 65535) != -1);
g_assert ((st = rspamd_mmaped_file_open (pool, TEST_FILENAME, 65535, FALSE)) != NULL);
/* Get and set random blocks */
rspamd_mmaped_file_lock_file (pool, st);
for (i = 0; i < HASHES_NUM; i ++) {
rspamd_mmaped_file_set_block (pool, st, random_hashes[i], random_hashes[i], now, 1.0);
}
rspamd_mmaped_file_unlock_file (pool, st);
for (i = 0; i < HASHES_NUM; i ++) {
v = rspamd_mmaped_file_get_block (pool, st, random_hashes[i], random_hashes[i], now);
g_assert(v == 1.0);
}
rspamd_mmaped_file_destroy (pool);
#endif
}
void
register_symbol_common (struct symbols_cache **cache,
const gchar *name,
double weight,
gint priority,
symbol_func_t func,
gpointer user_data,
enum rspamd_symbol_type type)
{
struct cache_item *item = NULL;
struct symbols_cache *pcache = *cache;
GList **target, *cur;
struct metric *m;
struct rspamd_symbol_def *s;
gboolean skipped;
if (*cache == NULL) {
pcache = g_new0 (struct symbols_cache, 1);
*cache = pcache;
pcache->static_pool =
rspamd_mempool_new (rspamd_mempool_suggest_size ());
pcache->items_by_symbol = g_hash_table_new (rspamd_str_hash,
rspamd_str_equal);
}
gint
main (gint argc, gchar **argv, gchar **env)
{
GError *error = NULL;
GOptionContext *context;
GOptionGroup *og;
struct rspamd_config *cfg;
GQuark process_quark;
gchar **nargv, **targv;
const gchar *cmd_name;
const struct rspamadm_command *cmd;
gint i, nargc, targc;
ucl_vars = g_hash_table_new_full (rspamd_strcase_hash,
rspamd_strcase_equal, g_free, g_free);
process_quark = g_quark_from_static_string ("rspamadm");
cfg = rspamd_config_new ();
cfg->libs_ctx = rspamd_init_libs ();
rspamd_main = g_malloc0 (sizeof (*rspamd_main));
rspamd_main->cfg = cfg;
rspamd_main->pid = getpid ();
rspamd_main->type = process_quark;
rspamd_main->server_pool = rspamd_mempool_new (rspamd_mempool_suggest_size (),
"rspamadm");
cfg->log_level = G_LOG_LEVEL_WARNING;
cfg->log_type = RSPAMD_LOG_CONSOLE;
rspamd_set_logger (cfg, process_quark, rspamd_main);
(void) rspamd_log_open (rspamd_main->logger);
g_log_set_default_handler (rspamd_glib_log_function, rspamd_main->logger);
g_set_printerr_handler (rspamd_glib_printerr_function);
rspamd_config_post_load (cfg, FALSE);
/* Setup logger */
if (verbose) {
cfg->log_level = G_LOG_LEVEL_DEBUG;
}
else {
cfg->log_level = G_LOG_LEVEL_INFO;
}
gperf_profiler_init (cfg, "rspamadm");
setproctitle ("rspamdadm");
/* Now read options and store everything till the first non-dash argument */
nargv = g_malloc0 (sizeof (gchar *) * (argc + 1));
nargv[0] = g_strdup (argv[0]);
for (i = 1, nargc = 1; i < argc; i ++) {
if (argv[i] && argv[i][0] == '-') {
/* Copy to nargv */
nargv[nargc] = g_strdup (argv[i]);
nargc ++;
}
else {
break;
}
}
context = g_option_context_new ("command - rspamd administration utility");
og = g_option_group_new ("global", "global options", "global options",
NULL, NULL);
g_option_context_set_help_enabled (context, FALSE);
g_option_group_add_entries (og, entries);
g_option_context_set_summary (context,
"Summary:\n Rspamd administration utility version "
RVERSION
"\n Release id: "
RID);
g_option_context_set_main_group (context, og);
targv = nargv;
targc = nargc;
if (!g_option_context_parse (context, &targc, &targv, &error)) {
fprintf (stderr, "option parsing failed: %s\n", error->message);
g_error_free (error);
exit (1);
}
g_strfreev (nargv);
if (show_version) {
rspamadm_version ();
exit (EXIT_SUCCESS);
}
if (show_help) {
rspamadm_usage (context);
exit (EXIT_SUCCESS);
}
if (list_commands) {
rspamadm_commands ();
exit (EXIT_SUCCESS);
}
cmd_name = argv[nargc];
if (cmd_name == NULL) {
cmd_name = "help";
}
cmd = rspamadm_search_command (cmd_name);
if (cmd == NULL) {
fprintf (stderr, "Invalid command name: %s\n", cmd_name);
exit (EXIT_FAILURE);
}
if (nargc < argc) {
nargv = g_malloc0 (sizeof (gchar *) * (argc - nargc + 1));
nargv[0] = g_strdup_printf ("%s %s", argv[0], cmd_name);
for (i = 1; i < argc - nargc; i ++) {
nargv[i] = g_strdup (argv[i + nargc]);
}
targc = argc - nargc;
targv = nargv;
cmd->run (targc, targv);
g_strfreev (nargv);
}
else {
cmd->run (0, NULL);
}
rspamd_log_close (rspamd_main->logger);
REF_RELEASE (rspamd_main->cfg);
g_free (rspamd_main);
return 0;
}
gint
dkim_module_init (struct rspamd_config *cfg, struct module_ctx **ctx)
{
dkim_module_ctx = g_malloc0 (sizeof (struct dkim_ctx));
dkim_module_ctx->dkim_pool = rspamd_mempool_new (rspamd_mempool_suggest_size (), "dkim");
dkim_module_ctx->sign_headers = "from:sender:reply-to:subject:date:message-id:"
"to:cc:mime-version:content-type:content-transfer-encoding:"
"resent-to:resent-cc:resent-from:resent-sender:resent-message-id:"
"in-reply-to:references:list-id:list-owner:list-unsubscribe:"
"list-subscribe:list-post";
dkim_module_ctx->sign_condition_ref = -1;
dkim_module_ctx->max_sigs = DEFAULT_MAX_SIGS;
*ctx = (struct module_ctx *)dkim_module_ctx;
rspamd_rcl_add_doc_by_path (cfg,
NULL,
"DKIM check plugin",
"dkim",
UCL_OBJECT,
NULL,
0,
NULL,
0);
rspamd_rcl_add_doc_by_path (cfg,
"dkim",
"Map of IP addresses that should be excluded from DKIM checks",
"whitelist",
UCL_STRING,
NULL,
0,
NULL,
0);
rspamd_rcl_add_doc_by_path (cfg,
"dkim",
"Symbol that is added if DKIM check is successful",
"symbol_allow",
UCL_STRING,
NULL,
0,
NULL,
0);
rspamd_rcl_add_doc_by_path (cfg,
"dkim",
"Symbol that is added if DKIM check is unsuccessful",
"symbol_reject",
UCL_STRING,
NULL,
0,
NULL,
0);
rspamd_rcl_add_doc_by_path (cfg,
"dkim",
"Symbol that is added if DKIM check can't be completed (e.g. DNS failure)",
"symbol_tempfail",
UCL_STRING,
NULL,
0,
NULL,
0);
rspamd_rcl_add_doc_by_path (cfg,
"dkim",
"Symbol that is added if mail is not signed",
"symbol_na",
UCL_STRING,
NULL,
0,
NULL,
0);
rspamd_rcl_add_doc_by_path (cfg,
"dkim",
"Size of DKIM keys cache",
"dkim_cache_size",
UCL_INT,
NULL,
0,
NULL,
0);
rspamd_rcl_add_doc_by_path (cfg,
"dkim",
"Allow this time difference when checking DKIM signature time validity",
"time_jitter",
UCL_TIME,
NULL,
0,
NULL,
0);
rspamd_rcl_add_doc_by_path (cfg,
"dkim",
"Domains to check DKIM for (check all domains if this option is empty)",
"domains",
UCL_STRING,
NULL,
0,
NULL,
0);
rspamd_rcl_add_doc_by_path (cfg,
"dkim",
"Map of domains that are treated as 'trusted' meaning that DKIM policy failure has more significant score",
"trusted_domains",
UCL_STRING,
NULL,
0,
NULL,
0);
rspamd_rcl_add_doc_by_path (cfg,
"dkim",
"Multiply dkim score by this factor for trusted domains",
"strict_multiplier",
UCL_FLOAT,
NULL,
0,
NULL,
0);
rspamd_rcl_add_doc_by_path (cfg,
"dkim",
"Check DKIM policies merely for `trusted_domains`",
"trusted_only",
UCL_BOOLEAN,
NULL,
0,
NULL,
0);
rspamd_rcl_add_doc_by_path (cfg,
"dkim",
"Do not check messages with multiple DKIM signatures",
"skip_multi",
UCL_BOOLEAN,
NULL,
0,
NULL,
0);
rspamd_rcl_add_doc_by_path (cfg,
"dkim",
"Lua script that tells if a message should be signed and with what params",
"sign_condition",
UCL_STRING,
NULL,
0,
NULL,
0);
rspamd_rcl_add_doc_by_path (cfg,
"dkim",
"Maximum number of DKIM signatures to check",
"max_sigs",
UCL_INT,
NULL,
0,
NULL,
0);
return 0;
}
void
rspamd_http_test_func (void)
{
struct event_base *ev_base = event_init ();
rspamd_mempool_t *pool = rspamd_mempool_new (rspamd_mempool_suggest_size ());
gpointer serv_key, client_key, peer_key;
struct rspamd_keypair_cache *c;
rspamd_mempool_mutex_t *mtx;
rspamd_inet_addr_t addr;
struct timespec ts1, ts2;
gchar filepath[PATH_MAX], buf[512];
gint fd, i, j;
pid_t sfd;
GString *b32_key;
double diff, total_diff = 0.0, latency[pconns * ntests], mean, std;
rspamd_cryptobox_init ();
rspamd_snprintf (filepath, sizeof (filepath), "/tmp/http-test-XXXXXX");
g_assert ((fd = mkstemp (filepath)) != -1);
for (i = 0; i < file_blocks; i ++) {
memset (buf, 0, sizeof (buf));
g_assert (write (fd, buf, sizeof (buf)) == sizeof (buf));
}
mtx = rspamd_mempool_get_mutex (pool);
rspamd_parse_inet_address (&addr, "127.0.0.1");
rspamd_inet_address_set_port (&addr, ottery_rand_range (30000) + 32768);
serv_key = rspamd_http_connection_gen_key ();
client_key = rspamd_http_connection_gen_key ();
c = rspamd_keypair_cache_new (16);
rspamd_mempool_lock_mutex (mtx);
sfd = fork ();
g_assert (sfd != -1);
if (sfd == 0) {
rspamd_http_server_func ("/tmp/", &addr, mtx, serv_key, c);
exit (EXIT_SUCCESS);
}
rspamd_mempool_lock_mutex (mtx);
/* Do client stuff */
for (i = 0; i < ntests; i ++) {
for (j = 0; j < pconns; j ++) {
rspamd_http_client_func (filepath + sizeof ("/tmp") - 1, &addr,
NULL, NULL, c, ev_base, &latency[i * pconns + j]);
}
clock_gettime (CLOCK_MONOTONIC, &ts1);
event_base_loop (ev_base, 0);
clock_gettime (CLOCK_MONOTONIC, &ts2);
diff = (ts2.tv_sec - ts1.tv_sec) * 1000. + /* Seconds */
(ts2.tv_nsec - ts1.tv_nsec) / 1000000.; /* Nanoseconds */
total_diff += diff;
}
msg_info ("Made %d connections of size %d in %.6f ms, %.6f cps",
ntests * pconns,
sizeof (buf) * file_blocks,
total_diff, ntests * pconns / total_diff * 1000.);
mean = rspamd_http_calculate_mean (latency, &std);
msg_info ("Latency: %.6f ms mean, %.6f dev",
mean, std);
/* Now test encrypted */
b32_key = rspamd_http_connection_print_key (serv_key,
RSPAMD_KEYPAIR_PUBKEY|RSPAMD_KEYPAIR_BASE32);
g_assert (b32_key != NULL);
peer_key = rspamd_http_connection_make_peer_key (b32_key->str);
g_assert (peer_key != NULL);
total_diff = 0.0;
for (i = 0; i < ntests; i ++) {
for (j = 0; j < pconns; j ++) {
rspamd_http_client_func (filepath + sizeof ("/tmp") - 1, &addr,
client_key, peer_key, c, ev_base, &latency[i * pconns + j]);
}
clock_gettime (CLOCK_MONOTONIC, &ts1);
event_base_loop (ev_base, 0);
clock_gettime (CLOCK_MONOTONIC, &ts2);
diff = (ts2.tv_sec - ts1.tv_sec) * 1000. + /* Seconds */
(ts2.tv_nsec - ts1.tv_nsec) / 1000000.; /* Nanoseconds */
total_diff += diff;
}
msg_info ("Made %d encrypted connections of size %d in %.6f ms, %.6f cps",
ntests * pconns,
sizeof (buf) * file_blocks,
total_diff, ntests * pconns / total_diff * 1000.);
mean = rspamd_http_calculate_mean (latency, &std);
msg_info ("Latency: %.6f ms mean, %.6f dev",
mean, std);
/* Restart server */
kill (sfd, SIGTERM);
wait (&i);
sfd = fork ();
g_assert (sfd != -1);
if (sfd == 0) {
rspamd_http_server_func ("/tmp/", &addr, mtx, serv_key, NULL);
exit (EXIT_SUCCESS);
}
rspamd_mempool_lock_mutex (mtx);
total_diff = 0.0;
for (i = 0; i < ntests; i ++) {
for (j = 0; j < pconns; j ++) {
rspamd_http_client_func (filepath + sizeof ("/tmp") - 1, &addr,
client_key, peer_key, c, ev_base, &latency[i * pconns + j]);
}
clock_gettime (CLOCK_MONOTONIC, &ts1);
event_base_loop (ev_base, 0);
clock_gettime (CLOCK_MONOTONIC, &ts2);
diff = (ts2.tv_sec - ts1.tv_sec) * 1000. + /* Seconds */
(ts2.tv_nsec - ts1.tv_nsec) / 1000000.; /* Nanoseconds */
total_diff += diff;
}
msg_info ("Made %d uncached encrypted connections of size %d in %.6f ms, %.6f cps",
ntests * pconns,
sizeof (buf) * file_blocks,
total_diff, ntests * pconns / total_diff * 1000.);
mean = rspamd_http_calculate_mean (latency, &std);
msg_info ("Latency: %.6f ms mean, %.6f dev",
mean, std);
close (fd);
unlink (filepath);
kill (sfd, SIGTERM);
}
struct rspamd_map *
rspamd_map_add (struct rspamd_config *cfg,
const gchar *map_line,
const gchar *description,
map_cb_t read_callback,
map_fin_cb_t fin_callback,
void **user_data)
{
struct rspamd_map *new_map;
const gchar *def;
struct file_map_data *fdata;
struct http_map_data *hdata;
gchar *cksum_encoded, cksum[rspamd_cryptobox_HASHBYTES];
rspamd_mempool_t *pool;
struct http_parser_url up;
rspamd_ftok_t tok;
if (cfg->map_pool == NULL) {
cfg->map_pool = rspamd_mempool_new (rspamd_mempool_suggest_size (),
"map");
memcpy (cfg->map_pool->tag.uid, cfg->cfg_pool->tag.uid,
sizeof (cfg->map_pool->tag.uid));
}
new_map = rspamd_mempool_alloc0 (cfg->map_pool, sizeof (struct rspamd_map));
/* First of all detect protocol line */
if (rspamd_map_check_proto (cfg, map_line, new_map) == NULL) {
return NULL;
}
new_map->read_callback = read_callback;
new_map->fin_callback = fin_callback;
new_map->user_data = user_data;
new_map->cfg = cfg;
new_map->id = g_random_int ();
new_map->locked =
rspamd_mempool_alloc0_shared (cfg->cfg_pool, sizeof (gint));
def = new_map->uri;
if (description != NULL) {
new_map->description =
rspamd_mempool_strdup (cfg->cfg_pool, description);
}
/* Now check for each proto separately */
if (new_map->protocol == MAP_PROTO_FILE) {
fdata =
rspamd_mempool_alloc0 (cfg->map_pool,
sizeof (struct file_map_data));
if (access (def, R_OK) == -1) {
if (errno != ENOENT) {
msg_err_config ("cannot open file '%s': %s", def, strerror
(errno));
return NULL;
}
msg_info_config (
"map '%s' is not found, but it can be loaded automatically later",
def);
/* We still can add this file */
fdata->st.st_mtime = -1;
}
else {
stat (def, &fdata->st);
}
fdata->filename = rspamd_mempool_strdup (cfg->map_pool, def);
new_map->map_data = fdata;
}
else if (new_map->protocol == MAP_PROTO_HTTP) {
hdata =
rspamd_mempool_alloc0 (cfg->map_pool,
sizeof (struct http_map_data));
memset (&up, 0, sizeof (up));
if (http_parser_parse_url (new_map->uri, strlen (new_map->uri), FALSE,
&up) != 0) {
msg_err_config ("cannot parse HTTP url: %s", new_map->uri);
return NULL;
}
else {
if (!(up.field_set & 1 << UF_HOST)) {
msg_err_config ("cannot parse HTTP url: %s: no host", new_map->uri);
return NULL;
}
tok.begin = new_map->uri + up.field_data[UF_HOST].off;
tok.len = up.field_data[UF_HOST].len;
hdata->host = rspamd_mempool_ftokdup (cfg->map_pool, &tok);
if (up.field_set & 1 << UF_PORT) {
hdata->port = up.port;
}
else {
hdata->port = 80;
}
if (up.field_set & 1 << UF_PATH) {
tok.begin = new_map->uri + up.field_data[UF_PATH].off;
tok.len = strlen (tok.begin);
hdata->path = rspamd_mempool_ftokdup (cfg->map_pool, &tok);
}
}
new_map->map_data = hdata;
}
/* Temp pool */
rspamd_cryptobox_hash (cksum, new_map->uri, strlen (new_map->uri), NULL, 0);
cksum_encoded = rspamd_encode_base32 (cksum, sizeof (cksum));
new_map->pool = rspamd_mempool_new (rspamd_mempool_suggest_size (), "map");
rspamd_strlcpy (new_map->pool->tag.uid, cksum_encoded,
sizeof (new_map->pool->tag.uid));
g_free (cksum_encoded);
pool = new_map->pool;
msg_info_pool ("added map %s", new_map->uri);
cfg->maps = g_list_prepend (cfg->maps, new_map);
return new_map;
}
void
rspamd_http_test_func (void)
{
struct event_base *ev_base = event_init ();
rspamd_mempool_t *pool = rspamd_mempool_new (rspamd_mempool_suggest_size (), NULL);
struct rspamd_cryptobox_keypair *serv_key, *client_key;
struct rspamd_cryptobox_pubkey *peer_key;
struct rspamd_keypair_cache *c;
rspamd_mempool_mutex_t *mtx;
rspamd_inet_addr_t *addr;
gdouble ts1, ts2;
gchar filepath[PATH_MAX], *buf;
gchar *env;
gint fd;
guint i, j;
pid_t *sfd;
GString *b32_key;
double diff, total_diff = 0.0, *latency, mean, std;
/* Read environment */
if ((env = getenv ("RSPAMD_HTTP_CONNS")) != NULL) {
pconns = strtoul (env, NULL, 10);
}
else {
return;
}
if ((env = getenv ("RSPAMD_HTTP_TESTS")) != NULL) {
ntests = strtoul (env, NULL, 10);
}
if ((env = getenv ("RSPAMD_HTTP_SIZE")) != NULL) {
file_size = strtoul (env, NULL, 10);
}
if ((env = getenv ("RSPAMD_HTTP_SERVERS")) != NULL) {
nservers = strtoul (env, NULL, 10);
}
rspamd_cryptobox_init ();
rspamd_snprintf (filepath, sizeof (filepath), "/tmp/http-test-XXXXXX");
g_assert ((fd = mkstemp (filepath)) != -1);
sfd = g_alloca (sizeof (*sfd) * nservers);
latency = g_malloc0 (pconns * ntests * sizeof (gdouble));
buf = g_malloc (file_size);
memset (buf, 0, file_size);
g_assert (write (fd, buf, file_size) == file_size);
g_free (buf);
mtx = rspamd_mempool_get_mutex (pool);
rspamd_parse_inet_address (&addr, "127.0.0.1", 0);
rspamd_inet_address_set_port (addr, 43898);
serv_key = rspamd_keypair_new (RSPAMD_KEYPAIR_KEX,
RSPAMD_CRYPTOBOX_MODE_25519);
client_key = rspamd_keypair_new (RSPAMD_KEYPAIR_KEX,
RSPAMD_CRYPTOBOX_MODE_25519);
c = rspamd_keypair_cache_new (16);
rspamd_http_start_servers (sfd, addr, serv_key, NULL);
usleep (100000);
/* Do client stuff */
gperf_profiler_init (NULL, "plain-http-client");
for (i = 0; i < ntests; i ++) {
for (j = 0; j < pconns; j ++) {
rspamd_http_client_func (filepath + sizeof ("/tmp") - 1, addr,
NULL, NULL, c, ev_base, &latency[i * pconns + j]);
}
ts1 = rspamd_get_ticks ();
event_base_loop (ev_base, 0);
ts2 = rspamd_get_ticks ();
diff = (ts2 - ts1) * 1000.0;
total_diff += diff;
}
gperf_profiler_stop ();
msg_info ("Made %d connections of size %d in %.6f ms, %.6f cps",
ntests * pconns,
file_size,
total_diff, ntests * pconns / total_diff * 1000.);
mean = rspamd_http_calculate_mean (latency, &std);
msg_info ("Latency: %.6f ms mean, %.6f dev",
mean, std);
rspamd_http_stop_servers (sfd);
rspamd_http_start_servers (sfd, addr, serv_key, c);
//rspamd_mempool_lock_mutex (mtx);
usleep (100000);
b32_key = rspamd_keypair_print (serv_key,
RSPAMD_KEYPAIR_PUBKEY|RSPAMD_KEYPAIR_BASE32);
g_assert (b32_key != NULL);
peer_key = rspamd_pubkey_from_base32 (b32_key->str, b32_key->len,
RSPAMD_KEYPAIR_KEX, RSPAMD_CRYPTOBOX_MODE_25519);
g_assert (peer_key != NULL);
total_diff = 0.0;
gperf_profiler_init (NULL, "cached-http-client");
for (i = 0; i < ntests; i ++) {
for (j = 0; j < pconns; j ++) {
rspamd_http_client_func (filepath + sizeof ("/tmp") - 1, addr,
client_key, peer_key, c, ev_base, &latency[i * pconns + j]);
}
ts1 = rspamd_get_ticks ();
event_base_loop (ev_base, 0);
ts2 = rspamd_get_ticks ();
diff = (ts2 - ts1) * 1000.0;
total_diff += diff;
}
gperf_profiler_stop ();
msg_info ("Made %d encrypted connections of size %d in %.6f ms, %.6f cps",
ntests * pconns,
file_size,
total_diff, ntests * pconns / total_diff * 1000.);
mean = rspamd_http_calculate_mean (latency, &std);
msg_info ("Latency: %.6f ms mean, %.6f dev",
mean, std);
/* Restart server */
rspamd_http_stop_servers (sfd);
/* No keypairs cache */
rspamd_http_start_servers (sfd, addr, serv_key, NULL);
usleep (100000);
total_diff = 0.0;
gperf_profiler_init (NULL, "fair-http-client");
for (i = 0; i < ntests; i ++) {
for (j = 0; j < pconns; j ++) {
rspamd_http_client_func (filepath + sizeof ("/tmp") - 1, addr,
client_key, peer_key, c, ev_base, &latency[i * pconns + j]);
}
ts1 = rspamd_get_ticks ();
event_base_loop (ev_base, 0);
ts2 = rspamd_get_ticks ();
diff = (ts2 - ts1) * 1000.0;
total_diff += diff;
}
gperf_profiler_stop ();
msg_info ("Made %d uncached encrypted connections of size %d in %.6f ms, %.6f cps",
ntests * pconns,
file_size,
total_diff, ntests * pconns / total_diff * 1000.);
mean = rspamd_http_calculate_mean (latency, &std);
msg_info ("Latency: %.6f ms mean, %.6f dev",
mean, std);
/* AES mode */
serv_key = rspamd_keypair_new (RSPAMD_KEYPAIR_KEX,
RSPAMD_CRYPTOBOX_MODE_NIST);
client_key = rspamd_keypair_new (RSPAMD_KEYPAIR_KEX,
RSPAMD_CRYPTOBOX_MODE_NIST);
c = rspamd_keypair_cache_new (16);
/* Restart server */
rspamd_http_stop_servers (sfd);
/* No keypairs cache */
rspamd_http_start_servers (sfd, addr, serv_key, c);
//rspamd_mempool_lock_mutex (mtx);
usleep (100000);
b32_key = rspamd_keypair_print (serv_key,
RSPAMD_KEYPAIR_PUBKEY | RSPAMD_KEYPAIR_BASE32);
g_assert (b32_key != NULL);
peer_key = rspamd_pubkey_from_base32 (b32_key->str, b32_key->len,
RSPAMD_KEYPAIR_KEX, RSPAMD_CRYPTOBOX_MODE_NIST);
g_assert (peer_key != NULL);
total_diff = 0.0;
gperf_profiler_init (NULL, "cached-http-client-aes");
for (i = 0; i < ntests; i++) {
for (j = 0; j < pconns; j++) {
rspamd_http_client_func (filepath + sizeof ("/tmp") - 1,
addr,
client_key,
peer_key,
NULL,
ev_base,
&latency[i * pconns + j]);
}
ts1 = rspamd_get_ticks ();
event_base_loop (ev_base, 0);
ts2 = rspamd_get_ticks ();
diff = (ts2 - ts1) * 1000.0;
total_diff += diff;
}
gperf_profiler_stop ();
msg_info (
"Made %d aes encrypted connections of size %d in %.6f ms, %.6f cps",
ntests * pconns,
file_size,
total_diff,
ntests * pconns / total_diff * 1000.);
mean = rspamd_http_calculate_mean (latency, &std);
msg_info ("Latency: %.6f ms mean, %.6f dev",
mean, std);
/* Restart server */
rspamd_http_stop_servers (sfd);
/* No keypairs cache */
rspamd_http_start_servers (sfd, addr, serv_key, NULL);
//rspamd_mempool_lock_mutex (mtx);
usleep (100000);
total_diff = 0.0;
gperf_profiler_init (NULL, "fair-http-client-aes");
for (i = 0; i < ntests; i++) {
for (j = 0; j < pconns; j++) {
rspamd_http_client_func (filepath + sizeof ("/tmp") - 1,
addr,
client_key,
peer_key,
c,
ev_base,
&latency[i * pconns + j]);
}
ts1 = rspamd_get_ticks ();
event_base_loop (ev_base, 0);
ts2 = rspamd_get_ticks ();
diff = (ts2 - ts1) * 1000.0;
total_diff += diff;
}
gperf_profiler_stop ();
msg_info (
"Made %d uncached aes encrypted connections of size %d in %.6f ms, %.6f cps",
ntests * pconns,
file_size,
total_diff,
ntests * pconns / total_diff * 1000.);
mean = rspamd_http_calculate_mean (latency, &std);
msg_info ("Latency: %.6f ms mean, %.6f dev",
mean, std);
close (fd);
unlink (filepath);
rspamd_http_stop_servers (sfd);
}
gboolean
rspamd_map_add (struct rspamd_config *cfg,
const gchar *map_line,
const gchar *description,
map_cb_t read_callback,
map_fin_cb_t fin_callback,
void **user_data)
{
struct rspamd_map *new_map;
enum fetch_proto proto;
const gchar *def, *p, *hostend;
struct file_map_data *fdata;
struct http_map_data *hdata;
gchar portbuf[6], *cksum_encoded, cksum[BLAKE2B_OUTBYTES];
gint i, s, r;
struct addrinfo hints, *res;
rspamd_mempool_t *pool;
/* First of all detect protocol line */
if (!rspamd_map_check_proto (map_line, (int *)&proto, &def)) {
return FALSE;
}
/* Constant pool */
if (cfg->map_pool == NULL) {
cfg->map_pool = rspamd_mempool_new (rspamd_mempool_suggest_size (),
"map");
memcpy (cfg->map_pool->tag.uid, cfg->cfg_pool->tag.uid,
sizeof (cfg->map_pool->tag.uid));
}
new_map = rspamd_mempool_alloc0 (cfg->map_pool, sizeof (struct rspamd_map));
new_map->read_callback = read_callback;
new_map->fin_callback = fin_callback;
new_map->user_data = user_data;
new_map->protocol = proto;
new_map->cfg = cfg;
new_map->id = g_random_int ();
new_map->locked =
rspamd_mempool_alloc0_shared (cfg->cfg_pool, sizeof (gint));
if (proto == MAP_PROTO_FILE) {
new_map->uri = rspamd_mempool_strdup (cfg->cfg_pool, def);
def = new_map->uri;
}
else {
new_map->uri = rspamd_mempool_strdup (cfg->cfg_pool, map_line);
}
if (description != NULL) {
new_map->description =
rspamd_mempool_strdup (cfg->cfg_pool, description);
}
/* Now check for each proto separately */
if (proto == MAP_PROTO_FILE) {
fdata =
rspamd_mempool_alloc0 (cfg->map_pool,
sizeof (struct file_map_data));
if (access (def, R_OK) == -1) {
if (errno != ENOENT) {
msg_err_config ("cannot open file '%s': %s", def, strerror
(errno));
return FALSE;
}
msg_info_config (
"map '%s' is not found, but it can be loaded automatically later",
def);
/* We still can add this file */
fdata->st.st_mtime = -1;
}
else {
stat (def, &fdata->st);
}
fdata->filename = rspamd_mempool_strdup (cfg->map_pool, def);
new_map->map_data = fdata;
}
else if (proto == MAP_PROTO_HTTP) {
hdata =
rspamd_mempool_alloc0 (cfg->map_pool,
sizeof (struct http_map_data));
/* Try to search port */
if ((p = strchr (def, ':')) != NULL) {
hostend = p;
i = 0;
p++;
while (g_ascii_isdigit (*p) && i < (gint)sizeof (portbuf) - 1) {
portbuf[i++] = *p++;
}
if (*p != '/') {
msg_info_config ("bad http map definition: %s", def);
return FALSE;
}
portbuf[i] = '\0';
hdata->port = atoi (portbuf);
}
else {
/* Default http port */
rspamd_snprintf (portbuf, sizeof (portbuf), "80");
hdata->port = 80;
/* Now separate host from path */
if ((p = strchr (def, '/')) == NULL) {
msg_info_config ("bad http map definition: %s", def);
return FALSE;
}
hostend = p;
}
hdata->host = rspamd_mempool_alloc (cfg->map_pool, hostend - def + 1);
rspamd_strlcpy (hdata->host, def, hostend - def + 1);
hdata->path = rspamd_mempool_strdup (cfg->map_pool, p);
/* Now try to resolve */
memset (&hints, 0, sizeof (hints));
hints.ai_family = AF_UNSPEC; /* Allow IPv4 or IPv6 */
hints.ai_socktype = SOCK_STREAM; /* Stream socket */
hints.ai_flags = 0;
hints.ai_protocol = 0; /* Any protocol */
hints.ai_canonname = NULL;
hints.ai_addr = NULL;
hints.ai_next = NULL;
if ((r = getaddrinfo (hdata->host, portbuf, &hints, &res)) == 0) {
hdata->addr = res;
rspamd_mempool_add_destructor (cfg->cfg_pool,
(rspamd_mempool_destruct_t)freeaddrinfo, hdata->addr);
}
else {
msg_err_config ("address resolution for %s failed: %s",
hdata->host,
gai_strerror (r));
return FALSE;
}
/* Now try to connect */
if ((s = rspamd_socket_tcp (hdata->addr, FALSE, FALSE)) == -1) {
msg_info_config ("cannot connect to http server %s: %d, %s",
hdata->host,
errno,
strerror (errno));
return FALSE;
}
close (s);
hdata->conn = rspamd_http_connection_new (http_map_read, http_map_error,
http_map_finish,
RSPAMD_HTTP_BODY_PARTIAL | RSPAMD_HTTP_CLIENT_SIMPLE,
RSPAMD_HTTP_CLIENT, NULL);
new_map->map_data = hdata;
}
/* Temp pool */
blake2b (cksum, new_map->uri, NULL, sizeof (cksum), strlen (new_map->uri), 0);
cksum_encoded = rspamd_encode_base32 (cksum, sizeof (cksum));
new_map->pool = rspamd_mempool_new (rspamd_mempool_suggest_size (), "map");
memcpy (new_map->pool->tag.uid, cksum_encoded,
sizeof (new_map->pool->tag.uid));
g_free (cksum_encoded);
pool = new_map->pool;
msg_info_pool ("added map %s", new_map->uri);
cfg->maps = g_list_prepend (cfg->maps, new_map);
return TRUE;
}
/*
* Create new task
*/
struct rspamd_task *
rspamd_task_new (struct rspamd_worker *worker)
{
struct rspamd_task *new_task;
new_task = g_slice_alloc0 (sizeof (struct rspamd_task));
new_task->worker = worker;
if (worker) {
new_task->cfg = worker->srv->cfg;
if (new_task->cfg->check_all_filters) {
new_task->flags |= RSPAMD_TASK_FLAG_PASS_ALL;
}
}
gettimeofday (&new_task->tv, NULL);
new_task->time_real = rspamd_get_ticks ();
new_task->time_virtual = rspamd_get_virtual_ticks ();
new_task->task_pool = rspamd_mempool_new (rspamd_mempool_suggest_size (), "task");
new_task->results = g_hash_table_new (rspamd_str_hash, rspamd_str_equal);
rspamd_mempool_add_destructor (new_task->task_pool,
(rspamd_mempool_destruct_t) g_hash_table_unref,
new_task->results);
new_task->re_cache = g_hash_table_new (rspamd_str_hash, rspamd_str_equal);
rspamd_mempool_add_destructor (new_task->task_pool,
(rspamd_mempool_destruct_t) g_hash_table_unref,
new_task->re_cache);
new_task->raw_headers = g_hash_table_new (rspamd_strcase_hash,
rspamd_strcase_equal);
new_task->request_headers = g_hash_table_new_full (rspamd_gstring_icase_hash,
rspamd_gstring_icase_equal, rspamd_gstring_free_hard,
rspamd_gstring_free_hard);
rspamd_mempool_add_destructor (new_task->task_pool,
(rspamd_mempool_destruct_t) g_hash_table_unref,
new_task->request_headers);
new_task->reply_headers = g_hash_table_new_full (rspamd_gstring_icase_hash,
rspamd_gstring_icase_equal, rspamd_gstring_free_hard,
rspamd_gstring_free_hard);
rspamd_mempool_add_destructor (new_task->task_pool,
(rspamd_mempool_destruct_t) g_hash_table_unref,
new_task->reply_headers);
rspamd_mempool_add_destructor (new_task->task_pool,
(rspamd_mempool_destruct_t) g_hash_table_unref,
new_task->raw_headers);
new_task->emails = g_hash_table_new (rspamd_url_hash, rspamd_emails_cmp);
rspamd_mempool_add_destructor (new_task->task_pool,
(rspamd_mempool_destruct_t) g_hash_table_unref,
new_task->emails);
new_task->urls = g_hash_table_new (rspamd_url_hash, rspamd_urls_cmp);
rspamd_mempool_add_destructor (new_task->task_pool,
(rspamd_mempool_destruct_t) g_hash_table_unref,
new_task->urls);
new_task->parts = g_ptr_array_sized_new (4);
rspamd_mempool_add_destructor (new_task->task_pool,
rspamd_ptr_array_free_hard, new_task->parts);
new_task->text_parts = g_ptr_array_sized_new (2);
rspamd_mempool_add_destructor (new_task->task_pool,
rspamd_ptr_array_free_hard, new_task->text_parts);
new_task->received = g_ptr_array_sized_new (8);
rspamd_mempool_add_destructor (new_task->task_pool,
rspamd_ptr_array_free_hard, new_task->received);
new_task->sock = -1;
new_task->flags |= (RSPAMD_TASK_FLAG_MIME|RSPAMD_TASK_FLAG_JSON);
new_task->pre_result.action = METRIC_ACTION_NOACTION;
new_task->message_id = new_task->queue_id = "undef";
return new_task;
}
static void
rspamd_process_file (const gchar *fname)
{
struct rspamd_task *task;
GIOChannel *f;
GError *err = NULL;
GString *buf;
struct received_header rh;
gdouble t1, t2;
f = g_io_channel_new_file (fname, "r", &err);
if (!f) {
rspamd_fprintf (stderr, "cannot open %s: %e\n", fname, err);
g_error_free (err);
return;
}
g_io_channel_set_encoding (f, NULL, NULL);
buf = g_string_sized_new (8192);
task = g_malloc0 (sizeof (*task));
task->task_pool = rspamd_mempool_new (rspamd_mempool_suggest_size (), "test");
while (g_io_channel_read_line_string (f, buf, NULL, &err)
== G_IO_STATUS_NORMAL) {
while (buf->len > 0 && g_ascii_isspace (buf->str[buf->len - 1])) {
buf->len --;
}
t1 = rspamd_get_virtual_ticks ();
rspamd_smtp_recieved_parse (task, buf->str, buf->len, &rh);
t2 = rspamd_get_virtual_ticks ();
total_time += t2 - t1;
total_parsed ++;
if (rh.addr) {
total_real_ip ++;
}
if (rh.real_hostname) {
total_real_host ++;
}
if (rh.type != RSPAMD_RECEIVED_UNKNOWN) {
total_known_proto ++;
}
if (rh.by_hostname || rh.timestamp > 0) {
total_valid ++;
}
if (rh.timestamp != 0) {
total_known_ts ++;
}
}
if (err) {
rspamd_fprintf (stderr, "cannot read %s: %e\n", fname, err);
g_error_free (err);
}
g_io_channel_unref (f);
g_string_free (buf, TRUE);
rspamd_mempool_delete (task->task_pool);
g_free (task);
}
/*
* Setup logger
*/
void
rspamd_set_logger (struct rspamd_config *cfg,
GQuark ptype,
struct rspamd_main *rspamd)
{
if (rspamd->logger == NULL) {
rspamd->logger = g_malloc (sizeof (rspamd_logger_t));
memset (rspamd->logger, 0, sizeof (rspamd_logger_t));
/* Small pool for interlocking */
rspamd->logger->pool = rspamd_mempool_new (512, NULL);
rspamd->logger->mtx = rspamd_mempool_get_mutex (rspamd->logger->pool);
}
rspamd->logger->type = cfg->log_type;
rspamd->logger->pid = getpid ();
rspamd->logger->process_type = ptype;
switch (cfg->log_type) {
case RSPAMD_LOG_CONSOLE:
rspamd->logger->log_func = file_log_function;
rspamd->logger->fd = STDERR_FILENO;
break;
case RSPAMD_LOG_SYSLOG:
rspamd->logger->log_func = syslog_log_function;
break;
case RSPAMD_LOG_FILE:
rspamd->logger->log_func = file_log_function;
break;
}
rspamd->logger->cfg = cfg;
/* Set up buffer */
if (rspamd->cfg->log_buffered) {
if (rspamd->cfg->log_buf_size != 0) {
rspamd->logger->io_buf.size = rspamd->cfg->log_buf_size;
}
else {
rspamd->logger->io_buf.size = BUFSIZ;
}
rspamd->logger->is_buffered = TRUE;
rspamd->logger->io_buf.buf = g_malloc (rspamd->logger->io_buf.size);
}
/* Set up conditional logging */
if (rspamd->cfg->debug_ip_map != NULL) {
/* Try to add it as map first of all */
if (rspamd->logger->debug_ip) {
radix_destroy_compressed (rspamd->logger->debug_ip);
}
rspamd->logger->debug_ip = radix_create_compressed ();
if (!rspamd_map_add (rspamd->cfg, rspamd->cfg->debug_ip_map,
"IP addresses for which debug logs are enabled",
rspamd_radix_read, rspamd_radix_fin,
(void **) &rspamd->logger->debug_ip)) {
radix_add_generic_iplist (rspamd->cfg->debug_ip_map,
&rspamd->logger->debug_ip);
}
}
else if (rspamd->logger->debug_ip) {
radix_destroy_compressed (rspamd->logger->debug_ip);
rspamd->logger->debug_ip = NULL;
}
default_logger = rspamd->logger;
}
