static void
rspamd_stat_cache_redis_generate_id (struct rspamd_task *task)
{
rspamd_cryptobox_hash_state_t st;
rspamd_token_t *tok;
guint i;
guchar out[rspamd_cryptobox_HASHBYTES];
gchar *b32out;
gchar *user = NULL;
rspamd_cryptobox_hash_init (&st, NULL, 0);
user = rspamd_mempool_get_variable (task->task_pool, "stat_user");
/* Use dedicated hash space for per users cache */
if (user != NULL) {
rspamd_cryptobox_hash_update (&st, user, strlen (user));
}
for (i = 0; i < task->tokens->len; i ++) {
tok = g_ptr_array_index (task->tokens, i);
rspamd_cryptobox_hash_update (&st, tok->data, tok->datalen);
}
rspamd_cryptobox_hash_final (&st, out);
b32out = rspamd_encode_base32 (out, sizeof (out));
g_assert (b32out != NULL);
rspamd_mempool_set_variable (task->task_pool, "words_hash", b32out, g_free);
}
static struct rspamd_lua_cryptobox_hash *
rspamd_lua_hash_create (const gchar *type)
{
struct rspamd_lua_cryptobox_hash *h;
h = g_slice_alloc0 (sizeof (*h));
if (type) {
if (g_ascii_strcasecmp (type, "md5") == 0) {
h->is_ssl = TRUE;
h->c = EVP_MD_CTX_create ();
EVP_DigestInit (h->c, EVP_md5 ());
goto ret;
}
else if (g_ascii_strcasecmp (type, "sha1") == 0 ||
g_ascii_strcasecmp (type, "sha") == 0) {
h->is_ssl = TRUE;
h->c = EVP_MD_CTX_create ();
EVP_DigestInit (h->c, EVP_sha1 ());
goto ret;
}
else if (g_ascii_strcasecmp (type, "sha256") == 0) {
h->is_ssl = TRUE;
h->c = EVP_MD_CTX_create ();
EVP_DigestInit (h->c, EVP_sha256 ());
goto ret;
}
else if (g_ascii_strcasecmp (type, "sha512") == 0) {
h->is_ssl = TRUE;
h->c = EVP_MD_CTX_create ();
EVP_DigestInit (h->c, EVP_sha512 ());
goto ret;
}
else if (g_ascii_strcasecmp (type, "sha384") == 0) {
h->is_ssl = TRUE;
h->c = EVP_MD_CTX_create ();
EVP_DigestInit (h->c, EVP_sha384 ());
goto ret;
}
}
h->h = g_slice_alloc0 (sizeof (*h->h));
rspamd_cryptobox_hash_init (h->h, NULL, 0);
ret:
return h;
}
/***
* @function rspamd_cryptobox_hash.create()
* Creates new hash context
* @param {string} data raw signature data
* @return {cryptobox_hash} hash object
*/
static gint
lua_cryptobox_hash_create (lua_State *L)
{
rspamd_cryptobox_hash_state_t *h, **ph;
h = g_slice_alloc (sizeof (*h));
rspamd_cryptobox_hash_init (h, NULL, 0);
ph = lua_newuserdata (L, sizeof (void *));
*ph = h;
rspamd_lua_setclass (L, "rspamd{cryptobox_hash}", -1);
return 1;
}
static void
rspamd_regexp_generate_id (const gchar *pattern, const gchar *flags,
regexp_id_t out)
{
rspamd_cryptobox_hash_state_t st;
rspamd_cryptobox_hash_init (&st, NULL, 0);
if (flags) {
rspamd_cryptobox_hash_update (&st, flags, strlen (flags));
}
rspamd_cryptobox_hash_update (&st, pattern, strlen (pattern));
rspamd_cryptobox_hash_final (&st, out);
}
/***
* @function rspamd_cryptobox_hash.create_keyed(key, [string])
* Creates new hash context with specified key
* @param {string} key key
* @return {cryptobox_hash} hash object
*/
static gint
lua_cryptobox_hash_create_keyed (lua_State *L)
{
struct rspamd_lua_cryptobox_hash *h, **ph;
const gchar *key, *s = NULL;
struct rspamd_lua_text *t;
gsize len = 0;
gsize keylen;
key = luaL_checklstring (L, 1, &keylen);
if (key != NULL) {
h = rspamd_lua_hash_create (NULL);
rspamd_cryptobox_hash_init (h->h, key, keylen);
if (lua_type (L, 2) == LUA_TSTRING) {
s = lua_tolstring (L, 2, &len);
}
else if (lua_type (L, 2) == LUA_TUSERDATA) {
t = lua_check_text (L, 2);
if (!t) {
return luaL_error (L, "invalid arguments");
}
s = t->start;
len = t->len;
}
if (s) {
rspamd_cryptobox_hash_update (h, s, len);
}
ph = lua_newuserdata (L, sizeof (void *));
*ph = h;
rspamd_lua_setclass (L, "rspamd{cryptobox_hash}", -1);
}
else {
return luaL_error (L, "invalid arguments");
}
return 1;
}
static void
rspamd_map_calculate_hash (struct rspamd_map *map)
{
struct rspamd_map_backend *bk;
guint i;
rspamd_cryptobox_hash_state_t st;
gchar *cksum_encoded, cksum[rspamd_cryptobox_HASHBYTES];
rspamd_cryptobox_hash_init (&st, NULL, 0);
for (i = 0; i < map->backends->len; i ++) {
bk = g_ptr_array_index (map->backends, i);
rspamd_cryptobox_hash_update (&st, bk->uri, strlen (bk->uri));
}
rspamd_cryptobox_hash_final (&st, cksum);
cksum_encoded = rspamd_encode_base32 (cksum, sizeof (cksum));
rspamd_strlcpy (map->tag, cksum_encoded, sizeof (map->tag));
g_free (cksum_encoded);
}
/***
* @method cryptobox_hash:reset()
* Resets hash to the initial state
*/
static gint
lua_cryptobox_hash_reset (lua_State *L)
{
struct rspamd_lua_cryptobox_hash *h = lua_check_cryptobox_hash (L, 1);
if (h) {
if (h->is_ssl) {
EVP_DigestInit (h->c, EVP_MD_CTX_md (h->c));
}
else {
memset (h->h, 0, sizeof (*h->h));
rspamd_cryptobox_hash_init (h->h, NULL, 0);
}
h->is_finished = FALSE;
}
else {
return luaL_error (L, "invalid arguments");
}
return 0;
}
/***
* @function rspamd_cryptobox_hash.create_keyed(key)
* Creates new hash context with specified key
* @param {string} key key
* @return {cryptobox_hash} hash object
*/
static gint
lua_cryptobox_hash_create_keyed (lua_State *L)
{
rspamd_cryptobox_hash_state_t *h, **ph;
const gchar *key;
gsize keylen;
key = luaL_checklstring (L, 1, &keylen);
if (key != NULL) {
h = g_slice_alloc (sizeof (*h));
rspamd_cryptobox_hash_init (h, key, keylen);
ph = lua_newuserdata (L, sizeof (void *));
*ph = h;
rspamd_lua_setclass (L, "rspamd{cryptobox_hash}", -1);
}
else {
return luaL_error (L, "invalid arguments");
}
return 1;
}
void
rspamd_re_cache_init (struct rspamd_re_cache *cache, struct rspamd_config *cfg)
{
guint i, fl;
GHashTableIter it;
gpointer k, v;
struct rspamd_re_class *re_class;
rspamd_cryptobox_hash_state_t st_global;
rspamd_regexp_t *re;
struct rspamd_re_cache_elt *elt;
guchar hash_out[rspamd_cryptobox_HASHBYTES];
g_assert (cache != NULL);
rspamd_cryptobox_hash_init (&st_global, NULL, 0);
/* Resort all regexps */
g_ptr_array_sort (cache->re, rspamd_re_cache_sort_func);
for (i = 0; i < cache->re->len; i ++) {
elt = g_ptr_array_index (cache->re, i);
re = elt->re;
re_class = rspamd_regexp_get_class (re);
g_assert (re_class != NULL);
rspamd_regexp_set_cache_id (re, i);
if (re_class->st == NULL) {
re_class->st = g_slice_alloc (sizeof (*re_class->st));
rspamd_cryptobox_hash_init (re_class->st, NULL, 0);
}
/* Update hashes */
rspamd_cryptobox_hash_update (re_class->st, (gpointer) &re_class->id,
sizeof (re_class->id));
rspamd_cryptobox_hash_update (&st_global, (gpointer) &re_class->id,
sizeof (re_class->id));
rspamd_cryptobox_hash_update (re_class->st, rspamd_regexp_get_id (re),
rspamd_cryptobox_HASHBYTES);
rspamd_cryptobox_hash_update (&st_global, rspamd_regexp_get_id (re),
rspamd_cryptobox_HASHBYTES);
fl = rspamd_regexp_get_pcre_flags (re);
rspamd_cryptobox_hash_update (re_class->st, (const guchar *)&fl,
sizeof (fl));
rspamd_cryptobox_hash_update (&st_global, (const guchar *) &fl,
sizeof (fl));
fl = rspamd_regexp_get_flags (re);
rspamd_cryptobox_hash_update (re_class->st, (const guchar *) &fl,
sizeof (fl));
rspamd_cryptobox_hash_update (&st_global, (const guchar *) &fl,
sizeof (fl));
fl = rspamd_regexp_get_maxhits (re);
rspamd_cryptobox_hash_update (re_class->st, (const guchar *) &fl,
sizeof (fl));
rspamd_cryptobox_hash_update (&st_global, (const guchar *) &fl,
sizeof (fl));
}
rspamd_cryptobox_hash_final (&st_global, hash_out);
rspamd_snprintf (cache->hash, sizeof (cache->hash), "%*xs",
(gint) rspamd_cryptobox_HASHBYTES, hash_out);
/* Now finalize all classes */
g_hash_table_iter_init (&it, cache->re_classes);
while (g_hash_table_iter_next (&it, &k, &v)) {
re_class = v;
if (re_class->st) {
/*
* We finally update all classes with the number of expressions
* in the cache to ensure that if even a single re has been changed
* we won't be broken due to id mismatch
*/
rspamd_cryptobox_hash_update (re_class->st,
(gpointer)&cache->re->len,
sizeof (cache->re->len));
rspamd_cryptobox_hash_final (re_class->st, hash_out);
rspamd_snprintf (re_class->hash, sizeof (re_class->hash), "%*xs",
(gint) rspamd_cryptobox_HASHBYTES, hash_out);
g_slice_free1 (sizeof (*re_class->st), re_class->st);
re_class->st = NULL;
}
}
#ifdef WITH_HYPERSCAN
const gchar *platform = "generic";
rspamd_fstring_t *features = rspamd_fstring_new ();
cache->disable_hyperscan = cfg->disable_hyperscan;
cache->vectorized_hyperscan = cfg->vectorized_hyperscan;
g_assert (hs_populate_platform (&cache->plt) == HS_SUCCESS);
/* Now decode what we do have */
switch (cache->plt.tune) {
case HS_TUNE_FAMILY_HSW:
platform = "haswell";
break;
case HS_TUNE_FAMILY_SNB:
platform = "sandy";
break;
case HS_TUNE_FAMILY_BDW:
platform = "broadwell";
break;
case HS_TUNE_FAMILY_IVB:
platform = "ivy";
break;
default:
break;
}
if (cache->plt.cpu_features & HS_CPU_FEATURES_AVX2) {
features = rspamd_fstring_append (features, "AVX2", 4);
}
hs_set_allocator (g_malloc, g_free);
msg_info_re_cache ("loaded hyperscan engine witch cpu tune '%s' and features '%V'",
platform, features);
rspamd_fstring_free (features);
#endif
}
void*
rspamd_fuzzy_backend_init_redis (struct rspamd_fuzzy_backend *bk,
const ucl_object_t *obj, struct rspamd_config *cfg, GError **err)
{
struct rspamd_fuzzy_backend_redis *backend;
const ucl_object_t *elt;
gboolean ret = FALSE;
guchar id_hash[rspamd_cryptobox_HASHBYTES];
rspamd_cryptobox_hash_state_t st;
backend = g_slice_alloc0 (sizeof (*backend));
backend->timeout = REDIS_DEFAULT_TIMEOUT;
backend->redis_object = REDIS_DEFAULT_OBJECT;
ret = rspamd_fuzzy_backend_redis_try_ucl (backend, obj, cfg);
/* Now try global redis settings */
if (!ret) {
elt = ucl_object_lookup (cfg->rcl_obj, "redis");
if (elt) {
const ucl_object_t *specific_obj;
specific_obj = ucl_object_lookup_any (elt, "fuzzy", "fuzzy_storage",
NULL);
if (specific_obj) {
ret = rspamd_fuzzy_backend_redis_try_ucl (backend, specific_obj,
cfg);
}
else {
ret = rspamd_fuzzy_backend_redis_try_ucl (backend, elt, cfg);
}
}
}
if (!ret) {
msg_err_config ("cannot init redis backend for fuzzy storage");
g_slice_free1 (sizeof (*backend), backend);
return NULL;
}
REF_INIT_RETAIN (backend, rspamd_fuzzy_backend_redis_dtor);
backend->pool = cfg->redis_pool;
rspamd_cryptobox_hash_init (&st, NULL, 0);
rspamd_cryptobox_hash_update (&st, backend->redis_object,
strlen (backend->redis_object));
if (backend->dbname) {
rspamd_cryptobox_hash_update (&st, backend->dbname,
strlen (backend->dbname));
}
if (backend->password) {
rspamd_cryptobox_hash_update (&st, backend->password,
strlen (backend->password));
}
rspamd_cryptobox_hash_final (&st, id_hash);
backend->id = rspamd_encode_base32 (id_hash, sizeof (id_hash));
return backend;
}