/**
* Write HTTP request
*/
static void
write_http_request (struct http_callback_data *cbd)
{
gchar datebuf[128];
struct rspamd_http_message *msg;
struct rspamd_map *map;
map = cbd->map;
if (cbd->fd != -1) {
close (cbd->fd);
}
cbd->fd = rspamd_inet_address_connect (cbd->addr, SOCK_STREAM, TRUE);
if (cbd->fd != -1) {
msg = rspamd_http_new_message (HTTP_REQUEST);
if (cbd->bk->protocol == MAP_PROTO_HTTPS) {
msg->flags |= RSPAMD_HTTP_FLAG_SSL;
}
if (cbd->check) {
msg->method = HTTP_HEAD;
}
if (cbd->stage == map_load_file) {
msg->url = rspamd_fstring_new_init (cbd->data->path, strlen (cbd->data->path));
if (cbd->check &&
cbd->data->last_checked != 0 && cbd->stage == map_load_file) {
rspamd_http_date_format (datebuf, sizeof (datebuf),
cbd->data->last_checked);
rspamd_http_message_add_header (msg, "If-Modified-Since", datebuf);
}
}
else if (cbd->stage == map_load_pubkey) {
msg->url = rspamd_fstring_new_init (cbd->data->path, strlen (cbd->data->path));
msg->url = rspamd_fstring_append (msg->url, ".pub", 4);
}
else if (cbd->stage == map_load_signature) {
msg->url = rspamd_fstring_new_init (cbd->data->path, strlen (cbd->data->path));
msg->url = rspamd_fstring_append (msg->url, ".sig", 4);
}
else {
g_assert_not_reached ();
}
rspamd_http_connection_write_message (cbd->conn, msg, cbd->data->host,
NULL, cbd, cbd->fd, &cbd->tv, cbd->ev_base);
MAP_RETAIN (cbd, "http_callback_data");
}
else {
msg_err_map ("cannot connect to %s: %s", cbd->data->host,
strerror (errno));
cbd->periodic->errored = TRUE;
}
}
/**
* Write HTTP request
*/
static void
write_http_request (struct http_callback_data *cbd)
{
gchar datebuf[128];
struct rspamd_http_message *msg;
rspamd_mempool_t *pool;
pool = cbd->map->pool;
if (cbd->fd != -1) {
close (cbd->fd);
}
cbd->fd = rspamd_inet_address_connect (cbd->addr, SOCK_STREAM, TRUE);
if (cbd->fd != -1) {
msg = rspamd_http_new_message (HTTP_REQUEST);
if (cbd->stage == map_load_file) {
msg->url = rspamd_fstring_new_init (cbd->data->path, strlen (cbd->data->path));
if (cbd->data->last_checked != 0 && cbd->stage == map_load_file) {
rspamd_http_date_format (datebuf, sizeof (datebuf),
cbd->data->last_checked);
rspamd_http_message_add_header (msg, "If-Modified-Since", datebuf);
}
}
else if (cbd->stage == map_load_pubkey) {
msg->url = rspamd_fstring_new_init (cbd->data->path, strlen (cbd->data->path));
msg->url = rspamd_fstring_append (msg->url, ".pub", 4);
}
else if (cbd->stage == map_load_signature) {
msg->url = rspamd_fstring_new_init (cbd->data->path, strlen (cbd->data->path));
msg->url = rspamd_fstring_append (msg->url, ".sig", 4);
}
else {
g_assert_not_reached ();
}
rspamd_http_connection_write_message (cbd->conn, msg, cbd->data->host,
NULL, cbd, cbd->fd, &cbd->tv, cbd->ev_base);
REF_RETAIN (cbd);
}
else {
msg_err_pool ("cannot connect to %s: %s", cbd->data->host,
strerror (errno));
}
}
gboolean
rspamd_client_command (struct rspamd_client_connection *conn,
const gchar *command, GQueue *attrs,
FILE *in, rspamd_client_callback cb,
gpointer ud, GError **err)
{
struct rspamd_client_request *req;
struct rspamd_http_client_header *nh;
gchar *p;
gsize remain, old_len;
GList *cur;
GString *input = NULL;
rspamd_fstring_t *body;
req = g_slice_alloc0 (sizeof (struct rspamd_client_request));
req->conn = conn;
req->cb = cb;
req->ud = ud;
req->msg = rspamd_http_new_message (HTTP_REQUEST);
if (conn->key) {
req->msg->peer_key = rspamd_pubkey_ref (conn->key);
}
if (in != NULL) {
/* Read input stream */
input = g_string_sized_new (BUFSIZ);
while (!feof (in)) {
p = input->str + input->len;
remain = input->allocated_len - input->len - 1;
if (remain == 0) {
old_len = input->len;
g_string_set_size (input, old_len * 2);
input->len = old_len;
continue;
}
remain = fread (p, 1, remain, in);
if (remain > 0) {
input->len += remain;
input->str[input->len] = '\0';
}
}
if (ferror (in) != 0) {
g_set_error (err, RCLIENT_ERROR, ferror (
in), "input IO error: %s", strerror (ferror (in)));
g_slice_free1 (sizeof (struct rspamd_client_request), req);
g_string_free (input, TRUE);
return FALSE;
}
body = rspamd_fstring_new_init (input->str, input->len);
rspamd_http_message_set_body_from_fstring_steal (req->msg, body);
req->input = input;
}
else {
req->input = NULL;
}
/* Convert headers */
cur = attrs->head;
while (cur != NULL) {
nh = cur->data;
rspamd_http_message_add_header (req->msg, nh->name, nh->value);
cur = g_list_next (cur);
}
req->msg->url = rspamd_fstring_append (req->msg->url, "/", 1);
req->msg->url = rspamd_fstring_append (req->msg->url, command, strlen (command));
conn->req = req;
rspamd_http_connection_write_message (conn->http_conn, req->msg, NULL,
"text/plain", req, conn->fd, &conn->timeout, conn->ev_base);
return TRUE;
}
void
rspamd_re_cache_init (struct rspamd_re_cache *cache, struct rspamd_config *cfg)
{
guint i, fl;
GHashTableIter it;
gpointer k, v;
struct rspamd_re_class *re_class;
rspamd_cryptobox_hash_state_t st_global;
rspamd_regexp_t *re;
struct rspamd_re_cache_elt *elt;
guchar hash_out[rspamd_cryptobox_HASHBYTES];
g_assert (cache != NULL);
rspamd_cryptobox_hash_init (&st_global, NULL, 0);
/* Resort all regexps */
g_ptr_array_sort (cache->re, rspamd_re_cache_sort_func);
for (i = 0; i < cache->re->len; i ++) {
elt = g_ptr_array_index (cache->re, i);
re = elt->re;
re_class = rspamd_regexp_get_class (re);
g_assert (re_class != NULL);
rspamd_regexp_set_cache_id (re, i);
if (re_class->st == NULL) {
re_class->st = g_slice_alloc (sizeof (*re_class->st));
rspamd_cryptobox_hash_init (re_class->st, NULL, 0);
}
/* Update hashes */
rspamd_cryptobox_hash_update (re_class->st, (gpointer) &re_class->id,
sizeof (re_class->id));
rspamd_cryptobox_hash_update (&st_global, (gpointer) &re_class->id,
sizeof (re_class->id));
rspamd_cryptobox_hash_update (re_class->st, rspamd_regexp_get_id (re),
rspamd_cryptobox_HASHBYTES);
rspamd_cryptobox_hash_update (&st_global, rspamd_regexp_get_id (re),
rspamd_cryptobox_HASHBYTES);
fl = rspamd_regexp_get_pcre_flags (re);
rspamd_cryptobox_hash_update (re_class->st, (const guchar *)&fl,
sizeof (fl));
rspamd_cryptobox_hash_update (&st_global, (const guchar *) &fl,
sizeof (fl));
fl = rspamd_regexp_get_flags (re);
rspamd_cryptobox_hash_update (re_class->st, (const guchar *) &fl,
sizeof (fl));
rspamd_cryptobox_hash_update (&st_global, (const guchar *) &fl,
sizeof (fl));
fl = rspamd_regexp_get_maxhits (re);
rspamd_cryptobox_hash_update (re_class->st, (const guchar *) &fl,
sizeof (fl));
rspamd_cryptobox_hash_update (&st_global, (const guchar *) &fl,
sizeof (fl));
}
rspamd_cryptobox_hash_final (&st_global, hash_out);
rspamd_snprintf (cache->hash, sizeof (cache->hash), "%*xs",
(gint) rspamd_cryptobox_HASHBYTES, hash_out);
/* Now finalize all classes */
g_hash_table_iter_init (&it, cache->re_classes);
while (g_hash_table_iter_next (&it, &k, &v)) {
re_class = v;
if (re_class->st) {
/*
* We finally update all classes with the number of expressions
* in the cache to ensure that if even a single re has been changed
* we won't be broken due to id mismatch
*/
rspamd_cryptobox_hash_update (re_class->st,
(gpointer)&cache->re->len,
sizeof (cache->re->len));
rspamd_cryptobox_hash_final (re_class->st, hash_out);
rspamd_snprintf (re_class->hash, sizeof (re_class->hash), "%*xs",
(gint) rspamd_cryptobox_HASHBYTES, hash_out);
g_slice_free1 (sizeof (*re_class->st), re_class->st);
re_class->st = NULL;
}
}
#ifdef WITH_HYPERSCAN
const gchar *platform = "generic";
rspamd_fstring_t *features = rspamd_fstring_new ();
cache->disable_hyperscan = cfg->disable_hyperscan;
cache->vectorized_hyperscan = cfg->vectorized_hyperscan;
g_assert (hs_populate_platform (&cache->plt) == HS_SUCCESS);
/* Now decode what we do have */
switch (cache->plt.tune) {
case HS_TUNE_FAMILY_HSW:
platform = "haswell";
break;
case HS_TUNE_FAMILY_SNB:
platform = "sandy";
break;
case HS_TUNE_FAMILY_BDW:
platform = "broadwell";
break;
case HS_TUNE_FAMILY_IVB:
platform = "ivy";
break;
default:
break;
}
if (cache->plt.cpu_features & HS_CPU_FEATURES_AVX2) {
features = rspamd_fstring_append (features, "AVX2", 4);
}
hs_set_allocator (g_malloc, g_free);
msg_info_re_cache ("loaded hyperscan engine witch cpu tune '%s' and features '%V'",
platform, features);
rspamd_fstring_free (features);
#endif
}
rspamd_shingles_from_text (GArray *input,
const guchar key[16],
rspamd_mempool_t *pool,
rspamd_shingles_filter filter,
gpointer filterd,
enum rspamd_shingle_alg alg)
{
struct rspamd_shingle *res;
guint64 **hashes;
guchar **keys;
rspamd_fstring_t *row;
rspamd_stat_token_t *word;
guint64 val;
gint i, j, k;
gsize hlen, beg = 0;
enum rspamd_cryptobox_fast_hash_type ht;
if (pool != NULL) {
res = rspamd_mempool_alloc (pool, sizeof (*res));
}
else {
res = g_malloc (sizeof (*res));
}
row = rspamd_fstring_sized_new (256);
/* Init hashes pipes and keys */
hashes = g_malloc (sizeof (*hashes) * RSPAMD_SHINGLE_SIZE);
hlen = input->len > SHINGLES_WINDOW ?
(input->len - SHINGLES_WINDOW + 1) : 1;
keys = rspamd_shingles_get_keys_cached (key);
for (i = 0; i < RSPAMD_SHINGLE_SIZE; i ++) {
hashes[i] = g_malloc (hlen * sizeof (guint64));
}
/* Now parse input words into a vector of hashes using rolling window */
if (alg == RSPAMD_SHINGLES_OLD) {
for (i = 0; i <= (gint)input->len; i ++) {
if (i - beg >= SHINGLES_WINDOW || i == (gint)input->len) {
for (j = beg; j < i; j ++) {
word = &g_array_index (input, rspamd_stat_token_t, j);
row = rspamd_fstring_append (row, word->begin, word->len);
}
/* Now we need to create a new row here */
for (j = 0; j < RSPAMD_SHINGLE_SIZE; j ++) {
rspamd_cryptobox_siphash ((guchar *)&val, row->str, row->len,
keys[j]);
g_assert (hlen > beg);
hashes[j][beg] = val;
}
beg++;
row = rspamd_fstring_assign (row, "", 0);
}
}
}
else {
guint64 res[SHINGLES_WINDOW * RSPAMD_SHINGLE_SIZE], seed;
switch (alg) {
case RSPAMD_SHINGLES_XXHASH:
ht = RSPAMD_CRYPTOBOX_XXHASH64;
break;
case RSPAMD_SHINGLES_MUMHASH:
ht = RSPAMD_CRYPTOBOX_MUMHASH;
break;
default:
ht = RSPAMD_CRYPTOBOX_HASHFAST_INDEPENDENT;
break;
}
memset (res, 0, sizeof (res));
for (i = 0; i <= (gint)input->len; i ++) {
if (i - beg >= SHINGLES_WINDOW || i == (gint)input->len) {
for (j = 0; j < RSPAMD_SHINGLE_SIZE; j ++) {
/* Shift hashes window to right */
for (k = 0; k < SHINGLES_WINDOW - 1; k ++) {
res[j * SHINGLES_WINDOW + k] =
res[j * SHINGLES_WINDOW + k + 1];
}
word = &g_array_index (input, rspamd_stat_token_t, beg);
/* Insert the last element to the pipe */
memcpy (&seed, keys[j], sizeof (seed));
res[j * SHINGLES_WINDOW + SHINGLES_WINDOW - 1] =
rspamd_cryptobox_fast_hash_specific (ht,
word->begin, word->len,
seed);
val = 0;
for (k = 0; k < SHINGLES_WINDOW; k ++) {
val ^= res[j * SHINGLES_WINDOW + k] >>
(8 * (SHINGLES_WINDOW - k - 1));
}
g_assert (hlen > beg);
hashes[j][beg] = val;
}
beg++;
}
}
}
/* Now we need to filter all hashes and make a shingles result */
for (i = 0; i < RSPAMD_SHINGLE_SIZE; i ++) {
res->hashes[i] = filter (hashes[i], hlen,
i, key, filterd);
g_free (hashes[i]);
}
g_free (hashes);
rspamd_fstring_free (row);
return res;
}
gboolean
rspamd_client_command (struct rspamd_client_connection *conn,
const gchar *command, GQueue *attrs,
FILE *in, rspamd_client_callback cb,
gpointer ud, gboolean compressed,
const gchar *comp_dictionary,
GError **err)
{
struct rspamd_client_request *req;
struct rspamd_http_client_header *nh;
gchar *p;
gsize remain, old_len;
GList *cur;
GString *input = NULL;
rspamd_fstring_t *body;
guint dict_id = 0;
gsize dict_len = 0;
void *dict = NULL;
ZSTD_CCtx *zctx;
req = g_slice_alloc0 (sizeof (struct rspamd_client_request));
req->conn = conn;
req->cb = cb;
req->ud = ud;
req->msg = rspamd_http_new_message (HTTP_REQUEST);
if (conn->key) {
req->msg->peer_key = rspamd_pubkey_ref (conn->key);
}
if (in != NULL) {
/* Read input stream */
input = g_string_sized_new (BUFSIZ);
while (!feof (in)) {
p = input->str + input->len;
remain = input->allocated_len - input->len - 1;
if (remain == 0) {
old_len = input->len;
g_string_set_size (input, old_len * 2);
input->len = old_len;
continue;
}
remain = fread (p, 1, remain, in);
if (remain > 0) {
input->len += remain;
input->str[input->len] = '\0';
}
}
if (ferror (in) != 0) {
g_set_error (err, RCLIENT_ERROR, ferror (
in), "input IO error: %s", strerror (ferror (in)));
g_slice_free1 (sizeof (struct rspamd_client_request), req);
g_string_free (input, TRUE);
return FALSE;
}
if (!compressed) {
body = rspamd_fstring_new_init (input->str, input->len);
}
else {
if (comp_dictionary) {
dict = rspamd_file_xmap (comp_dictionary, PROT_READ, &dict_len);
if (dict == NULL) {
g_set_error (err, RCLIENT_ERROR, errno,
"cannot open dictionary %s: %s",
comp_dictionary,
strerror (errno));
g_slice_free1 (sizeof (struct rspamd_client_request), req);
g_string_free (input, TRUE);
return FALSE;
}
dict_id = ZDICT_getDictID (comp_dictionary, dict_len);
if (dict_id == 0) {
g_set_error (err, RCLIENT_ERROR, errno,
"cannot open dictionary %s: %s",
comp_dictionary,
strerror (errno));
g_slice_free1 (sizeof (struct rspamd_client_request), req);
g_string_free (input, TRUE);
munmap (dict, dict_len);
return FALSE;
}
}
body = rspamd_fstring_sized_new (ZSTD_compressBound (input->len));
zctx = ZSTD_createCCtx ();
body->len = ZSTD_compress_usingDict (zctx, body->str, body->allocated,
input->str, input->len,
dict, dict_len,
1);
munmap (dict, dict_len);
if (ZSTD_isError (body->len)) {
g_set_error (err, RCLIENT_ERROR, ferror (
in), "compression error");
g_slice_free1 (sizeof (struct rspamd_client_request), req);
g_string_free (input, TRUE);
rspamd_fstring_free (body);
ZSTD_freeCCtx (zctx);
return FALSE;
}
ZSTD_freeCCtx (zctx);
}
rspamd_http_message_set_body_from_fstring_steal (req->msg, body);
req->input = input;
}
else {
req->input = NULL;
}
/* Convert headers */
cur = attrs->head;
while (cur != NULL) {
nh = cur->data;
rspamd_http_message_add_header (req->msg, nh->name, nh->value);
cur = g_list_next (cur);
}
if (compressed) {
rspamd_http_message_add_header (req->msg, "Compression", "zstd");
if (dict_id != 0) {
gchar dict_str[32];
rspamd_snprintf (dict_str, sizeof (dict_str), "%ud", dict_id);
rspamd_http_message_add_header (req->msg, "Dictionary", dict_str);
}
}
req->msg->url = rspamd_fstring_append (req->msg->url, "/", 1);
req->msg->url = rspamd_fstring_append (req->msg->url, command, strlen (command));
conn->req = req;
if (compressed) {
rspamd_http_connection_write_message (conn->http_conn, req->msg, NULL,
"application/x-compressed", req, conn->fd,
&conn->timeout, conn->ev_base);
}
else {
rspamd_http_connection_write_message (conn->http_conn, req->msg, NULL,
"text/plain", req, conn->fd, &conn->timeout, conn->ev_base);
}
return TRUE;
}
