void
rspamd_controller_send_ucl (struct rspamd_http_connection_entry *entry,
ucl_object_t *obj)
{
struct rspamd_http_message *msg;
rspamd_fstring_t *reply;
msg = rspamd_http_new_message (HTTP_RESPONSE);
msg->date = time (NULL);
msg->code = 200;
msg->status = rspamd_fstring_new_init ("OK", 2);
reply = rspamd_fstring_sized_new (BUFSIZ);
rspamd_ucl_emit_fstring (obj, UCL_EMIT_JSON_COMPACT, &reply);
rspamd_http_message_set_body_from_fstring_steal (msg, reply);
rspamd_http_connection_reset (entry->conn);
rspamd_http_router_insert_headers (entry->rt, msg);
rspamd_http_connection_write_message (entry->conn,
msg,
NULL,
"application/json",
entry,
entry->conn->fd,
entry->rt->ptv,
entry->rt->ev_base);
entry->is_reply = TRUE;
}
void
rspamd_controller_send_error (struct rspamd_http_connection_entry *entry,
gint code, const gchar *error_msg, ...)
{
struct rspamd_http_message *msg;
va_list args;
rspamd_fstring_t *reply;
msg = rspamd_http_new_message (HTTP_RESPONSE);
va_start (args, error_msg);
msg->status = rspamd_fstring_new ();
rspamd_vprintf_fstring (&msg->status, error_msg, args);
va_end (args);
msg->date = time (NULL);
msg->code = code;
reply = rspamd_fstring_sized_new (msg->status->len + 16);
rspamd_printf_fstring (&reply, "{\"error\":\"%V\"}", msg->status);
rspamd_http_message_set_body_from_fstring_steal (msg, reply);
rspamd_http_connection_reset (entry->conn);
rspamd_http_router_insert_headers (entry->rt, msg);
rspamd_http_connection_write_message (entry->conn,
msg,
NULL,
"application/json",
entry,
entry->conn->fd,
entry->rt->ptv,
entry->rt->ev_base);
entry->is_reply = TRUE;
}
/**
* Sign file using specified rsa key and signature
*
* arguments:
* (rsa_privkey, rsa_signature, string)
*
* returns:
* true - if string match rsa signature
* false - otherwise
*/
static gint
lua_rsa_sign_file (lua_State *L)
{
RSA *rsa;
rspamd_fstring_t *signature, **psig;
const gchar *filename;
gchar *data = NULL, *data_sig;
gint ret, fd;
struct stat st;
rsa = lua_check_rsa_privkey (L, 1);
filename = luaL_checkstring (L, 2);
if (rsa != NULL && filename != NULL) {
fd = open (filename, O_RDONLY);
if (fd == -1) {
msg_err ("cannot open file %s: %s", filename, strerror (errno));
lua_pushnil (L);
}
else {
if (fstat (fd, &st) == -1 ||
(data =
mmap (NULL, st.st_size, PROT_READ, MAP_SHARED, fd,
0)) == MAP_FAILED) {
msg_err ("cannot mmap file %s: %s", filename, strerror (errno));
lua_pushnil (L);
}
else {
signature = rspamd_fstring_sized_new (RSA_size (rsa));
data_sig = g_compute_checksum_for_string (G_CHECKSUM_SHA256,
data,
st.st_size);
ret = RSA_sign (NID_sha1, data_sig, strlen (data_sig),
signature->str, (guint *)&signature->len, rsa);
if (ret == 0) {
msg_info ("cannot make a signature for data: %s",
ERR_error_string (ERR_get_error (), NULL));
lua_pushnil (L);
rspamd_fstring_free (signature);
}
else {
psig = lua_newuserdata (L, sizeof (rspamd_fstring_t *));
rspamd_lua_setclass (L, "rspamd{rsa_signature}", -1);
*psig = signature;
}
g_free (data_sig);
munmap (data, st.st_size);
}
close (fd);
}
}
else {
lua_pushnil (L);
}
return 1;
}
static int
rspamd_server_finish (struct rspamd_http_connection *conn,
struct rspamd_http_message *msg)
{
struct rspamd_http_server_session *session = conn->ud;
struct rspamd_http_message *reply;
gulong size;
const gchar *url_str;
guint url_len;
rspamd_fstring_t *body;
if (!session->reply) {
session->reply = TRUE;
reply = rspamd_http_new_message (HTTP_RESPONSE);
url_str = msg->url->str;
url_len = msg->url->len;
if (url_str[0] == '/') {
url_str ++;
url_len --;
}
if (rspamd_strtoul (url_str, url_len, &size)) {
session->req_size = size;
reply->code = 200;
reply->status = rspamd_fstring_new_init ("OK", 2);
body = rspamd_fstring_sized_new (size);
body->len = size;
memset (body->str, 0, size);
rspamd_http_message_set_body_from_fstring_steal (msg, body);
}
else {
reply->code = 404;
reply->status = rspamd_fstring_new_init ("Not found", 9);
}
rspamd_http_connection_reset (conn);
rspamd_http_connection_write_message (conn, reply, NULL,
"application/octet-stream", session, session->fd,
&io_tv, session->ev_base);
}
else {
/* Destroy session */
rspamd_http_connection_unref (conn);
close (session->fd);
g_slice_free1 (sizeof (*session), session);
}
return 0;
}
/***
* @function rspamd_cryptobox.sign_memory(kp, data)
* Sign data using specified keypair
* @param {keypair} kp keypair to sign
* @param {string} data
* @return {cryptobox_signature} signature object
*/
static gint
lua_cryptobox_sign_memory (lua_State *L)
{
struct rspamd_cryptobox_keypair *kp;
const gchar *data;
struct rspamd_lua_text *t;
gsize len = 0;
rspamd_fstring_t *sig, **psig;
kp = lua_check_cryptobox_keypair (L, 1);
if (lua_isuserdata (L, 2)) {
t = lua_check_text (L, 2);
if (!t) {
return luaL_error (L, "invalid arguments");
}
data = t->start;
len = t->len;
}
else {
data = luaL_checklstring (L, 2, &len);
}
if (!kp || !data) {
return luaL_error (L, "invalid arguments");
}
sig = rspamd_fstring_sized_new (rspamd_cryptobox_signature_bytes (
rspamd_keypair_alg (kp)));
rspamd_cryptobox_sign (sig->str, &sig->len, data,
len, rspamd_keypair_component (kp, RSPAMD_KEYPAIR_COMPONENT_SK,
NULL), rspamd_keypair_alg (kp));
psig = lua_newuserdata (L, sizeof (void *));
*psig = sig;
rspamd_lua_setclass (L, "rspamd{cryptobox_signature}", -1);
return 1;
}
/***
* @function rspamd_cryptobox.sign_file(kp, file)
* Sign file using specified keypair
* @param {keypair} kp keypair to sign
* @param {string} filename
* @return {cryptobox_signature} signature object
*/
static gint
lua_cryptobox_sign_file (lua_State *L)
{
struct rspamd_cryptobox_keypair *kp;
const gchar *filename;
gchar *data;
gsize len = 0;
rspamd_fstring_t *sig, **psig;
kp = lua_check_cryptobox_keypair (L, 1);
filename = luaL_checkstring (L, 2);
if (!kp || !filename) {
return luaL_error (L, "invalid arguments");
}
data = rspamd_file_xmap (filename, PROT_READ, &len);
if (data == NULL) {
msg_err ("cannot mmap file %s: %s", filename, strerror (errno));
lua_pushnil (L);
}
else {
sig = rspamd_fstring_sized_new (rspamd_cryptobox_signature_bytes (
rspamd_keypair_alg (kp)));
rspamd_cryptobox_sign (sig->str, &sig->len, data,
len, rspamd_keypair_component (kp, RSPAMD_KEYPAIR_COMPONENT_SK,
NULL), rspamd_keypair_alg (kp));
psig = lua_newuserdata (L, sizeof (void *));
*psig = sig;
rspamd_lua_setclass (L, "rspamd{cryptobox_signature}", -1);
munmap (data, len);
}
return 1;
}
/**
* Sign memory using specified rsa key and signature
*
* arguments:
* (rsa_privkey, string)
*
* returns:
* rspamd_signature object
* nil - otherwise
*/
static gint
lua_rsa_sign_memory (lua_State *L)
{
RSA *rsa;
rspamd_fstring_t *signature, **psig;
const gchar *data;
guchar *data_sig;
gint ret;
rsa = lua_check_rsa_privkey (L, 1);
data = luaL_checkstring (L, 2);
if (rsa != NULL && data != NULL) {
signature = rspamd_fstring_sized_new (RSA_size (rsa));
data_sig = g_compute_checksum_for_string (G_CHECKSUM_SHA256, data, -1);
ret = RSA_sign (NID_sha1, data_sig, strlen (data_sig),
signature->str, (guint *)&signature->len, rsa);
if (ret == 0) {
msg_info ("cannot make a signature for data: %s",
ERR_error_string (ERR_get_error (), NULL));
lua_pushnil (L);
rspamd_fstring_free (signature);
}
else {
psig = lua_newuserdata (L, sizeof (rspamd_fstring_t *));
rspamd_lua_setclass (L, "rspamd{rsa_signature}", -1);
*psig = signature;
}
g_free (data_sig);
}
else {
lua_pushnil (L);
}
return 1;
}
rspamd_shingles_from_text (GArray *input,
const guchar key[16],
rspamd_mempool_t *pool,
rspamd_shingles_filter filter,
gpointer filterd,
enum rspamd_shingle_alg alg)
{
struct rspamd_shingle *res;
guint64 **hashes;
guchar **keys;
rspamd_fstring_t *row;
rspamd_stat_token_t *word;
guint64 val;
gint i, j, k;
gsize hlen, beg = 0;
enum rspamd_cryptobox_fast_hash_type ht;
if (pool != NULL) {
res = rspamd_mempool_alloc (pool, sizeof (*res));
}
else {
res = g_malloc (sizeof (*res));
}
row = rspamd_fstring_sized_new (256);
/* Init hashes pipes and keys */
hashes = g_malloc (sizeof (*hashes) * RSPAMD_SHINGLE_SIZE);
hlen = input->len > SHINGLES_WINDOW ?
(input->len - SHINGLES_WINDOW + 1) : 1;
keys = rspamd_shingles_get_keys_cached (key);
for (i = 0; i < RSPAMD_SHINGLE_SIZE; i ++) {
hashes[i] = g_malloc (hlen * sizeof (guint64));
}
/* Now parse input words into a vector of hashes using rolling window */
if (alg == RSPAMD_SHINGLES_OLD) {
for (i = 0; i <= (gint)input->len; i ++) {
if (i - beg >= SHINGLES_WINDOW || i == (gint)input->len) {
for (j = beg; j < i; j ++) {
word = &g_array_index (input, rspamd_stat_token_t, j);
row = rspamd_fstring_append (row, word->begin, word->len);
}
/* Now we need to create a new row here */
for (j = 0; j < RSPAMD_SHINGLE_SIZE; j ++) {
rspamd_cryptobox_siphash ((guchar *)&val, row->str, row->len,
keys[j]);
g_assert (hlen > beg);
hashes[j][beg] = val;
}
beg++;
row = rspamd_fstring_assign (row, "", 0);
}
}
}
else {
guint64 res[SHINGLES_WINDOW * RSPAMD_SHINGLE_SIZE], seed;
switch (alg) {
case RSPAMD_SHINGLES_XXHASH:
ht = RSPAMD_CRYPTOBOX_XXHASH64;
break;
case RSPAMD_SHINGLES_MUMHASH:
ht = RSPAMD_CRYPTOBOX_MUMHASH;
break;
default:
ht = RSPAMD_CRYPTOBOX_HASHFAST_INDEPENDENT;
break;
}
memset (res, 0, sizeof (res));
for (i = 0; i <= (gint)input->len; i ++) {
if (i - beg >= SHINGLES_WINDOW || i == (gint)input->len) {
for (j = 0; j < RSPAMD_SHINGLE_SIZE; j ++) {
/* Shift hashes window to right */
for (k = 0; k < SHINGLES_WINDOW - 1; k ++) {
res[j * SHINGLES_WINDOW + k] =
res[j * SHINGLES_WINDOW + k + 1];
}
word = &g_array_index (input, rspamd_stat_token_t, beg);
/* Insert the last element to the pipe */
memcpy (&seed, keys[j], sizeof (seed));
res[j * SHINGLES_WINDOW + SHINGLES_WINDOW - 1] =
rspamd_cryptobox_fast_hash_specific (ht,
word->begin, word->len,
seed);
val = 0;
for (k = 0; k < SHINGLES_WINDOW; k ++) {
val ^= res[j * SHINGLES_WINDOW + k] >>
(8 * (SHINGLES_WINDOW - k - 1));
}
g_assert (hlen > beg);
hashes[j][beg] = val;
}
beg++;
}
}
}
/* Now we need to filter all hashes and make a shingles result */
for (i = 0; i < RSPAMD_SHINGLE_SIZE; i ++) {
res->hashes[i] = filter (hashes[i], hlen,
i, key, filterd);
g_free (hashes[i]);
}
g_free (hashes);
rspamd_fstring_free (row);
return res;
}
static rspamd_fstring_t *
rspamd_redis_tokens_to_query (struct rspamd_task *task, GPtrArray *tokens,
const gchar *arg0, const gchar *arg1, gboolean learn, gint idx,
gboolean intvals)
{
rspamd_fstring_t *out;
rspamd_token_t *tok;
gchar n0[64], n1[64];
guint i, l0, l1, larg0, larg1;
guint64 num;
g_assert (tokens != NULL);
larg0 = strlen (arg0);
larg1 = strlen (arg1);
out = rspamd_fstring_sized_new (1024);
if (!learn) {
rspamd_printf_fstring (&out, ""
"*%d\r\n"
"$%d\r\n"
"%s\r\n"
"$%d\r\n"
"%s\r\n",
(tokens->len + 2),
larg0, arg0,
larg1, arg1);
}
for (i = 0; i < tokens->len; i ++) {
tok = g_ptr_array_index (tokens, i);
memcpy (&num, tok->data, sizeof (num));
if (learn) {
rspamd_printf_fstring (&out, ""
"*4\r\n"
"$%d\r\n"
"%s\r\n"
"$%d\r\n"
"%s\r\n",
larg0, arg0,
larg1, arg1);
l0 = rspamd_snprintf (n0, sizeof (n0), "%uL", num);
if (intvals) {
l1 = rspamd_snprintf (n1, sizeof (n1), "%L",
(gint64)tok->values[idx]);
}
else {
l1 = rspamd_snprintf (n1, sizeof (n1), "%f",
tok->values[idx]);
}
rspamd_printf_fstring (&out, ""
"$%d\r\n"
"%s\r\n"
"$%d\r\n"
"%s\r\n", l0, n0, l1, n1);
}
else {
l0 = rspamd_snprintf (n0, sizeof (n0), "%uL", num);
rspamd_printf_fstring (&out, ""
"$%d\r\n"
"%s\r\n", l0, n0);
}
}
return out;
}
gboolean
rspamd_client_command (struct rspamd_client_connection *conn,
const gchar *command, GQueue *attrs,
FILE *in, rspamd_client_callback cb,
gpointer ud, gboolean compressed,
const gchar *comp_dictionary,
GError **err)
{
struct rspamd_client_request *req;
struct rspamd_http_client_header *nh;
gchar *p;
gsize remain, old_len;
GList *cur;
GString *input = NULL;
rspamd_fstring_t *body;
guint dict_id = 0;
gsize dict_len = 0;
void *dict = NULL;
ZSTD_CCtx *zctx;
req = g_slice_alloc0 (sizeof (struct rspamd_client_request));
req->conn = conn;
req->cb = cb;
req->ud = ud;
req->msg = rspamd_http_new_message (HTTP_REQUEST);
if (conn->key) {
req->msg->peer_key = rspamd_pubkey_ref (conn->key);
}
if (in != NULL) {
/* Read input stream */
input = g_string_sized_new (BUFSIZ);
while (!feof (in)) {
p = input->str + input->len;
remain = input->allocated_len - input->len - 1;
if (remain == 0) {
old_len = input->len;
g_string_set_size (input, old_len * 2);
input->len = old_len;
continue;
}
remain = fread (p, 1, remain, in);
if (remain > 0) {
input->len += remain;
input->str[input->len] = '\0';
}
}
if (ferror (in) != 0) {
g_set_error (err, RCLIENT_ERROR, ferror (
in), "input IO error: %s", strerror (ferror (in)));
g_slice_free1 (sizeof (struct rspamd_client_request), req);
g_string_free (input, TRUE);
return FALSE;
}
if (!compressed) {
body = rspamd_fstring_new_init (input->str, input->len);
}
else {
if (comp_dictionary) {
dict = rspamd_file_xmap (comp_dictionary, PROT_READ, &dict_len);
if (dict == NULL) {
g_set_error (err, RCLIENT_ERROR, errno,
"cannot open dictionary %s: %s",
comp_dictionary,
strerror (errno));
g_slice_free1 (sizeof (struct rspamd_client_request), req);
g_string_free (input, TRUE);
return FALSE;
}
dict_id = ZDICT_getDictID (comp_dictionary, dict_len);
if (dict_id == 0) {
g_set_error (err, RCLIENT_ERROR, errno,
"cannot open dictionary %s: %s",
comp_dictionary,
strerror (errno));
g_slice_free1 (sizeof (struct rspamd_client_request), req);
g_string_free (input, TRUE);
munmap (dict, dict_len);
return FALSE;
}
}
body = rspamd_fstring_sized_new (ZSTD_compressBound (input->len));
zctx = ZSTD_createCCtx ();
body->len = ZSTD_compress_usingDict (zctx, body->str, body->allocated,
input->str, input->len,
dict, dict_len,
1);
munmap (dict, dict_len);
if (ZSTD_isError (body->len)) {
g_set_error (err, RCLIENT_ERROR, ferror (
in), "compression error");
g_slice_free1 (sizeof (struct rspamd_client_request), req);
g_string_free (input, TRUE);
rspamd_fstring_free (body);
ZSTD_freeCCtx (zctx);
return FALSE;
}
ZSTD_freeCCtx (zctx);
}
rspamd_http_message_set_body_from_fstring_steal (req->msg, body);
req->input = input;
}
else {
req->input = NULL;
}
/* Convert headers */
cur = attrs->head;
while (cur != NULL) {
nh = cur->data;
rspamd_http_message_add_header (req->msg, nh->name, nh->value);
cur = g_list_next (cur);
}
if (compressed) {
rspamd_http_message_add_header (req->msg, "Compression", "zstd");
if (dict_id != 0) {
gchar dict_str[32];
rspamd_snprintf (dict_str, sizeof (dict_str), "%ud", dict_id);
rspamd_http_message_add_header (req->msg, "Dictionary", dict_str);
}
}
req->msg->url = rspamd_fstring_append (req->msg->url, "/", 1);
req->msg->url = rspamd_fstring_append (req->msg->url, command, strlen (command));
conn->req = req;
if (compressed) {
rspamd_http_connection_write_message (conn->http_conn, req->msg, NULL,
"application/x-compressed", req, conn->fd,
&conn->timeout, conn->ev_base);
}
else {
rspamd_http_connection_write_message (conn->http_conn, req->msg, NULL,
"text/plain", req, conn->fd, &conn->timeout, conn->ev_base);
}
return TRUE;
}
