static gchar *
rspamd_humanize_number (gchar *buf, gchar *last, gint64 num, gboolean bytes)
{
const gchar *prefixes;
int i, r, remainder, sign;
gint64 divisor;
gsize len = last - buf;
remainder = 0;
if (!bytes) {
divisor = 1000;
prefixes = "\0\0\0k\0\0M\0\0G\0\0T\0\0P\0\0E";
}
else {
divisor = 1024;
prefixes = "B\0\0k\0\0M\0\0G\0\0T\0\0P\0\0E";
}
#define SCALE2PREFIX(scale) (&prefixes[(scale) * 3])
if (num < 0) {
sign = -1;
num = -num;
}
else {
sign = 1;
}
/*
* Divide the number until it fits the given column.
* If there will be an overflow by the rounding below,
* divide once more.
*/
for (i = 0; i < maxscale && num > divisor; i++) {
remainder = num % divisor;
num /= divisor;
}
if (remainder == 0 || num > divisor / 2) {
r = rspamd_snprintf (buf, len, "%L%s",
sign * (num + (remainder + 50) / divisor),
SCALE2PREFIX (i));
}
else {
/* Floating point version */
r = rspamd_snprintf (buf, len, "%.2f%s",
sign * (num + remainder / (gdouble)divisor),
SCALE2PREFIX (i));
}
#undef SCALE2PREFIX
return buf + r;
}
gboolean
make_smtp_tempfile (struct smtp_session *session)
{
gsize r;
r = strlen (session->cfg->temp_dir) + sizeof ("/rspamd-XXXXXX");
session->temp_name = rspamd_mempool_alloc (session->pool, r);
rspamd_snprintf (session->temp_name,
r,
"%s%crspamd-XXXXXX",
session->cfg->temp_dir,
G_DIR_SEPARATOR);
#ifdef HAVE_MKSTEMP
/* Umask is set before */
session->temp_fd = mkstemp (session->temp_name);
#else
session->temp_fd = g_mkstemp_full (session->temp_name,
O_RDWR,
S_IWUSR | S_IRUSR);
#endif
if (session->temp_fd == -1) {
msg_err ("mkstemp error: %s", strerror (errno));
return FALSE;
}
return TRUE;
}
static void
rspamd_http_client_func (const gchar *path, rspamd_inet_addr_t *addr,
gpointer kp, gpointer peer_kp, struct rspamd_keypair_cache *c,
struct event_base *ev_base, double *latency)
{
struct rspamd_http_message *msg;
struct rspamd_http_connection *conn;
gchar urlbuf[PATH_MAX];
struct client_cbdata *cb;
gint fd;
g_assert ((fd = rspamd_inet_address_connect (addr, SOCK_STREAM, TRUE)) != -1);
conn = rspamd_http_connection_new (rspamd_client_body, rspamd_client_err,
rspamd_client_finish, RSPAMD_HTTP_CLIENT_SIMPLE,
RSPAMD_HTTP_CLIENT, c);
rspamd_snprintf (urlbuf, sizeof (urlbuf), "http://127.0.0.1/%s", path);
msg = rspamd_http_message_from_url (urlbuf);
g_assert (conn != NULL && msg != NULL);
if (kp != NULL) {
g_assert (peer_kp != NULL);
rspamd_http_connection_set_key (conn, kp);
msg->peer_key = rspamd_http_connection_key_ref (peer_kp);
}
cb = g_malloc (sizeof (*cb));
clock_gettime (CLOCK_MONOTONIC, &cb->ts);
cb->lat = latency;
rspamd_http_connection_write_message (conn, msg, NULL, NULL, cb,
fd, NULL, ev_base);
}
/**
* Callback for destroying HTTP callback data
*/
static void
free_http_cbdata_common (struct http_callback_data *cbd, gboolean plan_new)
{
char fpath[PATH_MAX];
struct stat st;
struct map_periodic_cbdata *periodic = cbd->periodic;
if (cbd->out_fd != -1) {
close (cbd->out_fd);
}
rspamd_snprintf (fpath, sizeof (fpath), "%s", cbd->tmpfile);
if (stat (fpath, &st) != -1 && S_ISREG (st.st_mode)) {
(void)unlink (fpath);
}
rspamd_snprintf (fpath, sizeof (fpath), "%s.pub", cbd->tmpfile);
if (stat (fpath, &st) != -1 && S_ISREG (st.st_mode)) {
(void)unlink (fpath);
}
rspamd_snprintf (fpath, sizeof (fpath), "%s.sig", cbd->tmpfile);
if (stat (fpath, &st) != -1 && S_ISREG (st.st_mode)) {
(void)unlink (fpath);
}
if (cbd->pk) {
rspamd_pubkey_unref (cbd->pk);
}
if (cbd->conn) {
rspamd_http_connection_unref (cbd->conn);
cbd->conn = NULL;
}
if (cbd->fd != -1) {
close (cbd->fd);
}
if (cbd->addr) {
rspamd_inet_address_destroy (cbd->addr);
}
MAP_RELEASE (cbd->bk);
MAP_RELEASE (periodic);
g_slice_free1 (sizeof (struct http_callback_data), cbd);
}
void
rspamd_log_close_priv (rspamd_logger_t *rspamd_log, uid_t uid, gid_t gid)
{
gchar tmpbuf[256];
rspamd_log_flush (rspamd_log);
switch (rspamd_log->type) {
case RSPAMD_LOG_CONSOLE:
/* Do nothing special */
break;
case RSPAMD_LOG_SYSLOG:
#ifdef HAVE_SYSLOG_H
closelog ();
#endif
break;
case RSPAMD_LOG_FILE:
if (rspamd_log->enabled) {
if (rspamd_log->repeats > REPEATS_MIN) {
rspamd_snprintf (tmpbuf,
sizeof (tmpbuf),
"Last message repeated %ud times",
rspamd_log->repeats);
rspamd_log->repeats = 0;
if (rspamd_log->saved_message) {
file_log_function (rspamd_log->saved_module,
rspamd_log->saved_id,
rspamd_log->saved_function,
rspamd_log->saved_loglevel | RSPAMD_LOG_FORCED,
rspamd_log->saved_message,
rspamd_log);
g_free (rspamd_log->saved_message);
g_free (rspamd_log->saved_function);
g_free (rspamd_log->saved_module);
g_free (rspamd_log->saved_id);
rspamd_log->saved_message = NULL;
rspamd_log->saved_function = NULL;
rspamd_log->saved_module = NULL;
rspamd_log->saved_id = NULL;
}
/* It is safe to use temporary buffer here as it is not static */
file_log_function (NULL, NULL,
G_STRFUNC,
rspamd_log->saved_loglevel | RSPAMD_LOG_FORCED,
tmpbuf,
rspamd_log);
return;
}
if (fsync (rspamd_log->fd) == -1) {
msg_err ("error syncing log file: %s", strerror (errno));
}
close (rspamd_log->fd);
}
break;
}
rspamd_log->enabled = FALSE;
}
static gboolean
rspamd_map_check_file_sig (const char *fname,
struct rspamd_map *map,
struct rspamd_map_backend *bk,
const guchar *input,
gsize inlen)
{
gchar fpath[PATH_MAX];
guchar *data;
struct rspamd_cryptobox_pubkey *pk = NULL;
GString *b32_key;
gboolean ret;
gsize len = 0;
if (bk->trusted_pubkey == NULL) {
/* Try to load and check pubkey */
rspamd_snprintf (fpath, sizeof (fpath), "%s.pub", fname);
data = rspamd_file_xmap (fpath, PROT_READ, &len);
if (data == NULL) {
msg_err_map ("can't open pubkey %s: %s", fpath, strerror (errno));
return FALSE;
}
pk = rspamd_pubkey_from_base32 (data, len, RSPAMD_KEYPAIR_SIGN,
RSPAMD_CRYPTOBOX_MODE_25519);
munmap (data, len);
if (pk == NULL) {
msg_err_map ("can't load pubkey %s", fpath);
return FALSE;
}
/* We just check pk against the trusted db of keys */
b32_key = rspamd_pubkey_print (pk,
RSPAMD_KEYPAIR_BASE32|RSPAMD_KEYPAIR_PUBKEY);
g_assert (b32_key != NULL);
if (g_hash_table_lookup (map->cfg->trusted_keys, b32_key->str) == NULL) {
msg_err_map ("pubkey loaded from %s is untrusted: %v", fpath,
b32_key);
g_string_free (b32_key, TRUE);
rspamd_pubkey_unref (pk);
return FALSE;
}
g_string_free (b32_key, TRUE);
}
else {
pk = rspamd_pubkey_ref (bk->trusted_pubkey);
}
ret = rspamd_map_check_sig_pk (fname, map, input, inlen, pk);
rspamd_pubkey_unref (pk);
return ret;
}
/**
* Callback for destroying HTTP callback data
*/
static void
free_http_cbdata_common (struct http_callback_data *cbd)
{
char fpath[PATH_MAX];
struct stat st;
if (cbd->out_fd != -1) {
close (cbd->out_fd);
}
rspamd_snprintf (fpath, sizeof (fpath), "%s", cbd->tmpfile);
if (stat (fpath, &st) != -1 && S_ISREG (st.st_mode)) {
(void)unlink (fpath);
}
rspamd_snprintf (fpath, sizeof (fpath), "%s.pub", cbd->tmpfile);
if (stat (fpath, &st) != -1 && S_ISREG (st.st_mode)) {
(void)unlink (fpath);
}
rspamd_snprintf (fpath, sizeof (fpath), "%s.sig", cbd->tmpfile);
if (stat (fpath, &st) != -1 && S_ISREG (st.st_mode)) {
(void)unlink (fpath);
}
if (cbd->pk) {
rspamd_pubkey_unref (cbd->pk);
}
if (cbd->conn) {
rspamd_http_connection_unref (cbd->conn);
cbd->conn = NULL;
}
if (cbd->fd != -1) {
close (cbd->fd);
}
if (cbd->addr) {
rspamd_inet_address_destroy (cbd->addr);
}
g_atomic_int_set (cbd->map->locked, 0);
g_slice_free1 (sizeof (struct http_callback_data), cbd);
}
/*
* Write a line to log file (unbuffered)
*/
static void
direct_write_log_line (rspamd_logger_t *rspamd_log,
void *data,
gsize count,
gboolean is_iov)
{
gchar errmsg[128];
struct iovec *iov;
const gchar *line;
glong r;
if (rspamd_log->enabled) {
if (!rspamd_log->no_lock) {
rspamd_file_lock (rspamd_log->fd, FALSE);
}
if (is_iov) {
iov = (struct iovec *) data;
r = writev (rspamd_log->fd, iov, count);
}
else {
line = (const gchar *) data;
r = write (rspamd_log->fd, line, count);
}
if (!rspamd_log->no_lock) {
rspamd_file_unlock (rspamd_log->fd, FALSE);
}
if (r == -1) {
/* We cannot write message to file, so we need to detect error and make decision */
if (errno == EINTR) {
/* Try again */
direct_write_log_line (rspamd_log, data, count, is_iov);
return;
}
r = rspamd_snprintf (errmsg,
sizeof (errmsg),
"direct_write_log_line: cannot write log line: %s",
strerror (errno));
if (errno == EFAULT || errno == EINVAL || errno == EFBIG ||
errno == ENOSPC) {
/* Rare case */
rspamd_log->throttling = TRUE;
rspamd_log->throttling_time = time (NULL);
}
else if (errno == EPIPE || errno == EBADF) {
/* We write to some pipe and it disappears, disable logging or we has opened bad file descriptor */
rspamd_log->enabled = FALSE;
}
}
else if (rspamd_log->throttling) {
rspamd_log->throttling = FALSE;
}
}
}
static gchar *
rspamd_humanize_number (gchar *buf, gchar *last, gint64 num, gboolean bytes)
{
const gchar *prefixes;
int i, r, remainder, sign;
gint64 divisor;
gsize baselen, len = last - buf;
remainder = 0;
baselen = 1;
if (!bytes) {
divisor = 1000;
prefixes = "\0\0\0k\0\0M\0\0G\0\0T\0\0P\0\0E";
}
else {
divisor = 1024;
prefixes = "B\0\0k\0\0M\0\0G\0\0T\0\0P\0\0E";
}
#define SCALE2PREFIX(scale) (&prefixes[(scale) * 3])
if (num < 0) {
sign = -1;
num = -num;
baselen += 2; /* sign, digit */
}
else {
sign = 1;
baselen += 1; /* digit */
}
/* Check if enough room for `x y' + suffix + `\0' */
if (len < baselen + 1) {
return buf;
}
/*
* Divide the number until it fits the given column.
* If there will be an overflow by the rounding below,
* divide once more.
*/
for (i = 0; i < maxscale && num > divisor; i++) {
remainder = num % divisor;
num /= divisor;
}
r = rspamd_snprintf (buf, len, "%L%s",
sign * (num + (remainder + 50) / 1000),
SCALE2PREFIX (i));
#undef SCALE2PREFIX
return buf + r;
}
static void
smtp_metric_symbols_callback (gpointer key, gpointer value, void *user_data)
{
struct smtp_metric_callback_data *cd = user_data;
cd->log_offset += rspamd_snprintf (cd->log_buf + cd->log_offset,
cd->log_size - cd->log_offset,
"%s,",
(gchar *)key);
}
static gboolean
rspamd_map_check_sig_pk (const char *fname,
struct rspamd_map *map,
const guchar *input,
gsize inlen,
struct rspamd_cryptobox_pubkey *pk)
{
gchar fpath[PATH_MAX];
rspamd_mempool_t *pool = map->pool;
guchar *data;
GString *b32_key;
gsize len = 0;
/* Now load signature */
rspamd_snprintf (fpath, sizeof (fpath), "%s.sig", fname);
data = rspamd_file_xmap (fpath, PROT_READ, &len);
if (data == NULL) {
msg_err_pool ("can't open signature %s: %s", fpath, strerror (errno));
return FALSE;
}
if (len != rspamd_cryptobox_signature_bytes (RSPAMD_CRYPTOBOX_MODE_25519)) {
msg_err_pool ("can't open signature %s: invalid signature", fpath);
munmap (data, len);
return FALSE;
}
if (!rspamd_cryptobox_verify (data, input, inlen,
rspamd_pubkey_get_pk (pk, NULL), RSPAMD_CRYPTOBOX_MODE_25519)) {
msg_err_pool ("can't verify signature %s: incorrect signature", fpath);
munmap (data, len);
return FALSE;
}
b32_key = rspamd_pubkey_print (pk,
RSPAMD_KEYPAIR_BASE32|RSPAMD_KEYPAIR_PUBKEY);
msg_info_pool ("verified signature in file %s using trusted key %v",
fpath, b32_key);
g_string_free (b32_key, TRUE);
munmap (data, len);
return TRUE;
}
static inline void
rspamd_email_address_add (rspamd_mempool_t *pool,
GPtrArray *ar,
struct rspamd_email_address *addr,
GString *name)
{
struct rspamd_email_address *elt;
guint nlen;
elt = g_malloc0 (sizeof (*elt));
if (addr != NULL) {
memcpy (elt, addr, sizeof (*addr));
}
else {
elt->addr = "";
elt->domain = "";
elt->raw = "<>";
elt->raw_len = 2;
elt->user = "";
elt->flags |= RSPAMD_EMAIL_ADDR_EMPTY;
}
if ((elt->flags & RSPAMD_EMAIL_ADDR_QUOTED) && elt->addr[0] == '"') {
if (elt->flags & RSPAMD_EMAIL_ADDR_HAS_BACKSLASH) {
/* We also need to unquote user */
rspamd_email_address_unescape (elt);
}
/* We need to unquote addr */
nlen = elt->domain_len + elt->user_len + 2;
elt->addr = g_malloc (nlen + 1);
elt->addr_len = rspamd_snprintf ((char *)elt->addr, nlen, "%*s@%*s",
(gint)elt->user_len, elt->user,
(gint)elt->domain_len, elt->domain);
elt->flags |= RSPAMD_EMAIL_ADDR_ADDR_ALLOCATED;
}
if (name->len > 0) {
elt->name = rspamd_mime_header_decode (pool, name->str, name->len);
}
g_ptr_array_add (ar, elt);
}
size_t
smtp_upstream_write_list (GList *args, gchar *buf, size_t buflen)
{
GList *cur = args;
size_t r = 0;
rspamd_ftok_t *arg;
while (cur && r < buflen - 3) {
arg = cur->data;
r += rspamd_snprintf (buf + r, buflen - r, " %T", arg);
cur = g_list_next (cur);
}
buf[r++] = CR;
buf[r++] = LF;
buf[r] = '\0';
return r;
}
static void
log_next_sync (const gchar *symbol, time_t delay)
{
gchar outstr[200];
time_t t;
struct tm *tmp;
gint r;
t = time (NULL);
t += delay;
tmp = localtime (&t);
if (tmp) {
r = rspamd_snprintf (outstr,
sizeof (outstr),
"statfile_sync: next sync of %s at ",
symbol);
if ((r = strftime (outstr + r, sizeof(outstr) - r, "%T", tmp)) != 0) {
msg_info (outstr);
}
}
}
struct rspamd_email_address *
rspamd_email_address_from_smtp (const gchar *str, guint len)
{
struct rspamd_email_address addr, *ret;
gsize nlen;
if (str == NULL || len == 0) {
return NULL;
}
rspamd_smtp_addr_parse (str, len, &addr);
if (addr.flags & RSPAMD_EMAIL_ADDR_VALID) {
ret = g_slice_alloc (sizeof (*ret));
memcpy (ret, &addr, sizeof (addr));
if ((ret->flags & RSPAMD_EMAIL_ADDR_QUOTED) && ret->addr[0] == '"') {
if (ret->flags & RSPAMD_EMAIL_ADDR_HAS_BACKSLASH) {
/* We also need to unquote user */
rspamd_email_address_unescape (ret);
}
/* We need to unquote addr */
nlen = ret->domain_len + ret->user_len + 2;
ret->addr = g_malloc (nlen + 1);
ret->addr_len = rspamd_snprintf ((char *)ret->addr, nlen, "%*s@%*s",
(gint)ret->user_len, ret->user,
(gint)ret->domain_len, ret->domain);
ret->flags |= RSPAMD_EMAIL_ADDR_ADDR_ALLOCATED;
}
REF_INIT_RETAIN (ret, rspamd_email_addr_dtor);
return ret;
}
return NULL;
}
gchar *
make_smtp_error (rspamd_mempool_t *pool,
gint error_code,
const gchar *format,
...)
{
va_list vp;
gchar *result = NULL, *p;
size_t len;
va_start (vp, format);
len = g_printf_string_upper_bound (format, vp);
va_end (vp);
va_start (vp, format);
len += sizeof ("65535 ") + sizeof (CRLF) - 1;
result = rspamd_mempool_alloc (pool, len);
p = result + rspamd_snprintf (result, len, "%d ", error_code);
p = rspamd_vsnprintf (p, len - (p - result), format, vp);
*p++ = CR; *p++ = LF; *p = '\0';
va_end (vp);
return result;
}
gboolean
rspamd_map_add (struct rspamd_config *cfg,
const gchar *map_line,
const gchar *description,
map_cb_t read_callback,
map_fin_cb_t fin_callback,
void **user_data)
{
struct rspamd_map *new_map;
enum fetch_proto proto;
const gchar *def, *p, *hostend;
struct file_map_data *fdata;
struct http_map_data *hdata;
gchar portbuf[6], *cksum_encoded, cksum[BLAKE2B_OUTBYTES];
gint i, s, r;
struct addrinfo hints, *res;
rspamd_mempool_t *pool;
/* First of all detect protocol line */
if (!rspamd_map_check_proto (map_line, (int *)&proto, &def)) {
return FALSE;
}
/* Constant pool */
if (cfg->map_pool == NULL) {
cfg->map_pool = rspamd_mempool_new (rspamd_mempool_suggest_size (),
"map");
memcpy (cfg->map_pool->tag.uid, cfg->cfg_pool->tag.uid,
sizeof (cfg->map_pool->tag.uid));
}
new_map = rspamd_mempool_alloc0 (cfg->map_pool, sizeof (struct rspamd_map));
new_map->read_callback = read_callback;
new_map->fin_callback = fin_callback;
new_map->user_data = user_data;
new_map->protocol = proto;
new_map->cfg = cfg;
new_map->id = g_random_int ();
new_map->locked =
rspamd_mempool_alloc0_shared (cfg->cfg_pool, sizeof (gint));
if (proto == MAP_PROTO_FILE) {
new_map->uri = rspamd_mempool_strdup (cfg->cfg_pool, def);
def = new_map->uri;
}
else {
new_map->uri = rspamd_mempool_strdup (cfg->cfg_pool, map_line);
}
if (description != NULL) {
new_map->description =
rspamd_mempool_strdup (cfg->cfg_pool, description);
}
/* Now check for each proto separately */
if (proto == MAP_PROTO_FILE) {
fdata =
rspamd_mempool_alloc0 (cfg->map_pool,
sizeof (struct file_map_data));
if (access (def, R_OK) == -1) {
if (errno != ENOENT) {
msg_err_config ("cannot open file '%s': %s", def, strerror
(errno));
return FALSE;
}
msg_info_config (
"map '%s' is not found, but it can be loaded automatically later",
def);
/* We still can add this file */
fdata->st.st_mtime = -1;
}
else {
stat (def, &fdata->st);
}
fdata->filename = rspamd_mempool_strdup (cfg->map_pool, def);
new_map->map_data = fdata;
}
else if (proto == MAP_PROTO_HTTP) {
hdata =
rspamd_mempool_alloc0 (cfg->map_pool,
sizeof (struct http_map_data));
/* Try to search port */
if ((p = strchr (def, ':')) != NULL) {
hostend = p;
i = 0;
p++;
while (g_ascii_isdigit (*p) && i < (gint)sizeof (portbuf) - 1) {
portbuf[i++] = *p++;
}
if (*p != '/') {
msg_info_config ("bad http map definition: %s", def);
return FALSE;
}
portbuf[i] = '\0';
hdata->port = atoi (portbuf);
}
else {
/* Default http port */
rspamd_snprintf (portbuf, sizeof (portbuf), "80");
hdata->port = 80;
/* Now separate host from path */
if ((p = strchr (def, '/')) == NULL) {
msg_info_config ("bad http map definition: %s", def);
return FALSE;
}
hostend = p;
}
hdata->host = rspamd_mempool_alloc (cfg->map_pool, hostend - def + 1);
rspamd_strlcpy (hdata->host, def, hostend - def + 1);
hdata->path = rspamd_mempool_strdup (cfg->map_pool, p);
/* Now try to resolve */
memset (&hints, 0, sizeof (hints));
hints.ai_family = AF_UNSPEC; /* Allow IPv4 or IPv6 */
hints.ai_socktype = SOCK_STREAM; /* Stream socket */
hints.ai_flags = 0;
hints.ai_protocol = 0; /* Any protocol */
hints.ai_canonname = NULL;
hints.ai_addr = NULL;
hints.ai_next = NULL;
if ((r = getaddrinfo (hdata->host, portbuf, &hints, &res)) == 0) {
hdata->addr = res;
rspamd_mempool_add_destructor (cfg->cfg_pool,
(rspamd_mempool_destruct_t)freeaddrinfo, hdata->addr);
}
else {
msg_err_config ("address resolution for %s failed: %s",
hdata->host,
gai_strerror (r));
return FALSE;
}
/* Now try to connect */
if ((s = rspamd_socket_tcp (hdata->addr, FALSE, FALSE)) == -1) {
msg_info_config ("cannot connect to http server %s: %d, %s",
hdata->host,
errno,
strerror (errno));
return FALSE;
}
close (s);
hdata->conn = rspamd_http_connection_new (http_map_read, http_map_error,
http_map_finish,
RSPAMD_HTTP_BODY_PARTIAL | RSPAMD_HTTP_CLIENT_SIMPLE,
RSPAMD_HTTP_CLIENT, NULL);
new_map->map_data = hdata;
}
/* Temp pool */
blake2b (cksum, new_map->uri, NULL, sizeof (cksum), strlen (new_map->uri), 0);
cksum_encoded = rspamd_encode_base32 (cksum, sizeof (cksum));
new_map->pool = rspamd_mempool_new (rspamd_mempool_suggest_size (), "map");
memcpy (new_map->pool->tag.uid, cksum_encoded,
sizeof (new_map->pool->tag.uid));
g_free (cksum_encoded);
pool = new_map->pool;
msg_info_pool ("added map %s", new_map->uri);
cfg->maps = g_list_prepend (cfg->maps, new_map);
return TRUE;
}
void
rspamd_http_test_func (void)
{
struct event_base *ev_base = event_init ();
rspamd_mempool_t *pool = rspamd_mempool_new (rspamd_mempool_suggest_size ());
gpointer serv_key, client_key, peer_key;
struct rspamd_keypair_cache *c;
rspamd_mempool_mutex_t *mtx;
rspamd_inet_addr_t addr;
struct timespec ts1, ts2;
gchar filepath[PATH_MAX], buf[512];
gint fd, i, j;
pid_t sfd;
GString *b32_key;
double diff, total_diff = 0.0, latency[pconns * ntests], mean, std;
rspamd_cryptobox_init ();
rspamd_snprintf (filepath, sizeof (filepath), "/tmp/http-test-XXXXXX");
g_assert ((fd = mkstemp (filepath)) != -1);
for (i = 0; i < file_blocks; i ++) {
memset (buf, 0, sizeof (buf));
g_assert (write (fd, buf, sizeof (buf)) == sizeof (buf));
}
mtx = rspamd_mempool_get_mutex (pool);
rspamd_parse_inet_address (&addr, "127.0.0.1");
rspamd_inet_address_set_port (&addr, ottery_rand_range (30000) + 32768);
serv_key = rspamd_http_connection_gen_key ();
client_key = rspamd_http_connection_gen_key ();
c = rspamd_keypair_cache_new (16);
rspamd_mempool_lock_mutex (mtx);
sfd = fork ();
g_assert (sfd != -1);
if (sfd == 0) {
rspamd_http_server_func ("/tmp/", &addr, mtx, serv_key, c);
exit (EXIT_SUCCESS);
}
rspamd_mempool_lock_mutex (mtx);
/* Do client stuff */
for (i = 0; i < ntests; i ++) {
for (j = 0; j < pconns; j ++) {
rspamd_http_client_func (filepath + sizeof ("/tmp") - 1, &addr,
NULL, NULL, c, ev_base, &latency[i * pconns + j]);
}
clock_gettime (CLOCK_MONOTONIC, &ts1);
event_base_loop (ev_base, 0);
clock_gettime (CLOCK_MONOTONIC, &ts2);
diff = (ts2.tv_sec - ts1.tv_sec) * 1000. + /* Seconds */
(ts2.tv_nsec - ts1.tv_nsec) / 1000000.; /* Nanoseconds */
total_diff += diff;
}
msg_info ("Made %d connections of size %d in %.6f ms, %.6f cps",
ntests * pconns,
sizeof (buf) * file_blocks,
total_diff, ntests * pconns / total_diff * 1000.);
mean = rspamd_http_calculate_mean (latency, &std);
msg_info ("Latency: %.6f ms mean, %.6f dev",
mean, std);
/* Now test encrypted */
b32_key = rspamd_http_connection_print_key (serv_key,
RSPAMD_KEYPAIR_PUBKEY|RSPAMD_KEYPAIR_BASE32);
g_assert (b32_key != NULL);
peer_key = rspamd_http_connection_make_peer_key (b32_key->str);
g_assert (peer_key != NULL);
total_diff = 0.0;
for (i = 0; i < ntests; i ++) {
for (j = 0; j < pconns; j ++) {
rspamd_http_client_func (filepath + sizeof ("/tmp") - 1, &addr,
client_key, peer_key, c, ev_base, &latency[i * pconns + j]);
}
clock_gettime (CLOCK_MONOTONIC, &ts1);
event_base_loop (ev_base, 0);
clock_gettime (CLOCK_MONOTONIC, &ts2);
diff = (ts2.tv_sec - ts1.tv_sec) * 1000. + /* Seconds */
(ts2.tv_nsec - ts1.tv_nsec) / 1000000.; /* Nanoseconds */
total_diff += diff;
}
msg_info ("Made %d encrypted connections of size %d in %.6f ms, %.6f cps",
ntests * pconns,
sizeof (buf) * file_blocks,
total_diff, ntests * pconns / total_diff * 1000.);
mean = rspamd_http_calculate_mean (latency, &std);
msg_info ("Latency: %.6f ms mean, %.6f dev",
mean, std);
/* Restart server */
kill (sfd, SIGTERM);
wait (&i);
sfd = fork ();
g_assert (sfd != -1);
if (sfd == 0) {
rspamd_http_server_func ("/tmp/", &addr, mtx, serv_key, NULL);
exit (EXIT_SUCCESS);
}
rspamd_mempool_lock_mutex (mtx);
total_diff = 0.0;
for (i = 0; i < ntests; i ++) {
for (j = 0; j < pconns; j ++) {
rspamd_http_client_func (filepath + sizeof ("/tmp") - 1, &addr,
client_key, peer_key, c, ev_base, &latency[i * pconns + j]);
}
clock_gettime (CLOCK_MONOTONIC, &ts1);
event_base_loop (ev_base, 0);
clock_gettime (CLOCK_MONOTONIC, &ts2);
diff = (ts2.tv_sec - ts1.tv_sec) * 1000. + /* Seconds */
(ts2.tv_nsec - ts1.tv_nsec) / 1000000.; /* Nanoseconds */
total_diff += diff;
}
msg_info ("Made %d uncached encrypted connections of size %d in %.6f ms, %.6f cps",
ntests * pconns,
sizeof (buf) * file_blocks,
total_diff, ntests * pconns / total_diff * 1000.);
mean = rspamd_http_calculate_mean (latency, &std);
msg_info ("Latency: %.6f ms mean, %.6f dev",
mean, std);
close (fd);
unlink (filepath);
kill (sfd, SIGTERM);
}
static rspamd_fstring_t *
rspamd_redis_tokens_to_query (struct rspamd_task *task, GPtrArray *tokens,
const gchar *arg0, const gchar *arg1, gboolean learn, gint idx,
gboolean intvals)
{
rspamd_fstring_t *out;
rspamd_token_t *tok;
gchar n0[64], n1[64];
guint i, l0, l1, larg0, larg1;
guint64 num;
g_assert (tokens != NULL);
larg0 = strlen (arg0);
larg1 = strlen (arg1);
out = rspamd_fstring_sized_new (1024);
if (!learn) {
rspamd_printf_fstring (&out, ""
"*%d\r\n"
"$%d\r\n"
"%s\r\n"
"$%d\r\n"
"%s\r\n",
(tokens->len + 2),
larg0, arg0,
larg1, arg1);
}
for (i = 0; i < tokens->len; i ++) {
tok = g_ptr_array_index (tokens, i);
memcpy (&num, tok->data, sizeof (num));
if (learn) {
rspamd_printf_fstring (&out, ""
"*4\r\n"
"$%d\r\n"
"%s\r\n"
"$%d\r\n"
"%s\r\n",
larg0, arg0,
larg1, arg1);
l0 = rspamd_snprintf (n0, sizeof (n0), "%uL", num);
if (intvals) {
l1 = rspamd_snprintf (n1, sizeof (n1), "%L",
(gint64)tok->values[idx]);
}
else {
l1 = rspamd_snprintf (n1, sizeof (n1), "%f",
tok->values[idx]);
}
rspamd_printf_fstring (&out, ""
"$%d\r\n"
"%s\r\n"
"$%d\r\n"
"%s\r\n", l0, n0, l1, n1);
}
else {
l0 = rspamd_snprintf (n0, sizeof (n0), "%uL", num);
rspamd_printf_fstring (&out, ""
"$%d\r\n"
"%s\r\n", l0, n0);
}
}
return out;
}
/**
* Main file interface for logging
*/
static void
file_log_function (const gchar *module, const gchar *id,
const gchar *function,
gint level_flags,
const gchar *message,
gpointer arg)
{
gchar tmpbuf[256], timebuf[32], modulebuf[64];
gchar *m;
time_t now;
struct tm *tms;
struct iovec iov[5];
gulong r = 0, mr = 0;
guint64 cksum;
size_t mlen, mremain;
const gchar *cptype = NULL;
gboolean got_time = FALSE;
rspamd_logger_t *rspamd_log = arg;
if (!rspamd_log->enabled) {
return;
}
/* Check throttling due to write errors */
if (rspamd_log->throttling) {
now = time (NULL);
if (rspamd_log->throttling_time != now) {
rspamd_log->throttling_time = now;
got_time = TRUE;
}
else {
/* Do not try to write to file too often while throttling */
return;
}
}
/* Check repeats */
mlen = strlen (message);
cksum = rspamd_log_calculate_cksum (message, mlen);
if (cksum == rspamd_log->last_line_cksum) {
rspamd_log->repeats++;
if (rspamd_log->repeats > REPEATS_MIN && rspamd_log->repeats <
REPEATS_MAX) {
/* Do not log anything */
if (rspamd_log->saved_message == NULL) {
rspamd_log->saved_message = g_strdup (message);
rspamd_log->saved_function = g_strdup (function);
if (module) {
rspamd_log->saved_module = g_strdup (module);
}
if (id) {
rspamd_log->saved_id = g_strdup (id);
}
rspamd_log->saved_loglevel = level_flags;
}
return;
}
else if (rspamd_log->repeats > REPEATS_MAX) {
rspamd_snprintf (tmpbuf,
sizeof (tmpbuf),
"Last message repeated %ud times",
rspamd_log->repeats);
rspamd_log->repeats = 0;
/* It is safe to use temporary buffer here as it is not static */
if (rspamd_log->saved_message) {
file_log_function (rspamd_log->saved_module,
rspamd_log->saved_id,
rspamd_log->saved_function,
rspamd_log->saved_loglevel,
rspamd_log->saved_message,
arg);
g_free (rspamd_log->saved_message);
g_free (rspamd_log->saved_function);
g_free (rspamd_log->saved_module);
g_free (rspamd_log->saved_id);
rspamd_log->saved_message = NULL;
rspamd_log->saved_function = NULL;
rspamd_log->saved_module = NULL;
rspamd_log->saved_id = NULL;
}
file_log_function ("logger", NULL,
G_STRFUNC,
rspamd_log->saved_loglevel,
tmpbuf,
arg);
file_log_function (module, id,
function,
level_flags,
message,
arg);
rspamd_log->repeats = REPEATS_MIN + 1;
return;
}
}
else {
/* Reset counter if new message differs from saved message */
rspamd_log->last_line_cksum = cksum;
if (rspamd_log->repeats > REPEATS_MIN) {
rspamd_snprintf (tmpbuf,
sizeof (tmpbuf),
"Last message repeated %ud times",
rspamd_log->repeats);
rspamd_log->repeats = 0;
if (rspamd_log->saved_message) {
file_log_function (rspamd_log->saved_module,
rspamd_log->saved_id,
rspamd_log->saved_function,
rspamd_log->saved_loglevel,
rspamd_log->saved_message,
arg);
g_free (rspamd_log->saved_message);
g_free (rspamd_log->saved_function);
g_free (rspamd_log->saved_module);
g_free (rspamd_log->saved_id);
rspamd_log->saved_message = NULL;
rspamd_log->saved_function = NULL;
rspamd_log->saved_module = NULL;
rspamd_log->saved_id = NULL;
}
file_log_function ("logger", NULL,
G_STRFUNC,
level_flags,
tmpbuf,
arg);
/* It is safe to use temporary buffer here as it is not static */
file_log_function (module, id,
function,
level_flags,
message,
arg);
return;
}
else {
rspamd_log->repeats = 0;
}
}
if (rspamd_log->cfg->log_extended) {
if (!got_time) {
now = time (NULL);
}
/* Format time */
if (!rspamd_log->cfg->log_systemd) {
tms = localtime (&now);
strftime (timebuf, sizeof (timebuf), "%F %H:%M:%S", tms);
}
cptype = g_quark_to_string (rspamd_log->process_type);
if (rspamd_log->cfg->log_color) {
if (level_flags & G_LOG_LEVEL_INFO) {
/* White */
r = rspamd_snprintf (tmpbuf, sizeof (tmpbuf), "\033[0;37m");
}
else if (level_flags & G_LOG_LEVEL_WARNING) {
/* Magenta */
r = rspamd_snprintf (tmpbuf, sizeof (tmpbuf), "\033[0;32m");
}
else if (level_flags & G_LOG_LEVEL_CRITICAL) {
/* Red */
r = rspamd_snprintf (tmpbuf, sizeof (tmpbuf), "\033[1;31m");
}
}
else {
r = 0;
}
if (!rspamd_log->cfg->log_systemd) {
r += rspamd_snprintf (tmpbuf + r,
sizeof (tmpbuf) - r,
"%s #%P(%s) ",
timebuf,
rspamd_log->pid,
cptype);
}
else {
r += rspamd_snprintf (tmpbuf + r,
sizeof (tmpbuf) - r,
"(%s) ",
cptype);
}
modulebuf[0] = '\0';
mremain = sizeof (modulebuf);
m = modulebuf;
if (id != NULL) {
mr = rspamd_snprintf (m, mremain, "<%*.s>; ", LOG_ID,
id);
m += mr;
mremain -= mr;
}
if (module != NULL) {
mr = rspamd_snprintf (m, mremain, "%s; ", module);
m += mr;
mremain -= mr;
}
if (function != NULL) {
mr = rspamd_snprintf (m, mremain, "%s: ", function);
m += mr;
mremain -= mr;
}
else {
mr = rspamd_snprintf (m, mremain, ": ");
m += mr;
mremain -= mr;
}
/* Construct IOV for log line */
iov[0].iov_base = tmpbuf;
iov[0].iov_len = r;
iov[1].iov_base = modulebuf;
iov[1].iov_len = m - modulebuf;
iov[2].iov_base = (void *) message;
iov[2].iov_len = mlen;
iov[3].iov_base = (void *) &lf_chr;
iov[3].iov_len = 1;
if (rspamd_log->cfg->log_color) {
iov[4].iov_base = "\033[0m";
iov[4].iov_len = sizeof ("\033[0m") - 1;
/* Call helper (for buffering) */
file_log_helper (rspamd_log, iov, 5);
}
else {
/* Call helper (for buffering) */
file_log_helper (rspamd_log, iov, 4);
}
}
else {
iov[0].iov_base = (void *) message;
iov[0].iov_len = mlen;
iov[1].iov_base = (void *) &lf_chr;
iov[1].iov_len = 1;
if (rspamd_log->cfg->log_color) {
iov[2].iov_base = "\033[0m";
iov[2].iov_len = sizeof ("\033[0m") - 1;
/* Call helper (for buffering) */
file_log_helper (rspamd_log, iov, 3);
}
else {
/* Call helper (for buffering) */
file_log_helper (rspamd_log, iov, 2);
}
}
}
/**
* Async HTTP callback
*/
static void
http_callback (gint fd, short what, void *ud)
{
struct rspamd_map *map = ud;
struct http_map_data *data;
struct http_callback_data *cbd;
rspamd_mempool_t *pool;
gchar tmpbuf[PATH_MAX];
data = map->map_data;
pool = map->pool;
if (!g_atomic_int_compare_and_exchange (map->locked, 0, 1)) {
msg_debug_pool (
"don't try to reread map as it is locked by other process, will reread it later");
jitter_timeout_event (map, TRUE, FALSE, FALSE);
return;
}
/* Plan event */
cbd = g_slice_alloc0 (sizeof (struct http_callback_data));
rspamd_snprintf (tmpbuf, sizeof (tmpbuf),
"%s" G_DIR_SEPARATOR_S "rspamd_map%d-XXXXXX",
map->cfg->temp_dir, map->id);
cbd->out_fd = mkstemp (tmpbuf);
if (cbd->out_fd == -1) {
g_slice_free1 (sizeof (*cbd), cbd);
msg_err_pool ("cannot create tempfile: %s", strerror (errno));
jitter_timeout_event (map, FALSE, FALSE, TRUE);
g_atomic_int_set (map->locked, 0);
return;
}
cbd->tmpfile = g_strdup (tmpbuf);
cbd->ev_base = map->ev_base;
cbd->map = map;
cbd->data = data;
cbd->fd = -1;
cbd->cbdata.state = 0;
cbd->cbdata.prev_data = *cbd->map->user_data;
cbd->cbdata.cur_data = NULL;
cbd->cbdata.map = cbd->map;
cbd->stage = map_resolve_host2;
double_to_tv (map->cfg->map_timeout, &cbd->tv);
REF_INIT_RETAIN (cbd, free_http_cbdata);
msg_debug_pool ("reading map data from %s", data->host);
/* Send both A and AAAA requests */
if (map->r->r) {
if (rdns_make_request_full (map->r->r, rspamd_map_dns_callback, cbd,
map->cfg->dns_timeout, map->cfg->dns_retransmits, 1,
data->host, RDNS_REQUEST_A)) {
REF_RETAIN (cbd);
}
if (rdns_make_request_full (map->r->r, rspamd_map_dns_callback, cbd,
map->cfg->dns_timeout, map->cfg->dns_retransmits, 1,
data->host, RDNS_REQUEST_AAAA)) {
REF_RETAIN (cbd);
}
jitter_timeout_event (map, FALSE, FALSE, FALSE);
map->dtor = free_http_cbdata_dtor;
map->dtor_data = cbd;
}
else {
msg_warn_pool ("cannot load map: DNS resolver is not initialized");
jitter_timeout_event (map, FALSE, FALSE, TRUE);
}
/* We don't need own ref as it is now refcounted by DNS requests */
REF_RELEASE (cbd);
}
static int
http_map_finish (struct rspamd_http_connection *conn,
struct rspamd_http_message *msg)
{
struct http_callback_data *cbd = conn->ud;
struct rspamd_map *map;
rspamd_mempool_t *pool;
char fpath[PATH_MAX];
guchar *aux_data, *in = NULL;
gsize inlen = 0;
struct stat st;
map = cbd->map;
pool = cbd->map->pool;
if (msg->code == 200) {
if (cbd->stage == map_load_file) {
if (msg->last_modified) {
cbd->data->last_checked = msg->last_modified;
}
else {
cbd->data->last_checked = msg->date;
}
/* Maybe we need to check signature ? */
if (map->is_signed) {
close (cbd->out_fd);
if (map->trusted_pubkey) {
/* No need to load key */
cbd->stage = map_load_signature;
cbd->pk = rspamd_pubkey_ref (map->trusted_pubkey);
rspamd_snprintf (fpath, sizeof (fpath), "%s.sig",
cbd->tmpfile);
}
else {
rspamd_snprintf (fpath, sizeof (fpath), "%s.pub",
cbd->tmpfile);
cbd->stage = map_load_pubkey;
}
cbd->out_fd = rspamd_file_xopen (fpath, O_RDWR|O_CREAT, 00644);
if (cbd->out_fd == -1) {
msg_err_pool ("cannot open pubkey file %s for writing: %s",
fpath, strerror (errno));
goto end;
}
rspamd_http_connection_reset (cbd->conn);
write_http_request (cbd);
goto end;
}
else {
/* Unsinged version - just open file */
in = rspamd_file_xmap (cbd->tmpfile, PROT_READ, &inlen);
if (in == NULL) {
msg_err_pool ("cannot read tempfile %s: %s", cbd->tmpfile,
strerror (errno));
goto end;
}
}
}
else if (cbd->stage == map_load_pubkey) {
/* We now can load pubkey */
(void)lseek (cbd->out_fd, 0, SEEK_SET);
if (fstat (cbd->out_fd, &st) == -1) {
msg_err_pool ("cannot stat pubkey file %s: %s",
fpath, strerror (errno));
goto end;
}
aux_data = mmap (NULL, st.st_size, PROT_READ, MAP_SHARED,
cbd->out_fd, 0);
close (cbd->out_fd);
cbd->out_fd = -1;
if (aux_data == MAP_FAILED) {
msg_err_pool ("cannot map pubkey file %s: %s",
fpath, strerror (errno));
goto end;
}
cbd->pk = rspamd_pubkey_from_base32 (aux_data, st.st_size,
RSPAMD_KEYPAIR_SIGN, RSPAMD_CRYPTOBOX_MODE_25519);
munmap (aux_data, st.st_size);
if (cbd->pk == NULL) {
msg_err_pool ("cannot load pubkey file %s: bad pubkey",
fpath);
goto end;
}
rspamd_snprintf (fpath, sizeof (fpath), "%s.sig", cbd->tmpfile);
cbd->out_fd = rspamd_file_xopen (fpath, O_RDWR|O_CREAT, 00644);
if (cbd->out_fd == -1) {
msg_err_pool ("cannot open signature file %s for writing: %s",
fpath, strerror (errno));
goto end;
}
cbd->stage = map_load_signature;
rspamd_http_connection_reset (cbd->conn);
write_http_request (cbd);
goto end;
}
else if (cbd->stage == map_load_signature) {
/* We can now check signature */
close (cbd->out_fd);
cbd->out_fd = -1;
in = rspamd_file_xmap (cbd->tmpfile, PROT_READ, &inlen);
if (in == NULL) {
msg_err_pool ("cannot read tempfile %s: %s", cbd->tmpfile,
strerror (errno));
goto end;
}
if (!rspamd_map_check_sig_pk (cbd->tmpfile, map, in, inlen, cbd->pk)) {
goto end;
}
}
g_assert (in != NULL);
map->read_callback (map->pool, in, inlen, &cbd->cbdata, TRUE);
map->fin_callback (map->pool, &cbd->cbdata);
*map->user_data = cbd->cbdata.cur_data;
msg_info_pool ("read map data from %s", cbd->data->host);
}
else if (msg->code == 304 && cbd->stage == map_load_file) {
msg_debug_pool ("data is not modified for server %s",
cbd->data->host);
if (msg->last_modified) {
cbd->data->last_checked = msg->last_modified;
}
else {
cbd->data->last_checked = msg->date;
}
}
else {
msg_info_pool ("cannot load map %s from %s: HTTP error %d",
map->uri, cbd->data->host, msg->code);
}
end:
REF_RELEASE (cbd);
return 0;
}
static gboolean
rspamd_sqlite3_wait (const gchar *lock)
{
gint fd;
struct timespec sleep_ts = {
.tv_sec = 0,
.tv_nsec = 1000000
};
fd = open (lock, O_RDONLY);
if (fd == -1) {
msg_err ("cannot open lock file %s: %s", lock, strerror (errno));
return FALSE;
}
while (!rspamd_file_lock (fd, TRUE)) {
if (nanosleep (&sleep_ts, NULL) == -1 && errno != EINTR) {
close (fd);
msg_err ("cannot sleep open lock file %s: %s", lock, strerror (errno));
return FALSE;
}
}
rspamd_file_unlock (fd, FALSE);
close (fd);
return TRUE;
}
sqlite3 *
rspamd_sqlite3_open_or_create (const gchar *path, const
gchar *create_sql, GError **err)
{
sqlite3 *sqlite;
gint rc, flags, lock_fd;
gchar lock_path[PATH_MAX], dbdir[PATH_MAX], *pdir;
static const char sqlite_wal[] = "PRAGMA journal_mode=\"wal\";",
exclusive_lock_sql[] = "PRAGMA locking_mode=\"exclusive\";";
gboolean create = FALSE;
flags = SQLITE_OPEN_READWRITE;
#ifdef SQLITE_OPEN_SHAREDCACHE
flags |= SQLITE_OPEN_SHAREDCACHE;
#endif
#ifdef SQLITE_OPEN_WAL
flags |= SQLITE_OPEN_WAL;
#endif
rspamd_strlcpy (dbdir, path, sizeof (dbdir));
pdir = dirname (dbdir);
if (access (pdir, W_OK) == -1) {
g_set_error (err, rspamd_sqlite3_quark (),
errno, "cannot open sqlite directory %s: %s",
pdir, strerror (errno));
return NULL;
}
rspamd_snprintf (lock_path, sizeof (lock_path), "%s.lock", path);
if (access (path, R_OK) == -1 || access (lock_path, R_OK) != -1) {
flags |= SQLITE_OPEN_CREATE;
rspamd_snprintf (lock_path, sizeof (lock_path), "%s.lock", path);
lock_fd = open (lock_path, O_WRONLY|O_CREAT|O_EXCL, 00600);
if (lock_fd == -1 && (errno == EEXIST || errno == EBUSY)) {
msg_debug ("checking %s to wait for db being created", lock_path);
if (!rspamd_sqlite3_wait (lock_path)) {
g_set_error (err, rspamd_sqlite3_quark (),
errno, "cannot create sqlite file %s: %s",
path, strerror (errno));
return NULL;
}
/* At this point we have database created */
create = FALSE;
}
else {
msg_debug ("locking %s to block creating", lock_path);
g_assert (rspamd_file_lock (lock_fd, FALSE));
create = TRUE;
}
}
if ((rc = sqlite3_open_v2 (path, &sqlite,
flags, NULL)) != SQLITE_OK) {
#if SQLITE_VERSION_NUMBER >= 3008000
g_set_error (err, rspamd_sqlite3_quark (),
rc, "cannot open sqlite db %s: %s",
path, sqlite3_errstr (rc));
#else
g_set_error (err, rspamd_sqlite3_quark (),
rc, "cannot open sqlite db %s: %d",
path, rc);
#endif
return NULL;
}
if (create) {
if (sqlite3_exec (sqlite, sqlite_wal, NULL, NULL, NULL) != SQLITE_OK) {
msg_warn ("WAL mode is not supported (%s), locking issues might occur",
sqlite3_errmsg (sqlite));
}
if (sqlite3_exec (sqlite, exclusive_lock_sql, NULL, NULL, NULL) != SQLITE_OK) {
msg_warn ("cannot exclusively lock database to create schema: %s",
sqlite3_errmsg (sqlite));
}
if (sqlite3_exec (sqlite, create_sql, NULL, NULL, NULL) != SQLITE_OK) {
g_set_error (err, rspamd_sqlite3_quark (),
-1, "cannot execute create sql `%s`: %s",
create_sql, sqlite3_errmsg (sqlite));
sqlite3_close (sqlite);
rspamd_file_unlock (lock_fd, FALSE);
unlink (lock_path);
close (lock_fd);
return NULL;
}
msg_debug ("removing lock from %s", lock_path);
sqlite3_close (sqlite);
rspamd_file_unlock (lock_fd, FALSE);
unlink (lock_path);
close (lock_fd);
/* Reopen in normal mode */
msg_debug ("reopening %s in normal mode", path);
flags &= ~SQLITE_OPEN_CREATE;
if ((rc = sqlite3_open_v2 (path, &sqlite,
flags, NULL)) != SQLITE_OK) {
#if SQLITE_VERSION_NUMBER >= 3008000
g_set_error (err, rspamd_sqlite3_quark (),
rc, "cannot open sqlite db after creation %s: %s",
path, sqlite3_errstr (rc));
#else
g_set_error (err, rspamd_sqlite3_quark (),
rc, "cannot open sqlite db after creation %s: %d",
path, rc);
#endif
return NULL;
}
}
if (sqlite3_exec (sqlite, sqlite_wal, NULL, NULL, NULL) != SQLITE_OK) {
msg_warn ("WAL mode is not supported (%s), locking issues might occur",
sqlite3_errmsg (sqlite));
}
return sqlite;
}
static gboolean
sync_read (f_str_t * in, void *arg)
{
struct rspamd_sync_ctx *ctx = arg;
gchar buf[256];
guint64 rev = 0;
time_t ti = 0;
if (in->len == 0) {
/* Skip empty lines */
return TRUE;
}
switch (ctx->state) {
case SYNC_STATE_GREETING:
/* Skip greeting line and write sync command */
/* Write initial data */
statfile_get_revision (ctx->real_statfile, &rev, &ti);
rev = rspamd_snprintf (buf,
sizeof (buf),
"sync %s %uL %T" CRLF,
ctx->st->symbol,
rev,
ti);
ctx->state = SYNC_STATE_READ_LINE;
return rspamd_dispatcher_write (ctx->dispatcher, buf, rev, FALSE,
FALSE);
break;
case SYNC_STATE_READ_LINE:
/* Try to parse line from server */
if (!parse_revision_line (ctx, in)) {
msg_info ("cannot parse line of length %z: '%*s'",
in->len,
(gint)in->len,
in->begin);
close (ctx->sock);
rspamd_remove_dispatcher (ctx->dispatcher);
ctx->is_busy = FALSE;
return FALSE;
}
else if (ctx->state != SYNC_STATE_QUIT) {
if (ctx->new_len > 0) {
ctx->state = SYNC_STATE_READ_REV;
rspamd_set_dispatcher_policy (ctx->dispatcher,
BUFFER_CHARACTER,
ctx->new_len);
}
}
else {
/* Quit this session */
msg_info ("sync ended for: %s", ctx->st->symbol);
close (ctx->sock);
rspamd_remove_dispatcher (ctx->dispatcher);
ctx->is_busy = FALSE;
/* Immediately return from callback */
return FALSE;
}
break;
case SYNC_STATE_READ_REV:
/* In now contains all blocks of specified revision, so we can read them directly */
if (!read_blocks (ctx, in)) {
msg_info ("cannot read blocks");
close (ctx->sock);
rspamd_remove_dispatcher (ctx->dispatcher);
ctx->is_busy = FALSE;
return FALSE;
}
statfile_set_revision (ctx->real_statfile, ctx->new_rev, ctx->new_time);
msg_info ("set new revision: %uL, readed %z bytes",
ctx->new_rev,
in->len);
/* Now try to read other revision or END line */
ctx->state = SYNC_STATE_READ_LINE;
rspamd_set_dispatcher_policy (ctx->dispatcher, BUFFER_LINE, 0);
break;
case SYNC_STATE_QUIT:
close (ctx->sock);
rspamd_remove_dispatcher (ctx->dispatcher);
ctx->is_busy = FALSE;
return FALSE;
}
return TRUE;
}
void
rspamd_re_cache_init (struct rspamd_re_cache *cache, struct rspamd_config *cfg)
{
guint i, fl;
GHashTableIter it;
gpointer k, v;
struct rspamd_re_class *re_class;
rspamd_cryptobox_hash_state_t st_global;
rspamd_regexp_t *re;
struct rspamd_re_cache_elt *elt;
guchar hash_out[rspamd_cryptobox_HASHBYTES];
g_assert (cache != NULL);
rspamd_cryptobox_hash_init (&st_global, NULL, 0);
/* Resort all regexps */
g_ptr_array_sort (cache->re, rspamd_re_cache_sort_func);
for (i = 0; i < cache->re->len; i ++) {
elt = g_ptr_array_index (cache->re, i);
re = elt->re;
re_class = rspamd_regexp_get_class (re);
g_assert (re_class != NULL);
rspamd_regexp_set_cache_id (re, i);
if (re_class->st == NULL) {
re_class->st = g_slice_alloc (sizeof (*re_class->st));
rspamd_cryptobox_hash_init (re_class->st, NULL, 0);
}
/* Update hashes */
rspamd_cryptobox_hash_update (re_class->st, (gpointer) &re_class->id,
sizeof (re_class->id));
rspamd_cryptobox_hash_update (&st_global, (gpointer) &re_class->id,
sizeof (re_class->id));
rspamd_cryptobox_hash_update (re_class->st, rspamd_regexp_get_id (re),
rspamd_cryptobox_HASHBYTES);
rspamd_cryptobox_hash_update (&st_global, rspamd_regexp_get_id (re),
rspamd_cryptobox_HASHBYTES);
fl = rspamd_regexp_get_pcre_flags (re);
rspamd_cryptobox_hash_update (re_class->st, (const guchar *)&fl,
sizeof (fl));
rspamd_cryptobox_hash_update (&st_global, (const guchar *) &fl,
sizeof (fl));
fl = rspamd_regexp_get_flags (re);
rspamd_cryptobox_hash_update (re_class->st, (const guchar *) &fl,
sizeof (fl));
rspamd_cryptobox_hash_update (&st_global, (const guchar *) &fl,
sizeof (fl));
fl = rspamd_regexp_get_maxhits (re);
rspamd_cryptobox_hash_update (re_class->st, (const guchar *) &fl,
sizeof (fl));
rspamd_cryptobox_hash_update (&st_global, (const guchar *) &fl,
sizeof (fl));
}
rspamd_cryptobox_hash_final (&st_global, hash_out);
rspamd_snprintf (cache->hash, sizeof (cache->hash), "%*xs",
(gint) rspamd_cryptobox_HASHBYTES, hash_out);
/* Now finalize all classes */
g_hash_table_iter_init (&it, cache->re_classes);
while (g_hash_table_iter_next (&it, &k, &v)) {
re_class = v;
if (re_class->st) {
/*
* We finally update all classes with the number of expressions
* in the cache to ensure that if even a single re has been changed
* we won't be broken due to id mismatch
*/
rspamd_cryptobox_hash_update (re_class->st,
(gpointer)&cache->re->len,
sizeof (cache->re->len));
rspamd_cryptobox_hash_final (re_class->st, hash_out);
rspamd_snprintf (re_class->hash, sizeof (re_class->hash), "%*xs",
(gint) rspamd_cryptobox_HASHBYTES, hash_out);
g_slice_free1 (sizeof (*re_class->st), re_class->st);
re_class->st = NULL;
}
}
#ifdef WITH_HYPERSCAN
const gchar *platform = "generic";
rspamd_fstring_t *features = rspamd_fstring_new ();
cache->disable_hyperscan = cfg->disable_hyperscan;
cache->vectorized_hyperscan = cfg->vectorized_hyperscan;
g_assert (hs_populate_platform (&cache->plt) == HS_SUCCESS);
/* Now decode what we do have */
switch (cache->plt.tune) {
case HS_TUNE_FAMILY_HSW:
platform = "haswell";
break;
case HS_TUNE_FAMILY_SNB:
platform = "sandy";
break;
case HS_TUNE_FAMILY_BDW:
platform = "broadwell";
break;
case HS_TUNE_FAMILY_IVB:
platform = "ivy";
break;
default:
break;
}
if (cache->plt.cpu_features & HS_CPU_FEATURES_AVX2) {
features = rspamd_fstring_append (features, "AVX2", 4);
}
hs_set_allocator (g_malloc, g_free);
msg_info_re_cache ("loaded hyperscan engine witch cpu tune '%s' and features '%V'",
platform, features);
rspamd_fstring_free (features);
#endif
}
gboolean
rspamd_re_cache_load_hyperscan (struct rspamd_re_cache *cache,
const char *cache_dir)
{
g_assert (cache != NULL);
g_assert (cache_dir != NULL);
#ifndef WITH_HYPERSCAN
return FALSE;
#else
gchar path[PATH_MAX];
gint fd, i, n, *hs_ids = NULL, *hs_flags = NULL, total = 0, ret;
GHashTableIter it;
gpointer k, v;
guint8 *map, *p, *end;
struct rspamd_re_class *re_class;
struct rspamd_re_cache_elt *elt;
struct stat st;
g_hash_table_iter_init (&it, cache->re_classes);
while (g_hash_table_iter_next (&it, &k, &v)) {
re_class = v;
rspamd_snprintf (path, sizeof (path), "%s%c%s.hs", cache_dir,
G_DIR_SEPARATOR, re_class->hash);
if (rspamd_re_cache_is_valid_hyperscan_file (cache, path, FALSE, FALSE)) {
msg_debug_re_cache ("load hyperscan database from '%s'",
re_class->hash);
fd = open (path, O_RDONLY);
/* Read number of regexps */
g_assert (fd != -1);
fstat (fd, &st);
map = mmap (NULL, st.st_size, PROT_READ, MAP_SHARED, fd, 0);
if (map == MAP_FAILED) {
msg_err_re_cache ("cannot mmap %s: %s", path, strerror (errno));
close (fd);
return FALSE;
}
close (fd);
end = map + st.st_size;
p = map + RSPAMD_HS_MAGIC_LEN + sizeof (cache->plt);
n = *(gint *)p;
if (n <= 0 || 2 * n * sizeof (gint) + /* IDs + flags */
sizeof (guint64) + /* crc */
RSPAMD_HS_MAGIC_LEN + /* header */
sizeof (cache->plt) > (gsize)st.st_size) {
/* Some wrong amount of regexps */
msg_err_re_cache ("bad number of expressions in %s: %d",
path, n);
munmap (map, st.st_size);
return FALSE;
}
total += n;
p += sizeof (n);
hs_ids = g_malloc (n * sizeof (*hs_ids));
memcpy (hs_ids, p, n * sizeof (*hs_ids));
p += n * sizeof (*hs_ids);
hs_flags = g_malloc (n * sizeof (*hs_flags));
memcpy (hs_flags, p, n * sizeof (*hs_flags));
/* Skip crc */
p += n * sizeof (*hs_ids) + sizeof (guint64);
/* Cleanup */
if (re_class->hs_scratch != NULL) {
hs_free_scratch (re_class->hs_scratch);
}
if (re_class->hs_db != NULL) {
hs_free_database (re_class->hs_db);
}
if (re_class->hs_ids) {
g_free (re_class->hs_ids);
}
re_class->hs_ids = NULL;
re_class->hs_scratch = NULL;
re_class->hs_db = NULL;
if ((ret = hs_deserialize_database (p, end - p, &re_class->hs_db))
!= HS_SUCCESS) {
msg_err_re_cache ("bad hs database in %s: %d", path, ret);
munmap (map, st.st_size);
g_free (hs_ids);
g_free (hs_flags);
return FALSE;
}
munmap (map, st.st_size);
g_assert (hs_alloc_scratch (re_class->hs_db,
&re_class->hs_scratch) == HS_SUCCESS);
/*
* Now find hyperscan elts that are successfully compiled and
* specify that they should be matched using hyperscan
*/
for (i = 0; i < n; i ++) {
g_assert ((gint)cache->re->len > hs_ids[i] && hs_ids[i] >= 0);
elt = g_ptr_array_index (cache->re, hs_ids[i]);
if (hs_flags[i] & HS_FLAG_PREFILTER) {
elt->match_type = RSPAMD_RE_CACHE_HYPERSCAN_PRE;
}
else {
elt->match_type = RSPAMD_RE_CACHE_HYPERSCAN;
}
}
re_class->hs_ids = hs_ids;
g_free (hs_flags);
re_class->nhs = n;
}
else {
msg_err_re_cache ("invalid hyperscan hash file '%s'",
path);
return FALSE;
}
}
msg_info_re_cache ("hyperscan database of %d regexps has been loaded", total);
cache->hyperscan_loaded = TRUE;
return TRUE;
#endif
}
gint
rspamd_re_cache_compile_hyperscan (struct rspamd_re_cache *cache,
const char *cache_dir, gdouble max_time, gboolean silent,
GError **err)
{
g_assert (cache != NULL);
g_assert (cache_dir != NULL);
#ifndef WITH_HYPERSCAN
g_set_error (err, rspamd_re_cache_quark (), EINVAL, "hyperscan is disabled");
return -1;
#else
GHashTableIter it, cit;
gpointer k, v;
struct rspamd_re_class *re_class;
gchar path[PATH_MAX];
hs_database_t *test_db;
gint fd, i, n, *hs_ids = NULL, pcre_flags, re_flags;
guint64 crc;
rspamd_regexp_t *re;
hs_compile_error_t *hs_errors;
guint *hs_flags = NULL;
const gchar **hs_pats = NULL;
gchar *hs_serialized;
gsize serialized_len, total = 0;
struct iovec iov[7];
g_hash_table_iter_init (&it, cache->re_classes);
while (g_hash_table_iter_next (&it, &k, &v)) {
re_class = v;
rspamd_snprintf (path, sizeof (path), "%s%c%s.hs", cache_dir,
G_DIR_SEPARATOR, re_class->hash);
if (rspamd_re_cache_is_valid_hyperscan_file (cache, path, TRUE, TRUE)) {
fd = open (path, O_RDONLY, 00600);
/* Read number of regexps */
g_assert (fd != -1);
lseek (fd, RSPAMD_HS_MAGIC_LEN + sizeof (cache->plt), SEEK_SET);
read (fd, &n, sizeof (n));
close (fd);
if (re_class->type_len > 0) {
if (!silent) {
msg_info_re_cache (
"skip already valid class %s(%*s) to cache %6s, %d regexps",
rspamd_re_cache_type_to_string (re_class->type),
(gint) re_class->type_len - 1,
re_class->type_data,
re_class->hash,
n);
}
}
else {
if (!silent) {
msg_info_re_cache (
"skip already valid class %s to cache %6s, %d regexps",
rspamd_re_cache_type_to_string (re_class->type),
re_class->hash,
n);
}
}
continue;
}
fd = open (path, O_CREAT|O_TRUNC|O_EXCL|O_WRONLY, 00600);
if (fd == -1) {
g_set_error (err, rspamd_re_cache_quark (), errno, "cannot open file "
"%s: %s", path, strerror (errno));
return -1;
}
g_hash_table_iter_init (&cit, re_class->re);
n = g_hash_table_size (re_class->re);
hs_flags = g_malloc0 (sizeof (*hs_flags) * n);
hs_ids = g_malloc (sizeof (*hs_ids) * n);
hs_pats = g_malloc (sizeof (*hs_pats) * n);
i = 0;
while (g_hash_table_iter_next (&cit, &k, &v)) {
re = v;
pcre_flags = rspamd_regexp_get_pcre_flags (re);
re_flags = rspamd_regexp_get_flags (re);
if (re_flags & RSPAMD_REGEXP_FLAG_PCRE_ONLY) {
/* Do not try to compile bad regexp */
msg_info_re_cache (
"do not try compile %s to hyperscan as it is PCRE only",
rspamd_regexp_get_pattern (re));
continue;
}
hs_flags[i] = 0;
#ifndef WITH_PCRE2
if (pcre_flags & PCRE_FLAG(UTF8)) {
hs_flags[i] |= HS_FLAG_UTF8;
}
#else
if (pcre_flags & PCRE_FLAG(UTF)) {
hs_flags[i] |= HS_FLAG_UTF8;
}
#endif
if (pcre_flags & PCRE_FLAG(CASELESS)) {
hs_flags[i] |= HS_FLAG_CASELESS;
}
if (pcre_flags & PCRE_FLAG(MULTILINE)) {
hs_flags[i] |= HS_FLAG_MULTILINE;
}
if (pcre_flags & PCRE_FLAG(DOTALL)) {
hs_flags[i] |= HS_FLAG_DOTALL;
}
if (rspamd_regexp_get_maxhits (re) == 1) {
hs_flags[i] |= HS_FLAG_SINGLEMATCH;
}
if (hs_compile (rspamd_regexp_get_pattern (re),
hs_flags[i],
cache->vectorized_hyperscan ? HS_MODE_VECTORED : HS_MODE_BLOCK,
&cache->plt,
&test_db,
&hs_errors) != HS_SUCCESS) {
msg_info_re_cache ("cannot compile %s to hyperscan, try prefilter match",
rspamd_regexp_get_pattern (re));
hs_free_compile_error (hs_errors);
/* The approximation operation might take a significant
* amount of time, so we need to check if it's finite
*/
if (rspamd_re_cache_is_finite (cache, re, hs_flags[i], max_time)) {
hs_flags[i] |= HS_FLAG_PREFILTER;
hs_ids[i] = rspamd_regexp_get_cache_id (re);
hs_pats[i] = rspamd_regexp_get_pattern (re);
i++;
}
}
else {
hs_ids[i] = rspamd_regexp_get_cache_id (re);
hs_pats[i] = rspamd_regexp_get_pattern (re);
i ++;
hs_free_database (test_db);
}
}
/* Adjust real re number */
n = i;
if (n > 0) {
/* Create the hs tree */
if (hs_compile_multi (hs_pats,
hs_flags,
hs_ids,
n,
cache->vectorized_hyperscan ? HS_MODE_VECTORED : HS_MODE_BLOCK,
&cache->plt,
&test_db,
&hs_errors) != HS_SUCCESS) {
g_set_error (err, rspamd_re_cache_quark (), EINVAL,
"cannot create tree of regexp when processing '%s': %s",
hs_pats[hs_errors->expression], hs_errors->message);
g_free (hs_flags);
g_free (hs_ids);
g_free (hs_pats);
close (fd);
hs_free_compile_error (hs_errors);
return -1;
}
g_free (hs_pats);
if (hs_serialize_database (test_db, &hs_serialized,
&serialized_len) != HS_SUCCESS) {
g_set_error (err,
rspamd_re_cache_quark (),
errno,
"cannot serialize tree of regexp for %s",
re_class->hash);
close (fd);
g_free (hs_ids);
g_free (hs_flags);
hs_free_database (test_db);
return -1;
}
hs_free_database (test_db);
/*
* Magic - 8 bytes
* Platform - sizeof (platform)
* n - number of regexps
* n * <regexp ids>
* n * <regexp flags>
* crc - 8 bytes checksum
* <hyperscan blob>
*/
crc = XXH64 (hs_serialized, serialized_len, 0xdeadbabe);
if (cache->vectorized_hyperscan) {
iov[0].iov_base = (void *) rspamd_hs_magic_vector;
}
else {
iov[0].iov_base = (void *) rspamd_hs_magic;
}
iov[0].iov_len = RSPAMD_HS_MAGIC_LEN;
iov[1].iov_base = &cache->plt;
iov[1].iov_len = sizeof (cache->plt);
iov[2].iov_base = &n;
iov[2].iov_len = sizeof (n);
iov[3].iov_base = hs_ids;
iov[3].iov_len = sizeof (*hs_ids) * n;
iov[4].iov_base = hs_flags;
iov[4].iov_len = sizeof (*hs_flags) * n;
iov[5].iov_base = &crc;
iov[5].iov_len = sizeof (crc);
iov[6].iov_base = hs_serialized;
iov[6].iov_len = serialized_len;
if (writev (fd, iov, G_N_ELEMENTS (iov)) == -1) {
g_set_error (err,
rspamd_re_cache_quark (),
errno,
"cannot serialize tree of regexp to %s: %s",
path, strerror (errno));
close (fd);
g_free (hs_ids);
g_free (hs_flags);
g_free (hs_serialized);
return -1;
}
if (re_class->type_len > 0) {
msg_info_re_cache (
"compiled class %s(%*s) to cache %6s, %d regexps",
rspamd_re_cache_type_to_string (re_class->type),
(gint) re_class->type_len - 1,
re_class->type_data,
re_class->hash,
n);
}
else {
msg_info_re_cache (
"compiled class %s to cache %6s, %d regexps",
rspamd_re_cache_type_to_string (re_class->type),
re_class->hash,
n);
}
total += n;
g_free (hs_serialized);
g_free (hs_ids);
g_free (hs_flags);
}
close (fd);
}
return total;
#endif
}
gboolean
smtp_upstream_read_socket (rspamd_ftok_t * in, void *arg)
{
struct smtp_session *session = arg;
gchar outbuf[BUFSIZ];
gint r;
msg_debug ("in: %T, state: %d", in, session->upstream_state);
switch (session->upstream_state) {
case SMTP_STATE_GREETING:
r = check_smtp_ustream_reply (in, '2');
if (r == -1) {
session->error = rspamd_mempool_alloc (session->pool, in->len + 1);
rspamd_strlcpy (session->error, in->begin, in->len + 1);
/* XXX: assume upstream errors as critical errors */
session->state = SMTP_STATE_CRITICAL_ERROR;
rspamd_dispatcher_restore (session->dispatcher);
if (!rspamd_dispatcher_write (session->dispatcher, session->error,
in->len, FALSE, TRUE)) {
goto err;
}
if (!rspamd_dispatcher_write (session->dispatcher, CRLF,
sizeof (CRLF) - 1, FALSE, TRUE)) {
goto err;
}
rspamd_session_destroy (session->s);
return FALSE;
}
else if (r == 1) {
if (session->ctx->use_xclient) {
r = rspamd_snprintf (outbuf,
sizeof (outbuf),
"XCLIENT NAME=%s ADDR=%s" CRLF,
session->resolved ? session->hostname : "[UNDEFINED]",
inet_ntoa (session->client_addr));
session->upstream_state = SMTP_STATE_HELO;
return rspamd_dispatcher_write (session->upstream_dispatcher,
outbuf,
r,
FALSE,
FALSE);
}
else {
session->upstream_state = SMTP_STATE_FROM;
if (session->helo) {
r = rspamd_snprintf (outbuf, sizeof (outbuf), "%s %s" CRLF,
session->esmtp ? "EHLO" : "HELO",
session->helo);
}
else {
return smtp_upstream_read_socket (in, arg);
}
return rspamd_dispatcher_write (session->upstream_dispatcher,
outbuf,
r,
FALSE,
FALSE);
}
}
break;
case SMTP_STATE_HELO:
r = check_smtp_ustream_reply (in, '2');
if (r == -1) {
session->error = rspamd_mempool_alloc (session->pool, in->len + 1);
rspamd_strlcpy (session->error, in->begin, in->len + 1);
/* XXX: assume upstream errors as critical errors */
session->state = SMTP_STATE_CRITICAL_ERROR;
rspamd_dispatcher_restore (session->dispatcher);
if (!rspamd_dispatcher_write (session->dispatcher, session->error,
in->len, FALSE, TRUE)) {
goto err;
}
if (!rspamd_dispatcher_write (session->dispatcher, CRLF,
sizeof (CRLF) - 1, FALSE, TRUE)) {
goto err;
}
rspamd_session_destroy (session->s);
return FALSE;
}
else if (r == 1) {
session->upstream_state = SMTP_STATE_FROM;
if (session->helo) {
r = rspamd_snprintf (outbuf, sizeof (outbuf), "%s %s" CRLF,
session->esmtp ? "EHLO" : "HELO",
session->helo);
}
else {
return smtp_upstream_read_socket (in, arg);
}
return rspamd_dispatcher_write (session->upstream_dispatcher,
outbuf,
r,
FALSE,
FALSE);
}
break;
case SMTP_STATE_FROM:
r = check_smtp_ustream_reply (in, '2');
if (r == -1) {
session->error = rspamd_mempool_alloc (session->pool, in->len + 1);
rspamd_strlcpy (session->error, in->begin, in->len + 1);
/* XXX: assume upstream errors as critical errors */
session->state = SMTP_STATE_CRITICAL_ERROR;
rspamd_dispatcher_restore (session->dispatcher);
if (!rspamd_dispatcher_write (session->dispatcher, session->error,
in->len, FALSE, TRUE)) {
goto err;
}
if (!rspamd_dispatcher_write (session->dispatcher, CRLF,
sizeof (CRLF) - 1, FALSE, TRUE)) {
goto err;
}
rspamd_session_destroy (session->s);
return FALSE;
}
else if (r == 1) {
r = rspamd_snprintf (outbuf, sizeof (outbuf), "MAIL FROM: ");
r +=
smtp_upstream_write_list (session->from,
outbuf + r,
sizeof (outbuf) - r);
session->upstream_state = SMTP_STATE_RCPT;
return rspamd_dispatcher_write (session->upstream_dispatcher,
outbuf,
r,
FALSE,
FALSE);
}
break;
case SMTP_STATE_RCPT:
r = check_smtp_ustream_reply (in, '2');
if (r == -1) {
session->error = rspamd_mempool_alloc (session->pool, in->len + 1);
rspamd_strlcpy (session->error, in->begin, in->len + 1);
/* XXX: assume upstream errors as critical errors */
session->state = SMTP_STATE_CRITICAL_ERROR;
rspamd_dispatcher_restore (session->dispatcher);
if (!rspamd_dispatcher_write (session->dispatcher, session->error,
in->len, FALSE, TRUE)) {
goto err;
}
if (!rspamd_dispatcher_write (session->dispatcher, CRLF,
sizeof (CRLF) - 1, FALSE, TRUE)) {
goto err;
}
rspamd_session_destroy (session->s);
return FALSE;
}
else if (r == 1) {
r = rspamd_snprintf (outbuf, sizeof (outbuf), "RCPT TO: ");
session->cur_rcpt = g_list_first (session->rcpt);
r += smtp_upstream_write_list (session->cur_rcpt->data,
outbuf + r,
sizeof (outbuf) - r);
session->cur_rcpt = g_list_next (session->cur_rcpt);
session->upstream_state = SMTP_STATE_BEFORE_DATA;
return rspamd_dispatcher_write (session->upstream_dispatcher,
outbuf,
r,
FALSE,
FALSE);
}
break;
case SMTP_STATE_BEFORE_DATA:
r = check_smtp_ustream_reply (in, '2');
if (r == -1) {
session->error = rspamd_mempool_alloc (session->pool, in->len + 1);
rspamd_strlcpy (session->error, in->begin, in->len + 1);
rspamd_dispatcher_restore (session->dispatcher);
if (!rspamd_dispatcher_write (session->dispatcher, session->error,
in->len, FALSE, TRUE)) {
goto err;
}
if (!rspamd_dispatcher_write (session->dispatcher, CRLF,
sizeof (CRLF) - 1, FALSE, TRUE)) {
goto err;
}
if (session->cur_rcpt) {
session->rcpt = g_list_delete_link (session->rcpt,
session->cur_rcpt);
}
else {
session->rcpt =
g_list_delete_link (session->rcpt, session->rcpt);
}
session->errors++;
session->state = SMTP_STATE_RCPT;
return TRUE;
}
else if (r == 1) {
if (session->cur_rcpt != NULL) {
r = rspamd_snprintf (outbuf, sizeof (outbuf), "RCPT TO: ");
r += smtp_upstream_write_list (session->cur_rcpt,
outbuf + r,
sizeof (outbuf) - r);
session->cur_rcpt = g_list_next (session->cur_rcpt);
if (!rspamd_dispatcher_write (session->upstream_dispatcher,
outbuf, r, FALSE, FALSE)) {
goto err;
}
}
else {
session->upstream_state = SMTP_STATE_DATA;
rspamd_dispatcher_pause (session->upstream_dispatcher);
}
session->error = rspamd_mempool_alloc (session->pool, in->len + 1);
rspamd_strlcpy (session->error, in->begin, in->len + 1);
/* Write to client */
if (!rspamd_dispatcher_write (session->dispatcher, session->error,
in->len, FALSE, TRUE)) {
goto err;
}
if (!rspamd_dispatcher_write (session->dispatcher, CRLF,
sizeof (CRLF) - 1, FALSE, TRUE)) {
goto err;
}
if (session->state == SMTP_STATE_WAIT_UPSTREAM) {
rspamd_dispatcher_restore (session->dispatcher);
session->state = SMTP_STATE_RCPT;
}
}
break;
case SMTP_STATE_DATA:
r = check_smtp_ustream_reply (in, '3');
if (r == -1) {
session->error = rspamd_mempool_alloc (session->pool, in->len + 1);
rspamd_strlcpy (session->error, in->begin, in->len + 1);
/* XXX: assume upstream errors as critical errors */
session->state = SMTP_STATE_CRITICAL_ERROR;
rspamd_dispatcher_restore (session->dispatcher);
if (!rspamd_dispatcher_write (session->dispatcher, session->error,
0, FALSE, TRUE)) {
goto err;
}
if (!rspamd_dispatcher_write (session->dispatcher, CRLF,
sizeof (CRLF) - 1, FALSE, TRUE)) {
goto err;
}
rspamd_session_destroy (session->s);
return FALSE;
}
else if (r == 1) {
if (!make_smtp_tempfile (session)) {
session->error = SMTP_ERROR_FILE;
session->state = SMTP_STATE_CRITICAL_ERROR;
rspamd_dispatcher_restore (session->dispatcher);
if (!rspamd_dispatcher_write (session->dispatcher,
session->error, 0, FALSE, TRUE)) {
goto err;
}
rspamd_session_destroy (session->s);
return FALSE;
}
session->state = SMTP_STATE_AFTER_DATA;
session->error = SMTP_ERROR_DATA_OK;
rspamd_dispatcher_restore (session->dispatcher);
if (!rspamd_dispatcher_write (session->dispatcher, session->error,
0, FALSE, TRUE)) {
goto err;
}
rspamd_dispatcher_pause (session->upstream_dispatcher);
rspamd_set_dispatcher_policy (session->dispatcher, BUFFER_LINE, 0);
session->dispatcher->strip_eol = FALSE;
return TRUE;
}
break;
case SMTP_STATE_AFTER_DATA:
session->error = rspamd_mempool_alloc (session->pool, in->len + 1);
rspamd_strlcpy (session->error, in->begin, in->len + 1);
session->state = SMTP_STATE_DATA;
rspamd_dispatcher_restore (session->dispatcher);
if (!rspamd_dispatcher_write (session->dispatcher, session->error, 0,
FALSE, TRUE)) {
goto err;
}
if (!rspamd_dispatcher_write (session->dispatcher, CRLF, sizeof (CRLF) -
1, FALSE, TRUE)) {
goto err;
}
if (!rspamd_dispatcher_write (session->upstream_dispatcher, "QUIT" CRLF,
sizeof ("QUIT" CRLF) - 1, FALSE, TRUE)) {
goto err;
}
session->upstream_state = SMTP_STATE_END;
return TRUE;
break;
case SMTP_STATE_END:
r = check_smtp_ustream_reply (in, '5');
if (r == -1) {
session->error = rspamd_mempool_alloc (session->pool, in->len + 1);
rspamd_strlcpy (session->error, in->begin, in->len + 1);
/* XXX: assume upstream errors as critical errors */
session->state = SMTP_STATE_CRITICAL_ERROR;
rspamd_dispatcher_restore (session->dispatcher);
if (!rspamd_dispatcher_write (session->dispatcher, session->error,
0, FALSE, TRUE)) {
goto err;
}
if (!rspamd_dispatcher_write (session->dispatcher, CRLF,
sizeof (CRLF) - 1, FALSE, TRUE)) {
goto err;
}
rspamd_session_destroy (session->s);
return FALSE;
}
else {
rspamd_session_remove_event (session->s,
(event_finalizer_t)smtp_upstream_finalize_connection,
session);
}
return FALSE;
break;
default:
msg_err ("got upstream reply at unexpected state: %d, reply: %T",
session->upstream_state,
in);
session->state = SMTP_STATE_CRITICAL_ERROR;
rspamd_dispatcher_restore (session->dispatcher);
if (!rspamd_dispatcher_write (session->dispatcher, session->error, 0,
FALSE, TRUE)) {
goto err;
}
if (!rspamd_dispatcher_write (session->dispatcher, CRLF, sizeof (CRLF) -
1, FALSE, TRUE)) {
goto err;
}
rspamd_session_destroy (session->s);
return FALSE;
}
return TRUE;
err:
msg_warn ("write error occured");
return FALSE;
}
static void
smtp_metric_callback (gpointer key, gpointer value, gpointer ud)
{
struct smtp_metric_callback_data *cd = ud;
struct metric_result *metric_res = value;
enum rspamd_metric_action action = METRIC_ACTION_NOACTION;
double ms = 0, rs = 0;
gboolean is_spam = FALSE;
struct rspamd_task *task;
task = cd->session->task;
/* XXX rewrite */
ms = metric_res->metric->actions[METRIC_ACTION_REJECT].score;
rs = metric_res->metric->actions[METRIC_ACTION_REJECT].score;
#if 0
if (!check_metric_settings (metric_res, &ms, &rs)) {
ms = metric_res->metric->actions[METRIC_ACTION_REJECT].score;
rs = metric_res->metric->actions[METRIC_ACTION_REJECT].score;
}
if (!check_metric_action_settings (task, metric_res, metric_res->score,
&action)) {
action =
check_metric_action (metric_res->score, ms, metric_res->metric);
}
#endif
if (metric_res->score >= ms) {
is_spam = 1;
}
if (action < cd->action) {
cd->action = action;
cd->res = metric_res;
}
if (!task->is_skipped) {
cd->log_offset += rspamd_snprintf (cd->log_buf + cd->log_offset,
cd->log_size - cd->log_offset,
"(%s: %c (%s): [%.2f/%.2f/%.2f] [",
(gchar *)key,
is_spam ? 'T' : 'F',
rspamd_action_to_str (action),
metric_res->score,
ms,
rs);
}
else {
cd->log_offset += rspamd_snprintf (cd->log_buf + cd->log_offset,
cd->log_size - cd->log_offset,
"(%s: %c (default): [%.2f/%.2f/%.2f] [",
(gchar *)key,
'S',
metric_res->score,
ms,
rs);
}
g_hash_table_foreach (metric_res->symbols, smtp_metric_symbols_callback,
cd);
/* Remove last , from log buf */
if (cd->log_buf[cd->log_offset - 1] == ',') {
cd->log_buf[--cd->log_offset] = '\0';
}
#ifdef HAVE_CLOCK_GETTIME
cd->log_offset += rspamd_snprintf (cd->log_buf + cd->log_offset,
cd->log_size - cd->log_offset,
"]), len: %z, time: %s,",
task->msg.len,
calculate_check_time (&task->tv, &task->ts, task->cfg->clock_res,
&task->scan_milliseconds));
#else
cd->log_offset += rspamd_snprintf (cd->log_buf + cd->log_offset,
cd->log_size - cd->log_offset,
"]), len: %z, time: %s,",
task->msg.len,
calculate_check_time (&task->tv, task->cfg->clock_res,
&task->scan_milliseconds));
#endif
}
gboolean
write_smtp_reply (struct smtp_session *session)
{
gchar logbuf[1024], *new_subject;
const gchar *old_subject;
struct smtp_metric_callback_data cd;
GMimeStream *stream;
gint old_fd, sublen;
/* Check metrics */
cd.session = session;
cd.action = METRIC_ACTION_NOACTION;
cd.res = NULL;
cd.log_buf = logbuf;
cd.log_offset = rspamd_snprintf (logbuf,
sizeof (logbuf),
"id: <%s>, qid: <%s>, ",
session->task->message_id,
session->task->queue_id);
cd.log_size = sizeof (logbuf);
if (session->task->user) {
cd.log_offset += rspamd_snprintf (logbuf + cd.log_offset,
sizeof (logbuf) - cd.log_offset,
"user: %s, ",
session->task->user);
}
g_hash_table_foreach (session->task->results, smtp_metric_callback, &cd);
msg_info ("%s", logbuf);
if (cd.action <= METRIC_ACTION_REJECT) {
if (!rspamd_dispatcher_write (session->dispatcher,
session->ctx->reject_message, 0, FALSE, TRUE)) {
return FALSE;
}
if (!rspamd_dispatcher_write (session->dispatcher, CRLF, sizeof (CRLF) -
1, FALSE, TRUE)) {
return FALSE;
}
destroy_session (session->s);
return FALSE;
}
else if (cd.action <= METRIC_ACTION_ADD_HEADER || cd.action <=
METRIC_ACTION_REWRITE_SUBJECT) {
old_fd = session->temp_fd;
if (!make_smtp_tempfile (session)) {
session->error = SMTP_ERROR_FILE;
session->state = SMTP_STATE_CRITICAL_ERROR;
rspamd_dispatcher_restore (session->dispatcher);
if (!rspamd_dispatcher_write (session->dispatcher, session->error,
0, FALSE, TRUE)) {
goto err;
}
destroy_session (session->s);
return FALSE;
}
if (cd.action <= METRIC_ACTION_REWRITE_SUBJECT) {
/* XXX: add this action */
old_subject = g_mime_message_get_subject (session->task->message);
if (old_subject != NULL) {
sublen = strlen (old_subject) + sizeof (SPAM_SUBJECT);
new_subject = rspamd_mempool_alloc (session->pool, sublen);
rspamd_snprintf (new_subject,
sublen,
"%s%s",
SPAM_SUBJECT,
old_subject);
}
else {
new_subject = SPAM_SUBJECT;
}
g_mime_message_set_subject (session->task->message, new_subject);
}
else if (cd.action <= METRIC_ACTION_ADD_HEADER) {
#ifndef GMIME24
g_mime_message_add_header (session->task->message, "X-Spam",
"true");
#else
g_mime_object_append_header (GMIME_OBJECT (
session->task->message), "X-Spam", "true");
#endif
}
stream = g_mime_stream_fs_new (session->temp_fd);
g_mime_stream_fs_set_owner (GMIME_STREAM_FS (stream), FALSE);
close (old_fd);
if (g_mime_object_write_to_stream (GMIME_OBJECT (session->task->message),
stream) == -1) {
msg_err ("cannot write MIME object to stream: %s",
strerror (errno));
session->error = SMTP_ERROR_FILE;
session->state = SMTP_STATE_CRITICAL_ERROR;
rspamd_dispatcher_restore (session->dispatcher);
if (!rspamd_dispatcher_write (session->dispatcher, session->error,
0, FALSE, TRUE)) {
goto err;
}
destroy_session (session->s);
return FALSE;
}
g_object_unref (stream);
}
/* XXX: Add other actions */
return smtp_send_upstream_message (session);
err:
session->error = SMTP_ERROR_FILE;
session->state = SMTP_STATE_CRITICAL_ERROR;
if (!rspamd_dispatcher_write (session->dispatcher, session->error, 0, FALSE,
TRUE)) {
return FALSE;
}
destroy_session (session->s);
return FALSE;
}