/* Process regexp expression */
static gboolean
read_regexp_expression (rspamd_mempool_t * pool,
struct regexp_module_item *chain,
const gchar *symbol,
const gchar *line,
struct rspamd_config *cfg)
{
struct rspamd_expression *e = NULL;
GError *err = NULL;
if (!rspamd_parse_expression (line, 0, &mime_expr_subr, cfg, pool, &err,
&e)) {
msg_warn_pool ("%s = \"%s\" is invalid regexp expression: %e", symbol,
line,
err);
g_error_free (err);
return FALSE;
}
g_assert (e != NULL);
chain->expr = e;
return TRUE;
}
struct rspamd_content_disposition *
rspamd_content_disposition_parse (const gchar *in,
gsize len, rspamd_mempool_t *pool)
{
struct rspamd_content_disposition *res = NULL, val;
val.lc_data = rspamd_mempool_alloc (pool, len);
memcpy (val.lc_data, in, len);
rspamd_str_lc (val.lc_data, len);
if (rspamd_content_disposition_parser (in, len, &val, pool)) {
res = rspamd_mempool_alloc (pool, sizeof (val));
memcpy (res, &val, sizeof (val));
if (res->attrs) {
rspamd_mempool_add_destructor (pool,
(rspamd_mempool_destruct_t)g_hash_table_unref, res->attrs);
}
}
else {
msg_warn_pool ("cannot parse content disposition: %*s",
(gint)len, val.lc_data);
}
return res;
}
/**
* Callback for reading data from file
*/
static void
read_map_file (struct rspamd_map *map, struct file_map_data *data)
{
struct map_cb_data cbdata;
gchar buf[BUFSIZ], *remain;
ssize_t r;
gint fd, rlen, tlen;
rspamd_mempool_t *pool = map->pool;
if (map->read_callback == NULL || map->fin_callback == NULL) {
msg_err_pool ("bad callback for reading map file");
return;
}
if ((fd = open (data->filename, O_RDONLY)) == -1) {
msg_warn_pool ("cannot open file '%s': %s", data->filename,
strerror (errno));
return;
}
cbdata.state = 0;
cbdata.prev_data = *map->user_data;
cbdata.cur_data = NULL;
cbdata.map = map;
rlen = 0;
tlen = 0;
while ((r = read (fd, buf + rlen, sizeof (buf) - rlen - 2)) > 0) {
r += rlen;
tlen += r;
buf[r] = '\0';
remain = map->read_callback (map->pool, buf, r, &cbdata);
if (remain != NULL) {
/* copy remaining buffer to start of buffer */
rlen = r - (remain - buf);
memmove (buf, remain, rlen);
}
else {
rlen = 0;
}
}
if (remain != NULL && remain > buf) {
g_assert (rlen <= (gint)sizeof (buf) - 2);
buf[rlen++] = '\n';
buf[rlen] = '\0';
tlen += rlen;
map->read_callback (map->pool, buf, rlen, &cbdata);
}
close (fd);
if (tlen > 0) {
map->fin_callback (map->pool, &cbdata);
*map->user_data = cbdata.cur_data;
}
}
static void
free_http_cbdata_dtor (gpointer p)
{
struct http_callback_data *cbd = p;
rspamd_mempool_t *pool;
pool = cbd->map->pool;
msg_warn_pool ("connection with http server is terminated: worker is stopping");
free_http_cbdata_common (cbd);
}
/**
* Async HTTP callback
*/
static void
http_callback (gint fd, short what, void *ud)
{
struct rspamd_map *map = ud;
struct http_map_data *data;
struct http_callback_data *cbd;
rspamd_mempool_t *pool;
gchar tmpbuf[PATH_MAX];
data = map->map_data;
pool = map->pool;
if (!g_atomic_int_compare_and_exchange (map->locked, 0, 1)) {
msg_debug_pool (
"don't try to reread map as it is locked by other process, will reread it later");
jitter_timeout_event (map, TRUE, FALSE, FALSE);
return;
}
/* Plan event */
cbd = g_slice_alloc0 (sizeof (struct http_callback_data));
rspamd_snprintf (tmpbuf, sizeof (tmpbuf),
"%s" G_DIR_SEPARATOR_S "rspamd_map%d-XXXXXX",
map->cfg->temp_dir, map->id);
cbd->out_fd = mkstemp (tmpbuf);
if (cbd->out_fd == -1) {
g_slice_free1 (sizeof (*cbd), cbd);
msg_err_pool ("cannot create tempfile: %s", strerror (errno));
jitter_timeout_event (map, FALSE, FALSE, TRUE);
g_atomic_int_set (map->locked, 0);
return;
}
cbd->tmpfile = g_strdup (tmpbuf);
cbd->ev_base = map->ev_base;
cbd->map = map;
cbd->data = data;
cbd->fd = -1;
cbd->cbdata.state = 0;
cbd->cbdata.prev_data = *cbd->map->user_data;
cbd->cbdata.cur_data = NULL;
cbd->cbdata.map = cbd->map;
cbd->stage = map_resolve_host2;
double_to_tv (map->cfg->map_timeout, &cbd->tv);
REF_INIT_RETAIN (cbd, free_http_cbdata);
msg_debug_pool ("reading map data from %s", data->host);
/* Send both A and AAAA requests */
if (map->r->r) {
if (rdns_make_request_full (map->r->r, rspamd_map_dns_callback, cbd,
map->cfg->dns_timeout, map->cfg->dns_retransmits, 1,
data->host, RDNS_REQUEST_A)) {
REF_RETAIN (cbd);
}
if (rdns_make_request_full (map->r->r, rspamd_map_dns_callback, cbd,
map->cfg->dns_timeout, map->cfg->dns_retransmits, 1,
data->host, RDNS_REQUEST_AAAA)) {
REF_RETAIN (cbd);
}
jitter_timeout_event (map, FALSE, FALSE, FALSE);
map->dtor = free_http_cbdata_dtor;
map->dtor_data = cbd;
}
else {
msg_warn_pool ("cannot load map: DNS resolver is not initialized");
jitter_timeout_event (map, FALSE, FALSE, TRUE);
}
/* We don't need own ref as it is now refcounted by DNS requests */
REF_RELEASE (cbd);
}
static struct rspamd_osb_tokenizer_config *
rspamd_tokenizer_osb_config_from_ucl (rspamd_mempool_t * pool,
const ucl_object_t *obj)
{
const ucl_object_t *elt;
struct rspamd_osb_tokenizer_config *cf, *def;
guchar *key = NULL;
gsize keylen;
if (pool != NULL) {
cf = rspamd_mempool_alloc0 (pool, sizeof (*cf));
}
else {
cf = g_malloc0 (sizeof (*cf));
}
/* Use default config */
def = rspamd_tokenizer_osb_default_config ();
memcpy (cf, def, sizeof (*cf));
elt = ucl_object_lookup (obj, "hash");
if (elt != NULL && ucl_object_type (elt) == UCL_STRING) {
if (g_ascii_strncasecmp (ucl_object_tostring (elt), "xxh", 3)
== 0) {
cf->ht = RSPAMD_OSB_HASH_XXHASH;
elt = ucl_object_lookup (obj, "seed");
if (elt != NULL && ucl_object_type (elt) == UCL_INT) {
cf->seed = ucl_object_toint (elt);
}
}
else if (g_ascii_strncasecmp (ucl_object_tostring (elt), "sip", 3)
== 0) {
cf->ht = RSPAMD_OSB_HASH_SIPHASH;
elt = ucl_object_lookup (obj, "key");
if (elt != NULL && ucl_object_type (elt) == UCL_STRING) {
key = rspamd_decode_base32 (ucl_object_tostring (elt),
0, &keylen);
if (keylen < sizeof (rspamd_sipkey_t)) {
msg_warn ("siphash key is too short: %z", keylen);
g_free (key);
}
else {
memcpy (cf->sk, key, sizeof (cf->sk));
g_free (key);
}
}
else {
msg_warn_pool ("siphash cannot be used without key");
}
}
}
else {
elt = ucl_object_lookup (obj, "compat");
if (elt != NULL && ucl_object_toboolean (elt)) {
cf->ht = RSPAMD_OSB_HASH_COMPAT;
}
}
elt = ucl_object_lookup (obj, "window");
if (elt != NULL && ucl_object_type (elt) == UCL_INT) {
cf->window_size = ucl_object_toint (elt);
if (cf->window_size > DEFAULT_FEATURE_WINDOW_SIZE * 4) {
msg_err_pool ("too large window size: %d", cf->window_size);
cf->window_size = DEFAULT_FEATURE_WINDOW_SIZE;
}
}
return cf;
}
int
rspamd_sqlite3_run_prstmt (rspamd_mempool_t *pool, sqlite3 *db, GArray *stmts,
gint idx, ...)
{
gint retcode;
va_list ap;
sqlite3_stmt *stmt;
gint i, rowid, nargs, j;
gint64 len;
gpointer p;
struct rspamd_sqlite3_prstmt *nst;
const char *argtypes;
if (idx < 0 || idx >= (gint)stmts->len) {
return -1;
}
nst = &g_array_index (stmts, struct rspamd_sqlite3_prstmt, idx);
stmt = nst->stmt;
g_assert (nst != NULL);
msg_debug_pool ("executing `%s`", nst->sql);
argtypes = nst->args;
sqlite3_clear_bindings (stmt);
sqlite3_reset (stmt);
va_start (ap, idx);
nargs = 1;
for (i = 0, rowid = 1; argtypes[i] != '\0'; i ++) {
switch (argtypes[i]) {
case 'T':
for (j = 0; j < nargs; j ++, rowid ++) {
sqlite3_bind_text (stmt, rowid, va_arg (ap, const char*), -1,
SQLITE_STATIC);
}
nargs = 1;
break;
case 'V':
case 'B':
for (j = 0; j < nargs; j ++, rowid ++) {
len = va_arg (ap, gint64);
sqlite3_bind_text (stmt, rowid, va_arg (ap, const char*), len,
SQLITE_STATIC);
}
nargs = 1;
break;
case 'I':
for (j = 0; j < nargs; j ++, rowid ++) {
sqlite3_bind_int64 (stmt, rowid, va_arg (ap, gint64));
}
nargs = 1;
break;
case 'S':
for (j = 0; j < nargs; j ++, rowid ++) {
sqlite3_bind_int (stmt, rowid, va_arg (ap, gint));
}
nargs = 1;
break;
case '*':
nargs = va_arg (ap, gint);
break;
}
}
va_end (ap);
retcode = sqlite3_step (stmt);
if (retcode == nst->result) {
argtypes = nst->ret;
for (i = 0; argtypes != NULL && argtypes[i] != '\0'; i ++) {
switch (argtypes[i]) {
case 'T':
*va_arg (ap, char**) = g_strdup (sqlite3_column_text (stmt, i));
break;
case 'I':
*va_arg (ap, gint64*) = sqlite3_column_int64 (stmt, i);
break;
case 'S':
*va_arg (ap, int*) = sqlite3_column_int (stmt, i);
break;
case 'L':
*va_arg (ap, gint64*) = sqlite3_last_insert_rowid (db);
break;
case 'B':
len = sqlite3_column_bytes (stmt, i);
g_assert (len >= 0);
p = g_malloc (len);
memcpy (p, sqlite3_column_blob (stmt, i), len);
*va_arg (ap, gint64*) = len;
*va_arg (ap, gpointer*) = p;
break;
}
}
if (!(nst->flags & RSPAMD_SQLITE3_STMT_MULTIPLE)) {
sqlite3_clear_bindings (stmt);
sqlite3_reset (stmt);
}
return SQLITE_OK;
}
else if (retcode != SQLITE_DONE && retcode != SQLITE_OK && retcode != SQLITE_ROW) {
msg_warn_pool ("failed to execute query %s: %d, %s", nst->sql,
retcode, sqlite3_errmsg (db));
}
if (!(nst->flags & RSPAMD_SQLITE3_STMT_MULTIPLE)) {
sqlite3_clear_bindings (stmt);
sqlite3_reset (stmt);
}
return retcode;
}
static gboolean
rspamd_sqlite3_wait (rspamd_mempool_t *pool, const gchar *lock)
{
gint fd;
struct timespec sleep_ts = {
.tv_sec = 0,
.tv_nsec = 1000000
};
fd = open (lock, O_RDONLY);
if (fd == -1) {
msg_err_pool ("cannot open lock file %s: %s", lock, strerror (errno));
return FALSE;
}
while (!rspamd_file_lock (fd, TRUE)) {
if (nanosleep (&sleep_ts, NULL) == -1 && errno != EINTR) {
close (fd);
msg_err_pool ("cannot sleep open lock file %s: %s", lock, strerror (errno));
return FALSE;
}
}
rspamd_file_unlock (fd, FALSE);
close (fd);
return TRUE;
}
sqlite3 *
rspamd_sqlite3_open_or_create (rspamd_mempool_t *pool, const gchar *path, const
gchar *create_sql, GError **err)
{
sqlite3 *sqlite;
gint rc, flags, lock_fd;
gchar lock_path[PATH_MAX], dbdir[PATH_MAX], *pdir;
static const char sqlite_wal[] = "PRAGMA journal_mode=\"wal\";",
exclusive_lock_sql[] = "PRAGMA locking_mode=\"exclusive\";",
fsync_sql[] = "PRAGMA synchronous=1;",
foreign_keys[] = "PRAGMA foreign_keys=\"ON\";",
enable_mmap[] = "PRAGMA mmap_size=268435456;";
gboolean create = FALSE, has_lock = FALSE;
flags = SQLITE_OPEN_READWRITE;
#ifdef SQLITE_OPEN_SHAREDCACHE
flags |= SQLITE_OPEN_SHAREDCACHE;
#endif
#ifdef SQLITE_OPEN_WAL
flags |= SQLITE_OPEN_WAL;
#endif
rspamd_strlcpy (dbdir, path, sizeof (dbdir));
pdir = dirname (dbdir);
if (access (pdir, W_OK) == -1) {
g_set_error (err, rspamd_sqlite3_quark (),
errno, "cannot open sqlite directory %s: %s",
pdir, strerror (errno));
return NULL;
}
rspamd_snprintf (lock_path, sizeof (lock_path), "%s.lock", path);
if (access (path, R_OK) == -1 && create_sql != NULL) {
flags |= SQLITE_OPEN_CREATE;
create = TRUE;
}
rspamd_snprintf (lock_path, sizeof (lock_path), "%s.lock", path);
lock_fd = open (lock_path, O_WRONLY|O_CREAT|O_EXCL, 00600);
if (lock_fd == -1 && (errno == EEXIST || errno == EBUSY)) {
msg_debug_pool ("checking %s to wait for db being initialized", lock_path);
if (!rspamd_sqlite3_wait (pool, lock_path)) {
g_set_error (err, rspamd_sqlite3_quark (),
errno, "cannot create sqlite file %s: %s",
path, strerror (errno));
return NULL;
}
/* At this point we have database created */
create = FALSE;
has_lock = FALSE;
}
else {
msg_debug_pool ("locking %s to block other processes", lock_path);
g_assert (rspamd_file_lock (lock_fd, FALSE));
has_lock = TRUE;
}
if ((rc = sqlite3_open_v2 (path, &sqlite,
flags, NULL)) != SQLITE_OK) {
#if SQLITE_VERSION_NUMBER >= 3008000
g_set_error (err, rspamd_sqlite3_quark (),
rc, "cannot open sqlite db %s: %s",
path, sqlite3_errstr (rc));
#else
g_set_error (err, rspamd_sqlite3_quark (),
rc, "cannot open sqlite db %s: %d",
path, rc);
#endif
return NULL;
}
if (create) {
if (sqlite3_exec (sqlite, sqlite_wal, NULL, NULL, NULL) != SQLITE_OK) {
msg_warn_pool ("WAL mode is not supported (%s), locking issues might occur",
sqlite3_errmsg (sqlite));
}
if (sqlite3_exec (sqlite, exclusive_lock_sql, NULL, NULL, NULL) != SQLITE_OK) {
msg_warn_pool ("cannot exclusively lock database to create schema: %s",
sqlite3_errmsg (sqlite));
}
if (sqlite3_exec (sqlite, create_sql, NULL, NULL, NULL) != SQLITE_OK) {
g_set_error (err, rspamd_sqlite3_quark (),
-1, "cannot execute create sql `%s`: %s",
create_sql, sqlite3_errmsg (sqlite));
sqlite3_close (sqlite);
rspamd_file_unlock (lock_fd, FALSE);
unlink (lock_path);
close (lock_fd);
return NULL;
}
sqlite3_close (sqlite);
/* Reopen in normal mode */
msg_debug_pool ("reopening %s in normal mode", path);
flags &= ~SQLITE_OPEN_CREATE;
if ((rc = sqlite3_open_v2 (path, &sqlite,
flags, NULL)) != SQLITE_OK) {
#if SQLITE_VERSION_NUMBER >= 3008000
g_set_error (err, rspamd_sqlite3_quark (),
rc, "cannot open sqlite db after creation %s: %s",
path, sqlite3_errstr (rc));
#else
g_set_error (err, rspamd_sqlite3_quark (),
rc, "cannot open sqlite db after creation %s: %d",
path, rc);
#endif
rspamd_file_unlock (lock_fd, FALSE);
unlink (lock_path);
close (lock_fd);
return NULL;
}
}
if (sqlite3_exec (sqlite, sqlite_wal, NULL, NULL, NULL) != SQLITE_OK) {
msg_warn_pool ("WAL mode is not supported (%s), locking issues might occur",
sqlite3_errmsg (sqlite));
}
if (sqlite3_exec (sqlite, fsync_sql, NULL, NULL, NULL) != SQLITE_OK) {
msg_warn_pool ("cannot set synchronous: %s",
sqlite3_errmsg (sqlite));
}
if ((rc = sqlite3_exec (sqlite, foreign_keys, NULL, NULL, NULL)) !=
SQLITE_OK) {
msg_warn_pool ("cannot enable foreign keys: %s",
sqlite3_errmsg (sqlite));
}
if (sizeof (gpointer) >= 8 &&
(rc = sqlite3_exec (sqlite, enable_mmap, NULL, NULL, NULL)) !=
SQLITE_OK) {
msg_warn_pool ("cannot enable mmap: %s",
sqlite3_errmsg (sqlite));
}
if (has_lock) {
msg_debug_pool ("removing lock from %s", lock_path);
rspamd_file_unlock (lock_fd, FALSE);
unlink (lock_path);
close (lock_fd);
}
return sqlite;
}
struct rspamd_content_type *
rspamd_content_type_parse (const gchar *in,
gsize len, rspamd_mempool_t *pool)
{
struct rspamd_content_type *res = NULL, val;
rspamd_ftok_t srch;
val.lc_data = rspamd_mempool_alloc (pool, len);
memcpy (val.lc_data, in, len);
rspamd_str_lc (val.lc_data, len);
if (rspamd_content_type_parser (val.lc_data, len, &val, pool)) {
res = rspamd_mempool_alloc (pool, sizeof (val));
memcpy (res, &val, sizeof (val));
if (res->attrs) {
rspamd_mempool_add_destructor (pool,
(rspamd_mempool_destruct_t)g_hash_table_unref, res->attrs);
}
/* Now do some hacks to work with broken content types */
if (res->subtype.len == 0) {
res->flags |= RSPAMD_CONTENT_TYPE_BROKEN;
RSPAMD_FTOK_ASSIGN (&srch, "text");
if (rspamd_ftok_cmp (&res->type, &srch) == 0) {
/* Workaround for Content-Type: text */
/* Assume text/plain */
RSPAMD_FTOK_ASSIGN (&srch, "plain");
}
else {
RSPAMD_FTOK_ASSIGN (&srch, "html");
if (rspamd_ftok_cmp (&res->type, &srch) == 0) {
/* Workaround for Content-Type: html */
RSPAMD_FTOK_ASSIGN (&res->type, "text");
RSPAMD_FTOK_ASSIGN (&res->subtype, "html");
}
else {
RSPAMD_FTOK_ASSIGN (&srch, "application");
if (rspamd_ftok_cmp (&res->type, &srch) == 0) {
RSPAMD_FTOK_ASSIGN (&res->subtype, "octet-stream");
}
}
}
}
else {
/* Common mistake done by retards */
RSPAMD_FTOK_ASSIGN (&srch, "alternate");
if (rspamd_ftok_cmp (&res->subtype, &srch) == 0) {
res->flags |= RSPAMD_CONTENT_TYPE_BROKEN;
RSPAMD_FTOK_ASSIGN (&res->subtype, "alternative");
}
}
RSPAMD_FTOK_ASSIGN (&srch, "multipart");
if (rspamd_ftok_cmp (&res->type, &srch) == 0) {
res->flags |= RSPAMD_CONTENT_TYPE_MULTIPART;
}
else {
RSPAMD_FTOK_ASSIGN (&srch, "text");
if (rspamd_ftok_cmp (&res->type, &srch) == 0) {
res->flags |= RSPAMD_CONTENT_TYPE_TEXT;
}
else {
RSPAMD_FTOK_ASSIGN (&srch, "message");
if (rspamd_ftok_cmp (&res->type, &srch) == 0) {
res->flags |= RSPAMD_CONTENT_TYPE_MESSAGE;
}
}
}
}
else {
msg_warn_pool ("cannot parse content type: %*s", (gint)len, val.lc_data);
}
return res;
}
