/** * Async HTTP callback */ static void http_callback (gint fd, short what, void *ud) { struct rspamd_map *map = ud; struct http_map_data *data; gint sock; struct http_callback_data *cbd; rspamd_mempool_t *pool; data = map->map_data; pool = map->pool; if (g_atomic_int_get (map->locked)) { msg_info_pool ( "don't try to reread map as it is locked by other process, will reread it later"); if (data->conn->ud == NULL) { jitter_timeout_event (map, TRUE, TRUE); } else { jitter_timeout_event (map, TRUE, FALSE); } return; } g_atomic_int_inc (map->locked); jitter_timeout_event (map, FALSE, FALSE); /* Connect asynced */ if ((sock = connect_http (map, data, TRUE)) == -1) { g_atomic_int_set (map->locked, 0); return; } else { /* Plan event */ cbd = g_slice_alloc (sizeof (struct http_callback_data)); cbd->ev_base = map->ev_base; cbd->map = map; cbd->data = data; cbd->remain_buf = NULL; cbd->cbdata.state = 0; cbd->cbdata.prev_data = *cbd->map->user_data; cbd->cbdata.cur_data = NULL; cbd->cbdata.map = cbd->map; cbd->tv.tv_sec = HTTP_CONNECT_TIMEOUT; cbd->tv.tv_usec = 0; cbd->fd = sock; data->conn->ud = cbd; msg_debug_pool ("reading map data from %s", data->host); write_http_request (cbd); } }
/** * Common file callback */ static void file_callback (gint fd, short what, void *ud) { struct rspamd_map *map = ud; struct file_map_data *data = map->map_data; struct stat st; rspamd_mempool_t *pool; pool = map->pool; if (!g_atomic_int_compare_and_exchange (map->locked, 0, 1)) { msg_debug_pool ( "don't try to reread map as it is locked by other process, will reread it later"); jitter_timeout_event (map, TRUE, FALSE, FALSE); return; } if (stat (data->filename, &st) != -1 && (st.st_mtime > data->st.st_mtime || data->st.st_mtime == -1)) { /* File was modified since last check */ memcpy (&data->st, &st, sizeof (struct stat)); } else { g_atomic_int_set (map->locked, 0); jitter_timeout_event (map, FALSE, FALSE, FALSE); return; } msg_info_pool ("rereading map file %s", data->filename); if (!read_map_file (map, data)) { jitter_timeout_event (map, FALSE, FALSE, TRUE); } else { jitter_timeout_event (map, FALSE, FALSE, FALSE); } g_atomic_int_set (map->locked, 0); }
static int http_map_finish (struct rspamd_http_connection *conn, struct rspamd_http_message *msg) { struct http_callback_data *cbd = conn->ud; struct rspamd_map *map; rspamd_mempool_t *pool; map = cbd->map; pool = cbd->map->pool; if (msg->code == 200) { if (cbd->remain_buf != NULL) { /* Append \n to avoid issues */ g_string_append_c (cbd->remain_buf, '\n'); map->read_callback (map->pool, cbd->remain_buf->str, cbd->remain_buf->len, &cbd->cbdata); } map->fin_callback (map->pool, &cbd->cbdata); *map->user_data = cbd->cbdata.cur_data; cbd->data->last_checked = msg->date; msg_info_pool ("read map data from %s", cbd->data->host); } else if (msg->code == 304) { msg_debug_pool ("data is not modified for server %s", cbd->data->host); cbd->data->last_checked = msg->date; } else { msg_info_pool ("cannot load map %s from %s: HTTP error %d", map->uri, cbd->data->host, msg->code); } free_http_cbdata (cbd); return 0; }
/** * Async HTTP callback */ static void http_callback (gint fd, short what, void *ud) { struct rspamd_map *map = ud; struct http_map_data *data; struct http_callback_data *cbd; rspamd_mempool_t *pool; gchar tmpbuf[PATH_MAX]; data = map->map_data; pool = map->pool; if (!g_atomic_int_compare_and_exchange (map->locked, 0, 1)) { msg_debug_pool ( "don't try to reread map as it is locked by other process, will reread it later"); jitter_timeout_event (map, TRUE, FALSE, FALSE); return; } /* Plan event */ cbd = g_slice_alloc0 (sizeof (struct http_callback_data)); rspamd_snprintf (tmpbuf, sizeof (tmpbuf), "%s" G_DIR_SEPARATOR_S "rspamd_map%d-XXXXXX", map->cfg->temp_dir, map->id); cbd->out_fd = mkstemp (tmpbuf); if (cbd->out_fd == -1) { g_slice_free1 (sizeof (*cbd), cbd); msg_err_pool ("cannot create tempfile: %s", strerror (errno)); jitter_timeout_event (map, FALSE, FALSE, TRUE); g_atomic_int_set (map->locked, 0); return; } cbd->tmpfile = g_strdup (tmpbuf); cbd->ev_base = map->ev_base; cbd->map = map; cbd->data = data; cbd->fd = -1; cbd->cbdata.state = 0; cbd->cbdata.prev_data = *cbd->map->user_data; cbd->cbdata.cur_data = NULL; cbd->cbdata.map = cbd->map; cbd->stage = map_resolve_host2; double_to_tv (map->cfg->map_timeout, &cbd->tv); REF_INIT_RETAIN (cbd, free_http_cbdata); msg_debug_pool ("reading map data from %s", data->host); /* Send both A and AAAA requests */ if (map->r->r) { if (rdns_make_request_full (map->r->r, rspamd_map_dns_callback, cbd, map->cfg->dns_timeout, map->cfg->dns_retransmits, 1, data->host, RDNS_REQUEST_A)) { REF_RETAIN (cbd); } if (rdns_make_request_full (map->r->r, rspamd_map_dns_callback, cbd, map->cfg->dns_timeout, map->cfg->dns_retransmits, 1, data->host, RDNS_REQUEST_AAAA)) { REF_RETAIN (cbd); } jitter_timeout_event (map, FALSE, FALSE, FALSE); map->dtor = free_http_cbdata_dtor; map->dtor_data = cbd; } else { msg_warn_pool ("cannot load map: DNS resolver is not initialized"); jitter_timeout_event (map, FALSE, FALSE, TRUE); } /* We don't need own ref as it is now refcounted by DNS requests */ REF_RELEASE (cbd); }
static int http_map_finish (struct rspamd_http_connection *conn, struct rspamd_http_message *msg) { struct http_callback_data *cbd = conn->ud; struct rspamd_map *map; rspamd_mempool_t *pool; char fpath[PATH_MAX]; guchar *aux_data, *in = NULL; gsize inlen = 0; struct stat st; map = cbd->map; pool = cbd->map->pool; if (msg->code == 200) { if (cbd->stage == map_load_file) { if (msg->last_modified) { cbd->data->last_checked = msg->last_modified; } else { cbd->data->last_checked = msg->date; } /* Maybe we need to check signature ? */ if (map->is_signed) { close (cbd->out_fd); if (map->trusted_pubkey) { /* No need to load key */ cbd->stage = map_load_signature; cbd->pk = rspamd_pubkey_ref (map->trusted_pubkey); rspamd_snprintf (fpath, sizeof (fpath), "%s.sig", cbd->tmpfile); } else { rspamd_snprintf (fpath, sizeof (fpath), "%s.pub", cbd->tmpfile); cbd->stage = map_load_pubkey; } cbd->out_fd = rspamd_file_xopen (fpath, O_RDWR|O_CREAT, 00644); if (cbd->out_fd == -1) { msg_err_pool ("cannot open pubkey file %s for writing: %s", fpath, strerror (errno)); goto end; } rspamd_http_connection_reset (cbd->conn); write_http_request (cbd); goto end; } else { /* Unsinged version - just open file */ in = rspamd_file_xmap (cbd->tmpfile, PROT_READ, &inlen); if (in == NULL) { msg_err_pool ("cannot read tempfile %s: %s", cbd->tmpfile, strerror (errno)); goto end; } } } else if (cbd->stage == map_load_pubkey) { /* We now can load pubkey */ (void)lseek (cbd->out_fd, 0, SEEK_SET); if (fstat (cbd->out_fd, &st) == -1) { msg_err_pool ("cannot stat pubkey file %s: %s", fpath, strerror (errno)); goto end; } aux_data = mmap (NULL, st.st_size, PROT_READ, MAP_SHARED, cbd->out_fd, 0); close (cbd->out_fd); cbd->out_fd = -1; if (aux_data == MAP_FAILED) { msg_err_pool ("cannot map pubkey file %s: %s", fpath, strerror (errno)); goto end; } cbd->pk = rspamd_pubkey_from_base32 (aux_data, st.st_size, RSPAMD_KEYPAIR_SIGN, RSPAMD_CRYPTOBOX_MODE_25519); munmap (aux_data, st.st_size); if (cbd->pk == NULL) { msg_err_pool ("cannot load pubkey file %s: bad pubkey", fpath); goto end; } rspamd_snprintf (fpath, sizeof (fpath), "%s.sig", cbd->tmpfile); cbd->out_fd = rspamd_file_xopen (fpath, O_RDWR|O_CREAT, 00644); if (cbd->out_fd == -1) { msg_err_pool ("cannot open signature file %s for writing: %s", fpath, strerror (errno)); goto end; } cbd->stage = map_load_signature; rspamd_http_connection_reset (cbd->conn); write_http_request (cbd); goto end; } else if (cbd->stage == map_load_signature) { /* We can now check signature */ close (cbd->out_fd); cbd->out_fd = -1; in = rspamd_file_xmap (cbd->tmpfile, PROT_READ, &inlen); if (in == NULL) { msg_err_pool ("cannot read tempfile %s: %s", cbd->tmpfile, strerror (errno)); goto end; } if (!rspamd_map_check_sig_pk (cbd->tmpfile, map, in, inlen, cbd->pk)) { goto end; } } g_assert (in != NULL); map->read_callback (map->pool, in, inlen, &cbd->cbdata, TRUE); map->fin_callback (map->pool, &cbd->cbdata); *map->user_data = cbd->cbdata.cur_data; msg_info_pool ("read map data from %s", cbd->data->host); } else if (msg->code == 304 && cbd->stage == map_load_file) { msg_debug_pool ("data is not modified for server %s", cbd->data->host); if (msg->last_modified) { cbd->data->last_checked = msg->last_modified; } else { cbd->data->last_checked = msg->date; } } else { msg_info_pool ("cannot load map %s from %s: HTTP error %d", map->uri, cbd->data->host, msg->code); } end: REF_RELEASE (cbd); return 0; }
gchar * rspamd_parse_abstract_list (rspamd_mempool_t * pool, gchar * chunk, gint len, struct map_cb_data *data, insert_func func) { gchar *p, *c, *end, *s; p = chunk; c = p; end = p + len; while (p < end) { switch (data->state) { /* READ_SYMBOL */ case 0: if (*p == '#') { /* Got comment */ if (p > c) { /* Save previous string in lines like: "127.0.0.1 #localhost" */ s = strip_map_elt (pool, c, p - c); if (s) { func (data->cur_data, s, hash_fill); msg_debug_pool ("insert element (before comment): %s", s); } } c = p; data->state = 1; } else if (*p == '\r' || *p == '\n') { /* Got EOL marker, save stored string */ s = strip_map_elt (pool, c, p - c); if (s) { func (data->cur_data, s, hash_fill); msg_debug_pool ("insert element (before EOL): %s", s); } /* Skip EOL symbols */ while ((*p == '\r' || *p == '\n') && p < end) { p++; } if (p == end) { p ++; c = NULL; } else { c = p; } } else { p++; } break; /* SKIP_COMMENT */ case 1: /* Skip comment till end of line */ if (*p == '\r' || *p == '\n') { while ((*p == '\r' || *p == '\n') && p < end) { p++; } if (p == end) { p ++; c = NULL; } else { c = p; } data->state = 0; } else { p++; } break; } } if (c >= end) { c = NULL; } return c; }
/** * FSM for parsing lists */ gchar * abstract_parse_kv_list (rspamd_mempool_t * pool, gchar * chunk, gint len, struct map_cb_data *data, insert_func func) { gchar *c, *p, *key = NULL, *value = NULL, *end; p = chunk; c = p; end = p + len; while (p < end) { switch (data->state) { case 0: /* read key */ /* Check here comments, eol and end of buffer */ if (*p == '#') { if (key != NULL && p - c >= 0) { value = rspamd_mempool_alloc (pool, p - c + 1); memcpy (value, c, p - c); value[p - c] = '\0'; value = g_strstrip (value); func (data->cur_data, key, value); msg_debug_pool ("insert kv pair: %s -> %s", key, value); } data->state = 99; } else if (*p == '\r' || *p == '\n') { if (key != NULL && p - c >= 0) { value = rspamd_mempool_alloc (pool, p - c + 1); memcpy (value, c, p - c); value[p - c] = '\0'; value = g_strstrip (value); func (data->cur_data, key, value); msg_debug_pool ("insert kv pair: %s -> %s", key, value); } else if (key == NULL && p - c > 0) { /* Key only line */ key = rspamd_mempool_alloc (pool, p - c + 1); memcpy (key, c, p - c); key[p - c] = '\0'; value = rspamd_mempool_alloc (pool, 1); *value = '\0'; func (data->cur_data, key, value); msg_debug_pool ("insert kv pair: %s -> %s", key, value); } data->state = 100; key = NULL; } else if (g_ascii_isspace (*p)) { if (p - c > 0) { key = rspamd_mempool_alloc (pool, p - c + 1); memcpy (key, c, p - c); key[p - c] = '\0'; data->state = 2; } else { key = NULL; } } else { p++; } break; case 2: /* Skip spaces before value */ if (!g_ascii_isspace (*p)) { c = p; data->state = 0; } else { p++; } break; case 99: /* SKIP_COMMENT */ /* Skip comment till end of line */ if (*p == '\r' || *p == '\n') { while ((*p == '\r' || *p == '\n') && p < end) { p++; } c = p; key = NULL; data->state = 0; } else { p++; } break; case 100: /* Skip \r\n and whitespaces */ if (*p == '\r' || *p == '\n' || g_ascii_isspace (*p)) { p++; } else { c = p; key = NULL; data->state = 0; } break; } } return c; }
int rspamd_sqlite3_run_prstmt (rspamd_mempool_t *pool, sqlite3 *db, GArray *stmts, gint idx, ...) { gint retcode; va_list ap; sqlite3_stmt *stmt; gint i, rowid, nargs, j; gint64 len; gpointer p; struct rspamd_sqlite3_prstmt *nst; const char *argtypes; if (idx < 0 || idx >= (gint)stmts->len) { return -1; } nst = &g_array_index (stmts, struct rspamd_sqlite3_prstmt, idx); stmt = nst->stmt; g_assert (nst != NULL); msg_debug_pool ("executing `%s`", nst->sql); argtypes = nst->args; sqlite3_clear_bindings (stmt); sqlite3_reset (stmt); va_start (ap, idx); nargs = 1; for (i = 0, rowid = 1; argtypes[i] != '\0'; i ++) { switch (argtypes[i]) { case 'T': for (j = 0; j < nargs; j ++, rowid ++) { sqlite3_bind_text (stmt, rowid, va_arg (ap, const char*), -1, SQLITE_STATIC); } nargs = 1; break; case 'V': case 'B': for (j = 0; j < nargs; j ++, rowid ++) { len = va_arg (ap, gint64); sqlite3_bind_text (stmt, rowid, va_arg (ap, const char*), len, SQLITE_STATIC); } nargs = 1; break; case 'I': for (j = 0; j < nargs; j ++, rowid ++) { sqlite3_bind_int64 (stmt, rowid, va_arg (ap, gint64)); } nargs = 1; break; case 'S': for (j = 0; j < nargs; j ++, rowid ++) { sqlite3_bind_int (stmt, rowid, va_arg (ap, gint)); } nargs = 1; break; case '*': nargs = va_arg (ap, gint); break; } } va_end (ap); retcode = sqlite3_step (stmt); if (retcode == nst->result) { argtypes = nst->ret; for (i = 0; argtypes != NULL && argtypes[i] != '\0'; i ++) { switch (argtypes[i]) { case 'T': *va_arg (ap, char**) = g_strdup (sqlite3_column_text (stmt, i)); break; case 'I': *va_arg (ap, gint64*) = sqlite3_column_int64 (stmt, i); break; case 'S': *va_arg (ap, int*) = sqlite3_column_int (stmt, i); break; case 'L': *va_arg (ap, gint64*) = sqlite3_last_insert_rowid (db); break; case 'B': len = sqlite3_column_bytes (stmt, i); g_assert (len >= 0); p = g_malloc (len); memcpy (p, sqlite3_column_blob (stmt, i), len); *va_arg (ap, gint64*) = len; *va_arg (ap, gpointer*) = p; break; } } if (!(nst->flags & RSPAMD_SQLITE3_STMT_MULTIPLE)) { sqlite3_clear_bindings (stmt); sqlite3_reset (stmt); } return SQLITE_OK; } else if (retcode != SQLITE_DONE && retcode != SQLITE_OK && retcode != SQLITE_ROW) { msg_warn_pool ("failed to execute query %s: %d, %s", nst->sql, retcode, sqlite3_errmsg (db)); } if (!(nst->flags & RSPAMD_SQLITE3_STMT_MULTIPLE)) { sqlite3_clear_bindings (stmt); sqlite3_reset (stmt); } return retcode; }
static gboolean rspamd_sqlite3_wait (rspamd_mempool_t *pool, const gchar *lock) { gint fd; struct timespec sleep_ts = { .tv_sec = 0, .tv_nsec = 1000000 }; fd = open (lock, O_RDONLY); if (fd == -1) { msg_err_pool ("cannot open lock file %s: %s", lock, strerror (errno)); return FALSE; } while (!rspamd_file_lock (fd, TRUE)) { if (nanosleep (&sleep_ts, NULL) == -1 && errno != EINTR) { close (fd); msg_err_pool ("cannot sleep open lock file %s: %s", lock, strerror (errno)); return FALSE; } } rspamd_file_unlock (fd, FALSE); close (fd); return TRUE; } sqlite3 * rspamd_sqlite3_open_or_create (rspamd_mempool_t *pool, const gchar *path, const gchar *create_sql, GError **err) { sqlite3 *sqlite; gint rc, flags, lock_fd; gchar lock_path[PATH_MAX], dbdir[PATH_MAX], *pdir; static const char sqlite_wal[] = "PRAGMA journal_mode=\"wal\";", exclusive_lock_sql[] = "PRAGMA locking_mode=\"exclusive\";", fsync_sql[] = "PRAGMA synchronous=1;", foreign_keys[] = "PRAGMA foreign_keys=\"ON\";", enable_mmap[] = "PRAGMA mmap_size=268435456;"; gboolean create = FALSE, has_lock = FALSE; flags = SQLITE_OPEN_READWRITE; #ifdef SQLITE_OPEN_SHAREDCACHE flags |= SQLITE_OPEN_SHAREDCACHE; #endif #ifdef SQLITE_OPEN_WAL flags |= SQLITE_OPEN_WAL; #endif rspamd_strlcpy (dbdir, path, sizeof (dbdir)); pdir = dirname (dbdir); if (access (pdir, W_OK) == -1) { g_set_error (err, rspamd_sqlite3_quark (), errno, "cannot open sqlite directory %s: %s", pdir, strerror (errno)); return NULL; } rspamd_snprintf (lock_path, sizeof (lock_path), "%s.lock", path); if (access (path, R_OK) == -1 && create_sql != NULL) { flags |= SQLITE_OPEN_CREATE; create = TRUE; } rspamd_snprintf (lock_path, sizeof (lock_path), "%s.lock", path); lock_fd = open (lock_path, O_WRONLY|O_CREAT|O_EXCL, 00600); if (lock_fd == -1 && (errno == EEXIST || errno == EBUSY)) { msg_debug_pool ("checking %s to wait for db being initialized", lock_path); if (!rspamd_sqlite3_wait (pool, lock_path)) { g_set_error (err, rspamd_sqlite3_quark (), errno, "cannot create sqlite file %s: %s", path, strerror (errno)); return NULL; } /* At this point we have database created */ create = FALSE; has_lock = FALSE; } else { msg_debug_pool ("locking %s to block other processes", lock_path); g_assert (rspamd_file_lock (lock_fd, FALSE)); has_lock = TRUE; } if ((rc = sqlite3_open_v2 (path, &sqlite, flags, NULL)) != SQLITE_OK) { #if SQLITE_VERSION_NUMBER >= 3008000 g_set_error (err, rspamd_sqlite3_quark (), rc, "cannot open sqlite db %s: %s", path, sqlite3_errstr (rc)); #else g_set_error (err, rspamd_sqlite3_quark (), rc, "cannot open sqlite db %s: %d", path, rc); #endif return NULL; } if (create) { if (sqlite3_exec (sqlite, sqlite_wal, NULL, NULL, NULL) != SQLITE_OK) { msg_warn_pool ("WAL mode is not supported (%s), locking issues might occur", sqlite3_errmsg (sqlite)); } if (sqlite3_exec (sqlite, exclusive_lock_sql, NULL, NULL, NULL) != SQLITE_OK) { msg_warn_pool ("cannot exclusively lock database to create schema: %s", sqlite3_errmsg (sqlite)); } if (sqlite3_exec (sqlite, create_sql, NULL, NULL, NULL) != SQLITE_OK) { g_set_error (err, rspamd_sqlite3_quark (), -1, "cannot execute create sql `%s`: %s", create_sql, sqlite3_errmsg (sqlite)); sqlite3_close (sqlite); rspamd_file_unlock (lock_fd, FALSE); unlink (lock_path); close (lock_fd); return NULL; } sqlite3_close (sqlite); /* Reopen in normal mode */ msg_debug_pool ("reopening %s in normal mode", path); flags &= ~SQLITE_OPEN_CREATE; if ((rc = sqlite3_open_v2 (path, &sqlite, flags, NULL)) != SQLITE_OK) { #if SQLITE_VERSION_NUMBER >= 3008000 g_set_error (err, rspamd_sqlite3_quark (), rc, "cannot open sqlite db after creation %s: %s", path, sqlite3_errstr (rc)); #else g_set_error (err, rspamd_sqlite3_quark (), rc, "cannot open sqlite db after creation %s: %d", path, rc); #endif rspamd_file_unlock (lock_fd, FALSE); unlink (lock_path); close (lock_fd); return NULL; } } if (sqlite3_exec (sqlite, sqlite_wal, NULL, NULL, NULL) != SQLITE_OK) { msg_warn_pool ("WAL mode is not supported (%s), locking issues might occur", sqlite3_errmsg (sqlite)); } if (sqlite3_exec (sqlite, fsync_sql, NULL, NULL, NULL) != SQLITE_OK) { msg_warn_pool ("cannot set synchronous: %s", sqlite3_errmsg (sqlite)); } if ((rc = sqlite3_exec (sqlite, foreign_keys, NULL, NULL, NULL)) != SQLITE_OK) { msg_warn_pool ("cannot enable foreign keys: %s", sqlite3_errmsg (sqlite)); } if (sizeof (gpointer) >= 8 && (rc = sqlite3_exec (sqlite, enable_mmap, NULL, NULL, NULL)) != SQLITE_OK) { msg_warn_pool ("cannot enable mmap: %s", sqlite3_errmsg (sqlite)); } if (has_lock) { msg_debug_pool ("removing lock from %s", lock_path); rspamd_file_unlock (lock_fd, FALSE); unlink (lock_path); close (lock_fd); } return sqlite; }