static gboolean
rspamd_symbols_cache_save_items (struct symbols_cache *cache, const gchar *name)
{
struct rspamd_symbols_cache_header hdr;
ucl_object_t *top, *elt;
GHashTableIter it;
struct cache_item *item;
struct ucl_emitter_functions *efunc;
gpointer k, v;
gint fd;
bool ret;
fd = open (name, O_CREAT | O_TRUNC | O_WRONLY, 00644);
if (fd == -1) {
msg_info ("cannot open file %s, error %d, %s", name,
errno, strerror (errno));
return FALSE;
}
memset (&hdr, 0, sizeof (hdr));
memcpy (hdr.magic, rspamd_symbols_cache_magic,
sizeof (rspamd_symbols_cache_magic));
if (write (fd, &hdr, sizeof (hdr)) == -1) {
msg_info ("cannot write to file %s, error %d, %s", name,
errno, strerror (errno));
close (fd);
return FALSE;
}
top = ucl_object_typed_new (UCL_OBJECT);
g_hash_table_iter_init (&it, cache->items_by_symbol);
while (g_hash_table_iter_next (&it, &k, &v)) {
item = v;
elt = ucl_object_typed_new (UCL_OBJECT);
ucl_object_insert_key (elt, ucl_object_fromdouble (item->weight),
"weight", 0, false);
ucl_object_insert_key (elt, ucl_object_fromdouble (item->avg_time),
"time", 0, false);
ucl_object_insert_key (elt, ucl_object_fromdouble (item->avg_counter),
"count", 0, false);
ucl_object_insert_key (elt, ucl_object_fromint (item->frequency),
"frequency", 0, false);
ucl_object_insert_key (top, elt, k, 0, false);
}
efunc = ucl_object_emit_fd_funcs (fd);
ret = ucl_object_emit_full (top, UCL_EMIT_JSON_COMPACT, efunc);
ucl_object_emit_funcs_free (efunc);
close (fd);
return ret;
}
void
pkg_reset(struct pkg *pkg, pkg_t type)
{
int i;
if (pkg == NULL)
return;
ucl_object_unref(pkg->fields);
pkg->fields = ucl_object_typed_new(UCL_OBJECT);
pkg->flags &= ~PKG_LOAD_CATEGORIES;
pkg->flags &= ~PKG_LOAD_LICENSES;
pkg->flags &= ~PKG_LOAD_ANNOTATIONS;
for (i = 0; i < PKG_NUM_SCRIPTS; i++)
sbuf_reset(pkg->scripts[i]);
pkg_list_free(pkg, PKG_DEPS);
pkg_list_free(pkg, PKG_RDEPS);
pkg_list_free(pkg, PKG_FILES);
pkg_list_free(pkg, PKG_DIRS);
pkg_list_free(pkg, PKG_OPTIONS);
pkg_list_free(pkg, PKG_USERS);
pkg_list_free(pkg, PKG_GROUPS);
pkg_list_free(pkg, PKG_SHLIBS_REQUIRED);
pkg_list_free(pkg, PKG_SHLIBS_PROVIDED);
pkg->type = type;
}
ucl_object_t *
rspamd_log_errorbuf_export (const rspamd_logger_t *logger)
{
struct rspamd_logger_error_elt *cpy, *cur;
ucl_object_t *top = ucl_object_typed_new (UCL_ARRAY);
guint i;
if (logger->errlog == NULL) {
return top;
}
cpy = g_malloc0_n (logger->errlog->max_elts,
sizeof (*cpy) + logger->errlog->elt_len);
memcpy (cpy, logger->errlog->elts, logger->errlog->max_elts *
(sizeof (*cpy) + logger->errlog->elt_len));
for (i = 0; i < logger->errlog->max_elts; i ++) {
cur = (struct rspamd_logger_error_elt *)((guchar *)cpy +
i * ((sizeof (*cpy) + logger->errlog->elt_len)));
if (cur->completed) {
ucl_object_t *obj = ucl_object_typed_new (UCL_OBJECT);
ucl_object_insert_key (obj, ucl_object_fromdouble (cur->ts),
"ts", 0, false);
ucl_object_insert_key (obj, ucl_object_fromint (cur->pid),
"pid", 0, false);
ucl_object_insert_key (obj,
ucl_object_fromstring (g_quark_to_string (cur->ptype)),
"type", 0, false);
ucl_object_insert_key (obj, ucl_object_fromstring (cur->id),
"id", 0, false);
ucl_object_insert_key (obj, ucl_object_fromstring (cur->module),
"module", 0, false);
ucl_object_insert_key (obj, ucl_object_fromstring (cur->message),
"message", 0, false);
ucl_array_append (top, obj);
}
}
ucl_object_array_sort (top, rspamd_log_errlog_cmp);
g_free (cpy);
return top;
}
static ucl_object_t*
new_dynamic_metric (const gchar *metric_name, ucl_object_t *top)
{
ucl_object_t *metric;
metric = ucl_object_typed_new (UCL_OBJECT);
ucl_object_insert_key (metric, ucl_object_fromstring (metric_name),
"metric", sizeof ("metric") - 1, true);
ucl_object_insert_key (metric, ucl_object_typed_new (UCL_ARRAY),
"actions", sizeof ("actions") - 1, false);
ucl_object_insert_key (metric, ucl_object_typed_new (UCL_ARRAY),
"symbols", sizeof ("symbols") - 1, false);
ucl_array_append (top, metric);
return metric;
}
int
pkg_new(struct pkg **pkg, pkg_t type)
{
if ((*pkg = calloc(1, sizeof(struct pkg))) == NULL) {
pkg_emit_errno("calloc", "pkg");
return EPKG_FATAL;
}
(*pkg)->fields = ucl_object_typed_new(UCL_OBJECT);
(*pkg)->type = type;
return (EPKG_OK);
}
static ucl_object_t *
rspamadm_confighelp_search_word (const ucl_object_t *obj, const gchar *str)
{
gsize len = strlen (str);
GString *path = g_string_new ("");
ucl_object_t *res;
res = ucl_object_typed_new (UCL_OBJECT);
rspamadm_confighelp_search_word_step (obj, res, str, len, path);
return res;
}
static ucl_object_t *
new_dynamic_elt (ucl_object_t *arr, const gchar *name, gdouble value)
{
ucl_object_t *n;
n = ucl_object_typed_new (UCL_OBJECT);
ucl_object_insert_key (n, ucl_object_fromstring (name), "name",
sizeof ("name") - 1, false);
ucl_object_insert_key (n, ucl_object_fromdouble (value), "value",
sizeof ("value") - 1, false);
ucl_array_append (arr, n);
return n;
}
ucl_object_t *
rspamd_symbols_cache_counters (struct symbols_cache * cache)
{
ucl_object_t *top;
struct counters_cbdata cbd;
g_assert (cache != NULL);
top = ucl_object_typed_new (UCL_ARRAY);
cbd.top = top;
cbd.cache = cache;
g_ptr_array_foreach (cache->items_by_order,
rspamd_symbols_cache_counters_cb, &cbd);
return top;
}
static void
rspamd_redis_async_stat_cb (struct rspamd_stat_async_elt *elt, gpointer d)
{
struct redis_stat_ctx *ctx;
struct rspamd_redis_stat_elt *redis_elt = elt->ud;
struct rspamd_redis_stat_cbdata *cbdata;
rspamd_inet_addr_t *addr;
g_assert (redis_elt != NULL);
ctx = redis_elt->ctx;
if (redis_elt->cbdata) {
/* We have some other process pending */
rspamd_redis_async_cbdata_cleanup (redis_elt->cbdata);
}
/* Disable further events unless needed */
elt->enabled = FALSE;
cbdata = g_slice_alloc0 (sizeof (*cbdata));
cbdata->selected = rspamd_upstream_get (ctx->read_servers,
RSPAMD_UPSTREAM_ROUND_ROBIN,
NULL,
0);
g_assert (cbdata->selected != NULL);
addr = rspamd_upstream_addr (cbdata->selected);
g_assert (addr != NULL);
cbdata->redis = redisAsyncConnect (rspamd_inet_address_to_string (addr),
rspamd_inet_address_get_port (addr));
g_assert (cbdata->redis != NULL);
redisLibeventAttach (cbdata->redis, redis_elt->ev_base);
cbdata->inflight = 1;
cbdata->cur = ucl_object_typed_new (UCL_OBJECT);
cbdata->elt = redis_elt;
cbdata->cur_keys = g_ptr_array_new ();
redis_elt->cbdata = cbdata;
/* XXX: deal with timeouts maybe */
/* Get keys in redis that match our symbol */
rspamd_redis_maybe_auth (ctx, cbdata->redis);
redisAsyncCommand (cbdata->redis, rspamd_redis_stat_keys, cbdata,
"SMEMBERS %s_keys",
ctx->stcf->symbol);
}
static void
rspamd_symbols_cache_counters_cb (gpointer v, gpointer ud)
{
struct counters_cbdata *cbd = ud;
ucl_object_t *obj, *top;
struct cache_item *item = v, *parent;
top = cbd->top;
if (item->type != SYMBOL_TYPE_CALLBACK) {
obj = ucl_object_typed_new (UCL_OBJECT);
ucl_object_insert_key (obj, ucl_object_fromstring (item->symbol),
"symbol", 0, false);
if (item->type == SYMBOL_TYPE_VIRTUAL && item->parent != -1) {
g_assert (item->parent < (gint)cbd->cache->items_by_id->len);
parent = g_ptr_array_index (cbd->cache->items_by_id,
item->parent);
ucl_object_insert_key (obj, ucl_object_fromdouble (item->weight),
"weight", 0, false);
ucl_object_insert_key (obj, ucl_object_fromint (item->frequency),
"frequency", 0, false);
ucl_object_insert_key (obj, ucl_object_fromdouble (parent->avg_time),
"time", 0, false);
}
else {
ucl_object_insert_key (obj, ucl_object_fromdouble (item->weight),
"weight", 0, false);
ucl_object_insert_key (obj, ucl_object_fromint (item->frequency),
"frequency", 0, false);
ucl_object_insert_key (obj, ucl_object_fromdouble (item->avg_time),
"time", 0, false);
}
ucl_array_append (top, obj);
}
}
bool
ucl_parse_csexp (struct ucl_parser *parser)
{
const unsigned char *p, *end;
ucl_object_t *obj;
struct ucl_stack *st;
uint64_t len = 0, mult = 1;
enum {
start_parse,
read_obrace,
read_length,
read_value,
read_ebrace,
parse_err
} state = start_parse;
assert (parser != NULL);
assert (parser->chunks != NULL);
assert (parser->chunks->begin != NULL);
assert (parser->chunks->remain != 0);
p = parser->chunks->begin;
end = p + parser->chunks->remain;
while (p < end) {
switch (state) {
case start_parse:
/* At this point we expect open brace */
if (*p == '(') {
state = read_obrace;
}
else {
ucl_create_err (&parser->err, "bad starting character for "
"sexp block: %x", (int)*p);
state = parse_err;
}
break;
case read_obrace:
st = calloc (1, sizeof (*st));
if (st == NULL) {
ucl_create_err (&parser->err, "no memory");
state = parse_err;
continue;
}
st->obj = ucl_object_typed_new (UCL_ARRAY);
if (st->obj == NULL) {
ucl_create_err (&parser->err, "no memory");
state = parse_err;
continue;
}
if (parser->stack == NULL) {
/* We have no stack */
parser->stack = st;
if (parser->top_obj == NULL) {
parser->top_obj = st->obj;
}
}
else {
/* Prepend new element to the stack */
LL_PREPEND (parser->stack, st);
}
p ++;
NEXT_STATE;
break;
case read_length:
if (*p == ':') {
if (len == 0) {
ucl_create_err (&parser->err, "zero length element");
state = parse_err;
continue;
}
state = read_value;
}
else if (*p >= '0' && *p <= '9') {
len += (*p - '0') * mult;
mult *= 10;
if (len > UINT32_MAX) {
ucl_create_err (&parser->err, "too big length of an "
"element");
state = parse_err;
continue;
}
}
else {
ucl_create_err (&parser->err, "bad length character: %x",
(int)*p);
state = parse_err;
continue;
}
p ++;
break;
case read_value:
if ((uint64_t)(end - p) > len || len == 0) {
ucl_create_err (&parser->err, "invalid length: %llu, %ld "
"remain", (long long unsigned)len, (long)(end - p));
state = parse_err;
continue;
}
obj = ucl_object_typed_new (UCL_STRING);
obj->value.sv = (const char*)p;
obj->len = len;
obj->flags |= UCL_OBJECT_BINARY;
if (!(parser->flags & UCL_PARSER_ZEROCOPY)) {
ucl_copy_value_trash (obj);
}
ucl_array_append (parser->stack->obj, obj);
p += len;
NEXT_STATE;
break;
case read_ebrace:
if (parser->stack == NULL) {
/* We have an extra end brace */
ucl_create_err (&parser->err, "invalid length: %llu, %ld "
"remain", (long long unsigned)len, (long)(end - p));
state = parse_err;
continue;
}
/* Pop the container */
st = parser->stack;
parser->stack = st->next;
if (parser->stack->obj->type == UCL_ARRAY) {
ucl_array_append (parser->stack->obj, st->obj);
}
else {
ucl_create_err (&parser->err, "bad container object, array "
"expected");
state = parse_err;
continue;
}
free (st);
p++;
NEXT_STATE;
break;
case parse_err:
default:
return false;
}
}
if (state != read_ebrace) {
ucl_create_err (&parser->err, "invalid finishing state: %d", state);
return false;
}
return true;
}
static void
rspamadm_statconvert (gint argc, gchar **argv)
{
GOptionContext *context;
GError *error = NULL;
lua_State *L;
ucl_object_t *obj;
context = g_option_context_new (
"statconvert - converts statistics from sqlite3 to redis");
g_option_context_set_summary (context,
"Summary:\n Rspamd administration utility version "
RVERSION
"\n Release id: "
RID);
g_option_context_add_main_entries (context, entries, NULL);
g_option_context_set_ignore_unknown_options (context, TRUE);
if (!g_option_context_parse (context, &argc, &argv, &error)) {
rspamd_fprintf (stderr, "option parsing failed: %s\n", error->message);
g_error_free (error);
exit (1);
}
if (!source_db) {
rspamd_fprintf (stderr, "source db is missing\n");
exit (1);
}
if (!redis_host) {
rspamd_fprintf (stderr, "redis host is missing\n");
exit (1);
}
if (!symbol) {
rspamd_fprintf (stderr, "symbol is missing\n");
exit (1);
}
L = rspamd_lua_init ();
obj = ucl_object_typed_new (UCL_OBJECT);
ucl_object_insert_key (obj, ucl_object_fromstring (source_db),
"source_db", 0, false);
ucl_object_insert_key (obj, ucl_object_fromstring (redis_host),
"redis_host", 0, false);
ucl_object_insert_key (obj, ucl_object_fromstring (symbol),
"symbol", 0, false);
if (cache_db != NULL) {
ucl_object_insert_key (obj, ucl_object_fromstring (cache_db),
"cache_db", 0, false);
}
rspamadm_execute_lua_ucl_subr (L,
argc,
argv,
obj,
rspamadm_script_stat_convert);
lua_close (L);
ucl_object_unref (obj);
}
int
main (int argc, char **argv)
{
ucl_object_t *obj, *cur, *ar, *ref;
const ucl_object_t *found;
FILE *out;
unsigned char *emitted;
const char *fname_out = NULL;
int ret = 0;
switch (argc) {
case 2:
fname_out = argv[1];
break;
}
if (fname_out != NULL) {
out = fopen (fname_out, "w");
if (out == NULL) {
exit (-errno);
}
}
else {
out = stdout;
}
obj = ucl_object_typed_new (UCL_OBJECT);
/* Create some strings */
cur = ucl_object_fromstring_common (" test string ", 0, UCL_STRING_TRIM);
ucl_object_insert_key (obj, cur, "key1", 0, false);
cur = ucl_object_fromstring_common (" test \nstring\n ", 0, UCL_STRING_TRIM | UCL_STRING_ESCAPE);
ucl_object_insert_key (obj, cur, "key2", 0, false);
cur = ucl_object_fromstring_common (" test string \n", 0, 0);
ucl_object_insert_key (obj, cur, "key3", 0, false);
/* Array of numbers */
ar = ucl_object_typed_new (UCL_ARRAY);
cur = ucl_object_fromint (10);
ucl_array_append (ar, cur);
cur = ucl_object_fromdouble (10.1);
ucl_array_append (ar, cur);
cur = ucl_object_fromdouble (9.999);
ucl_array_prepend (ar, cur);
/* Removing from an array */
cur = ucl_object_fromdouble (1.0);
ucl_array_append (ar, cur);
cur = ucl_array_delete (ar, cur);
assert (ucl_object_todouble (cur) == 1.0);
ucl_object_unref (cur);
cur = ucl_object_fromdouble (2.0);
ucl_array_append (ar, cur);
cur = ucl_array_pop_last (ar);
assert (ucl_object_todouble (cur) == 2.0);
ucl_object_unref (cur);
cur = ucl_object_fromdouble (3.0);
ucl_array_prepend (ar, cur);
cur = ucl_array_pop_first (ar);
assert (ucl_object_todouble (cur) == 3.0);
ucl_object_unref (cur);
ucl_object_insert_key (obj, ar, "key4", 0, false);
cur = ucl_object_frombool (true);
/* Ref object to test refcounts */
ref = ucl_object_ref (cur);
ucl_object_insert_key (obj, cur, "key4", 0, false);
/* Empty strings */
cur = ucl_object_fromstring_common (" ", 0, UCL_STRING_TRIM);
ucl_object_insert_key (obj, cur, "key5", 0, false);
cur = ucl_object_fromstring_common ("", 0, UCL_STRING_ESCAPE);
ucl_object_insert_key (obj, cur, "key6", 0, false);
cur = ucl_object_fromstring_common (" \n", 0, UCL_STRING_ESCAPE);
ucl_object_insert_key (obj, cur, "key7", 0, false);
/* Numbers and booleans */
cur = ucl_object_fromstring_common ("1mb", 0, UCL_STRING_ESCAPE | UCL_STRING_PARSE);
ucl_object_insert_key (obj, cur, "key8", 0, false);
cur = ucl_object_fromstring_common ("3.14", 0, UCL_STRING_PARSE);
ucl_object_insert_key (obj, cur, "key9", 0, false);
cur = ucl_object_fromstring_common ("true", 0, UCL_STRING_PARSE);
ucl_object_insert_key (obj, cur, "key10", 0, false);
cur = ucl_object_fromstring_common (" off ", 0, UCL_STRING_PARSE | UCL_STRING_TRIM);
ucl_object_insert_key (obj, cur, "key11", 0, false);
cur = ucl_object_fromstring_common ("*****@*****.**", 0, UCL_STRING_PARSE_INT);
ucl_object_insert_key (obj, cur, "key12", 0, false);
cur = ucl_object_fromstring_common ("#test", 0, UCL_STRING_PARSE_INT);
ucl_object_insert_key (obj, cur, "key13", 0, false);
cur = ucl_object_frombool (true);
ucl_object_insert_key (obj, cur, "k=3", 0, false);
/* Try to find using path */
/* Should exist */
found = ucl_lookup_path (obj, "key4.1");
assert (found != NULL && ucl_object_toint (found) == 10);
/* . should be ignored */
found = ucl_lookup_path (obj, ".key4.1");
assert (found != NULL && ucl_object_toint (found) == 10);
/* moar dots... */
found = ucl_lookup_path (obj, ".key4........1...");
assert (found != NULL && ucl_object_toint (found) == 10);
/* No such index */
found = ucl_lookup_path (obj, ".key4.3");
assert (found == NULL);
/* No such key */
found = ucl_lookup_path (obj, "key9..key1");
assert (found == NULL);
emitted = ucl_object_emit (obj, UCL_EMIT_CONFIG);
fprintf (out, "%s\n", emitted);
ucl_object_unref (obj);
if (emitted != NULL) {
free (emitted);
}
fclose (out);
/* Ref should still be accessible */
ref->value.iv = 100500;
ucl_object_unref (ref);
return ret;
}
static void
rspamadm_confighelp (gint argc, gchar **argv, const struct rspamadm_command *cmd)
{
struct rspamd_config *cfg;
ucl_object_t *doc_obj;
const ucl_object_t *elt;
GOptionContext *context;
GError *error = NULL;
module_t *mod, **pmod;
worker_t **pworker;
struct module_ctx *mod_ctx;
gint i, ret = 0, processed_args = 0;
context = g_option_context_new (
"confighelp - displays help for the configuration options");
g_option_context_set_summary (context,
"Summary:\n Rspamd administration utility version "
RVERSION
"\n Release id: "
RID);
g_option_context_add_main_entries (context, entries, NULL);
g_option_context_set_ignore_unknown_options (context, TRUE);
if (!g_option_context_parse (context, &argc, &argv, &error)) {
rspamd_fprintf (stderr, "option parsing failed: %s\n", error->message);
g_error_free (error);
exit (1);
}
pworker = &workers[0];
while (*pworker) {
/* Init string quarks */
(void) g_quark_from_static_string ((*pworker)->name);
pworker++;
}
cfg = rspamd_config_new (RSPAMD_CONFIG_INIT_SKIP_LUA);
cfg->lua_state = rspamd_main->cfg->lua_state;
cfg->compiled_modules = modules;
cfg->compiled_workers = workers;
rspamd_rcl_config_init (cfg, NULL);
lua_pushboolean (cfg->lua_state, true);
lua_setglobal (cfg->lua_state, "confighelp");
rspamd_rcl_add_lua_plugins_path (cfg, plugins_path, FALSE, NULL, NULL);
/* Init modules to get documentation strings */
i = 0;
for (pmod = cfg->compiled_modules; pmod != NULL && *pmod != NULL; pmod++) {
mod = *pmod;
mod_ctx = g_malloc0 (sizeof (struct module_ctx));
if (mod->module_init_func (cfg, &mod_ctx) == 0) {
g_ptr_array_add (cfg->c_modules, mod_ctx);
mod_ctx->mod = mod;
mod->ctx_offset = i++;
mod_ctx->mod = mod;
}
}
/* Also init all workers */
for (pworker = cfg->compiled_workers; *pworker != NULL; pworker ++) {
(*pworker)->worker_init_func (cfg);
}
/* Init lua modules */
rspamd_lua_set_path (cfg->lua_state, cfg->rcl_obj, ucl_vars);
rspamd_init_lua_filters (cfg, TRUE);
if (argc > 1) {
for (i = 1; i < argc; i ++) {
if (argv[i][0] != '-') {
if (keyword) {
doc_obj = rspamadm_confighelp_search_word (cfg->doc_strings,
argv[i]);
}
else {
doc_obj = ucl_object_typed_new (UCL_OBJECT);
elt = ucl_object_lookup_path (cfg->doc_strings, argv[i]);
if (elt) {
ucl_object_insert_key (doc_obj, ucl_object_ref (elt),
argv[i], 0, false);
}
}
if (doc_obj != NULL) {
rspamadm_confighelp_show (cfg, argc, argv, argv[i], doc_obj);
ucl_object_unref (doc_obj);
}
else {
rspamd_fprintf (stderr,
"Cannot find help for %s\n",
argv[i]);
ret = EXIT_FAILURE;
}
processed_args ++;
}
}
}
if (processed_args == 0) {
/* Show all documentation strings */
rspamadm_confighelp_show (cfg, argc, argv, NULL, cfg->doc_strings);
}
rspamd_config_free (cfg);
exit (ret);
}
static ucl_object_t *
_iterate_python (PyObject *obj)
{
if (obj == Py_None) {
return ucl_object_new();
}
else if (PyBool_Check (obj)) {
return ucl_object_frombool (obj == Py_True);
}
#if PY_MAJOR_VERSION < 3
else if (PyInt_Check (obj)) {
return ucl_object_fromint (PyInt_AsLong (obj));
}
#endif
else if (PyLong_Check (obj)) {
return ucl_object_fromint (PyLong_AsLong (obj));
}
else if (PyFloat_Check (obj)) {
return ucl_object_fromdouble (PyFloat_AsDouble (obj));
}
else if (PyUnicode_Check (obj)) {
ucl_object_t *ucl_str;
PyObject *str = PyUnicode_AsASCIIString(obj);
ucl_str = ucl_object_fromstring (PyBytes_AsString (str));
Py_DECREF(str);
return ucl_str;
}
#if PY_MAJOR_VERSION < 3
else if (PyString_Check (obj)) {
return ucl_object_fromstring (PyString_AsString (obj));
}
#endif
else if (PyDict_Check(obj)) {
PyObject *key, *value;
Py_ssize_t pos = 0;
ucl_object_t *top, *elm;
char *keystr = NULL;
top = ucl_object_typed_new (UCL_OBJECT);
while (PyDict_Next(obj, &pos, &key, &value)) {
elm = _iterate_python(value);
if (PyUnicode_Check(key)) {
PyObject *keyascii = PyUnicode_AsASCIIString(key);
keystr = PyBytes_AsString(keyascii);
Py_DECREF(keyascii);
}
#if PY_MAJOR_VERSION < 3
else if (PyString_Check(key)) {
keystr = PyString_AsString(key);
}
#endif
else {
PyErr_SetString(PyExc_TypeError, "Unknown key type");
return NULL;
}
ucl_object_insert_key (top, elm, keystr, 0, true);
}
return top;
}
else if (PySequence_Check(obj)) {
PyObject *value;
Py_ssize_t len, pos;
ucl_object_t *top, *elm;
len = PySequence_Length(obj);
top = ucl_object_typed_new (UCL_ARRAY);
for (pos = 0; pos < len; pos++) {
value = PySequence_GetItem(obj, pos);
elm = _iterate_python(value);
ucl_array_append(top, elm);
}
return top;
}
else {
PyErr_SetString(PyExc_TypeError, "Unhandled object type");
return NULL;
}
return NULL;
}
static void
rspamadm_rescore (gint argc, gchar **argv) {
GOptionContext *context;
GError *error = NULL;
lua_State *L;
ucl_object_t *obj;
context = g_option_context_new (
"rescore - Estimate optimal symbol weights from log files");
g_option_context_set_summary (context,
"Summary:\n Rspamd administration utility version "
RVERSION
"\n Release id: "
RID);
g_option_context_add_main_entries (context, entries, NULL);
g_option_context_set_ignore_unknown_options (context, TRUE);
if (!g_option_context_parse (context, &argc, &argv, &error)) {
rspamd_fprintf (stderr, "option parsing failed: %s\n", error->message);
g_error_free (error);
exit (EXIT_FAILURE);
}
if (!HAS_TORCH) {
rspamd_fprintf (stderr, "Torch is not enabled. "
"Use -DENABLE_TORCH=ON option while running cmake.\n");
exit (EXIT_FAILURE);
}
if (logdir == NULL) {
rspamd_fprintf (stderr, "Please specify log directory.\n");
exit (EXIT_FAILURE);
}
L = rspamd_lua_init ();
rspamd_lua_set_path (L, NULL, NULL);
obj = ucl_object_typed_new (UCL_OBJECT);
ucl_object_insert_key (obj, ucl_object_fromstring (logdir),
"logdir", 0, false);
ucl_object_insert_key (obj, ucl_object_fromstring (output),
"output", 0, false);
ucl_object_insert_key (obj, ucl_object_fromdouble (threshold),
"threshold", 0, false);
ucl_object_insert_key (obj, ucl_object_fromint (iters),
"iters", 0, false);
ucl_object_insert_key (obj, ucl_object_frombool (score_diff),
"diff", 0, false);
rspamadm_execute_lua_ucl_subr (L,
argc,
argv,
obj,
"rescore");
lua_close (L);
ucl_object_unref (obj);
}
static void
rspamadm_statconvert (gint argc, gchar **argv, const struct rspamadm_command *cmd)
{
GOptionContext *context;
GError *error = NULL;
ucl_object_t *obj;
context = g_option_context_new (
"statconvert - converts statistics from sqlite3 to redis");
g_option_context_set_summary (context,
"Summary:\n Rspamd administration utility version "
RVERSION
"\n Release id: "
RID);
g_option_context_add_main_entries (context, entries, NULL);
g_option_context_set_ignore_unknown_options (context, TRUE);
if (!g_option_context_parse (context, &argc, &argv, &error)) {
rspamd_fprintf (stderr, "option parsing failed: %s\n", error->message);
g_error_free (error);
exit (1);
}
if (config_file) {
/* Load config file, assuming that it has all information required */
struct ucl_parser *parser;
parser = ucl_parser_new (0);
rspamd_ucl_add_conf_variables (parser, ucl_vars);
if (!ucl_parser_add_file (parser, config_file)) {
msg_err ("ucl parser error: %s", ucl_parser_get_error (parser));
ucl_parser_free (parser);
exit (EXIT_FAILURE);
}
obj = ucl_parser_get_object (parser);
ucl_parser_free (parser);
}
else {
/* We need to get all information from the command line */
ucl_object_t *classifier, *statfile_ham, *statfile_spam, *tmp, *redis;
/* Check arguments sanity */
if (spam_db == NULL) {
msg_err ("No spam-db specified");
exit (EXIT_FAILURE);
}
if (ham_db == NULL) {
msg_err ("No ham-db specified");
exit (EXIT_FAILURE);
}
if (redis_host == NULL) {
msg_err ("No redis-host specified");
exit (EXIT_FAILURE);
}
if (symbol_ham == NULL) {
msg_err ("No symbol-ham specified");
exit (EXIT_FAILURE);
}
if (symbol_spam == NULL) {
msg_err ("No symbol-spam specified");
exit (EXIT_FAILURE);
}
obj = ucl_object_typed_new (UCL_OBJECT);
classifier = ucl_object_typed_new (UCL_OBJECT);
ucl_object_insert_key (obj, classifier, "classifier", 0, false);
/* Now we need to create "bayes" key in it */
tmp = ucl_object_typed_new (UCL_OBJECT);
ucl_object_insert_key (classifier, tmp, "bayes", 0, false);
classifier = tmp;
ucl_object_insert_key (classifier, ucl_object_fromstring ("sqlite3"),
"backend", 0, false);
if (cache_db != NULL) {
ucl_object_t *cache;
cache = ucl_object_typed_new (UCL_OBJECT);
ucl_object_insert_key (cache, ucl_object_fromstring ("sqlite3"),
"type", 0, false);
ucl_object_insert_key (cache, ucl_object_fromstring (cache_db),
"file", 0, false);
ucl_object_insert_key (classifier, cache, "cache", 0, false);
}
statfile_ham = ucl_object_typed_new (UCL_OBJECT);
ucl_object_insert_key (statfile_ham, ucl_object_fromstring (symbol_ham),
"symbol", 0, false);
ucl_object_insert_key (statfile_ham, ucl_object_frombool (false),
"spam", 0, false);
ucl_object_insert_key (statfile_ham, ucl_object_fromstring (ham_db),
"db", 0, false);
statfile_spam = ucl_object_typed_new (UCL_OBJECT);
ucl_object_insert_key (statfile_spam, ucl_object_fromstring (symbol_spam),
"symbol", 0, false);
ucl_object_insert_key (statfile_spam, ucl_object_frombool (true),
"spam", 0, false);
ucl_object_insert_key (statfile_spam, ucl_object_fromstring (spam_db),
"db", 0, false);
DL_APPEND (statfile_ham, statfile_spam);
ucl_object_insert_key (classifier, statfile_ham,
"statfile", 0, false);
/* Deal with redis */
redis = ucl_object_typed_new (UCL_OBJECT);
ucl_object_insert_key (obj, redis, "redis", 0, false);
ucl_object_insert_key (redis, ucl_object_fromstring (redis_host),
"servers", 0, false);
if (redis_db) {
ucl_object_insert_key (redis, ucl_object_fromstring (redis_db),
"dbname", 0, false);
}
if (redis_password) {
ucl_object_insert_key (redis, ucl_object_fromstring (redis_password),
"password", 0, false);
}
}
ucl_object_insert_key (obj, ucl_object_frombool (reset_previous),
"reset_previous", 0, false);
if (expire != 0) {
ucl_object_insert_key (obj, ucl_object_fromdouble (expire),
"expire", 0, false);
}
rspamadm_execute_lua_ucl_subr (argc,
argv,
obj,
"stat_convert",
TRUE);
ucl_object_unref (obj);
}
/**
* Apply configuration to the specified configuration
* @param conf_metrics
* @param cfg
*/
static void
apply_dynamic_conf (const ucl_object_t *top, struct rspamd_config *cfg)
{
gint test_act;
const ucl_object_t *cur_elt, *cur_nm, *it_val;
ucl_object_iter_t it = NULL;
struct metric *real_metric;
struct metric_action *cur_action;
struct rspamd_symbol_def *s;
while ((cur_elt = ucl_iterate_object (top, &it, true))) {
if (ucl_object_type (cur_elt) != UCL_OBJECT) {
msg_err ("loaded json array element is not an object");
continue;
}
cur_nm = ucl_object_find_key (cur_elt, "metric");
if (!cur_nm || ucl_object_type (cur_nm) != UCL_STRING) {
msg_err (
"loaded json metric object element has no 'metric' attribute");
continue;
}
real_metric = g_hash_table_lookup (cfg->metrics,
ucl_object_tostring (cur_nm));
if (real_metric == NULL) {
msg_warn ("cannot find metric %s", ucl_object_tostring (cur_nm));
continue;
}
cur_nm = ucl_object_find_key (cur_elt, "symbols");
/* Parse symbols */
if (cur_nm && ucl_object_type (cur_nm) == UCL_ARRAY) {
ucl_object_iter_t nit = NULL;
while ((it_val = ucl_iterate_object (cur_nm, &nit, true))) {
if (ucl_object_find_key (it_val, "name") &&
ucl_object_find_key (it_val, "value")) {
const ucl_object_t *n =
ucl_object_find_key (it_val, "name");
const ucl_object_t *v =
ucl_object_find_key (it_val, "value");
if((s = g_hash_table_lookup (real_metric->symbols,
ucl_object_tostring (n))) != NULL) {
*s->weight_ptr = ucl_object_todouble (v);
}
}
else {
msg_info (
"json symbol object has no mandatory 'name' and 'value' attributes");
}
}
}
else {
ucl_object_t *arr;
arr = ucl_object_typed_new (UCL_ARRAY);
ucl_object_insert_key ((ucl_object_t *)cur_elt, arr, "symbols",
sizeof ("symbols") - 1, false);
}
cur_nm = ucl_object_find_key (cur_elt, "actions");
/* Parse actions */
if (cur_nm && ucl_object_type (cur_nm) == UCL_ARRAY) {
ucl_object_iter_t nit = NULL;
while ((it_val = ucl_iterate_object (cur_nm, &nit, true))) {
if (ucl_object_find_key (it_val, "name") &&
ucl_object_find_key (it_val, "value")) {
if (!rspamd_action_from_str (ucl_object_tostring (
ucl_object_find_key (it_val, "name")), &test_act)) {
msg_err ("unknown action: %s",
ucl_object_tostring (ucl_object_find_key (it_val,
"name")));
continue;
}
cur_action = &real_metric->actions[test_act];
cur_action->action = test_act;
cur_action->score =
ucl_object_todouble (ucl_object_find_key (it_val,
"value"));
}
else {
msg_info (
"json action object has no mandatory 'name' and 'value' attributes");
}
}
}
else {
ucl_object_t *arr;
arr = ucl_object_typed_new (UCL_ARRAY);
ucl_object_insert_key ((ucl_object_t *)cur_elt, arr, "actions",
sizeof ("actions") - 1, false);
}
}
}
static gint
lua_worker_get_stat (lua_State *L)
{
struct rspamd_worker *w = lua_check_worker (L, 1);
if (w) {
rspamd_mempool_stat_t mem_st;
struct rspamd_stat *stat, stat_copy;
ucl_object_t *top, *sub;
gint i;
guint64 spam = 0, ham = 0;
memset (&mem_st, 0, sizeof (mem_st));
rspamd_mempool_stat (&mem_st);
memcpy (&stat_copy, w->srv->stat, sizeof (stat_copy));
stat = &stat_copy;
top = ucl_object_typed_new (UCL_OBJECT);
ucl_object_insert_key (top, ucl_object_fromint (
stat->messages_scanned), "scanned", 0, false);
ucl_object_insert_key (top, ucl_object_fromint (
stat->messages_learned), "learned", 0, false);
if (stat->messages_scanned > 0) {
sub = ucl_object_typed_new (UCL_OBJECT);
for (i = METRIC_ACTION_REJECT; i <= METRIC_ACTION_NOACTION; i++) {
ucl_object_insert_key (sub,
ucl_object_fromint (stat->actions_stat[i]),
rspamd_action_to_str (i), 0, false);
if (i < METRIC_ACTION_GREYLIST) {
spam += stat->actions_stat[i];
}
else {
ham += stat->actions_stat[i];
}
}
ucl_object_insert_key (top, sub, "actions", 0, false);
}
else {
sub = ucl_object_typed_new (UCL_OBJECT);
for (i = METRIC_ACTION_REJECT; i <= METRIC_ACTION_NOACTION; i++) {
ucl_object_insert_key (sub,
0,
rspamd_action_to_str (i), 0, false);
}
ucl_object_insert_key (top, sub, "actions", 0, false);
}
ucl_object_insert_key (top, ucl_object_fromint (
spam), "spam_count", 0, false);
ucl_object_insert_key (top, ucl_object_fromint (
ham), "ham_count", 0, false);
ucl_object_insert_key (top,
ucl_object_fromint (stat->connections_count), "connections", 0, false);
ucl_object_insert_key (top,
ucl_object_fromint (stat->control_connections_count),
"control_connections", 0, false);
ucl_object_insert_key (top,
ucl_object_fromint (mem_st.pools_allocated), "pools_allocated", 0,
false);
ucl_object_insert_key (top,
ucl_object_fromint (mem_st.pools_freed), "pools_freed", 0, false);
ucl_object_insert_key (top,
ucl_object_fromint (mem_st.bytes_allocated), "bytes_allocated", 0,
false);
ucl_object_insert_key (top,
ucl_object_fromint (
mem_st.chunks_allocated), "chunks_allocated", 0, false);
ucl_object_insert_key (top,
ucl_object_fromint (mem_st.shared_chunks_allocated),
"shared_chunks_allocated", 0, false);
ucl_object_insert_key (top,
ucl_object_fromint (mem_st.chunks_freed), "chunks_freed", 0, false);
ucl_object_insert_key (top,
ucl_object_fromint (
mem_st.oversized_chunks), "chunks_oversized", 0, false);
ucl_object_push_lua (L, top, true);
ucl_object_unref (top);
}
else {
return luaL_error (L, "invalid arguments");
}
return 1;
}
int
main (int argc, char **argv)
{
ucl_object_t *obj, *cur, *ar, *ar1, *ref, *test_obj, *comments;
ucl_object_iter_t it;
const ucl_object_t *found, *it_obj, *test;
struct ucl_emitter_functions *fn;
FILE *out;
unsigned char *emitted;
const char *fname_out = NULL;
struct ucl_parser *parser;
int ret = 0;
switch (argc) {
case 2:
fname_out = argv[1];
break;
}
if (fname_out != NULL) {
out = fopen (fname_out, "w");
if (out == NULL) {
exit (-errno);
}
}
else {
out = stdout;
}
obj = ucl_object_typed_new (UCL_OBJECT);
/* Keys replacing */
cur = ucl_object_fromstring_common ("value1", 0, UCL_STRING_TRIM);
ucl_object_insert_key (obj, cur, "key0", 0, false);
cur = ucl_object_fromdouble (0.1);
assert (ucl_object_replace_key (obj, cur, "key0", 0, false));
/* Create some strings */
cur = ucl_object_fromstring_common (" test string ", 0, UCL_STRING_TRIM);
ucl_object_insert_key (obj, cur, "key1", 0, false);
cur = ucl_object_fromstring_common (" test \nstring\n\r\n\b\t\f\\\" ", 0,
UCL_STRING_TRIM | UCL_STRING_ESCAPE);
ucl_object_insert_key (obj, cur, "key2", 0, false);
cur = ucl_object_fromstring_common (" test string \n", 0, 0);
ucl_object_insert_key (obj, cur, "key3", 0, false);
/* Array of numbers */
ar = ucl_object_typed_new (UCL_ARRAY);
cur = ucl_object_fromint (10);
ucl_array_append (ar, cur);
assert (ucl_array_index_of (ar, cur) == 0);
cur = ucl_object_fromdouble (10.1);
ucl_array_append (ar, cur);
assert (ucl_array_index_of (ar, cur) == 1);
cur = ucl_object_fromdouble (9.999);
ucl_array_prepend (ar, cur);
assert (ucl_array_index_of (ar, cur) == 0);
ar1 = ucl_object_copy (ar);
cur = ucl_object_fromstring ("abc");
ucl_array_prepend (ar1, cur);
cur = ucl_object_fromstring ("cde");
ucl_array_prepend (ar1, cur);
cur = ucl_object_fromstring ("аАаБаВ"); /* UTF8 */
ucl_array_prepend (ar1, cur);
cur = ucl_object_fromstring ("ааБаВ"); /* UTF8 */
ucl_array_prepend (ar1, cur);
/*
* This is ususally broken or fragile as utf collate is far from perfect
cur = ucl_object_fromstring ("баБаВ");
ucl_array_prepend (ar1, cur);
cur = ucl_object_fromstring ("ааБаВ"); // hello to @bapt
ucl_array_prepend (ar1, cur);
*/
cur = ucl_object_fromstring ("№"); /* everybody likes emoji in the code */
ucl_array_prepend (ar1, cur);
ucl_object_array_sort (ar1, ucl_object_compare_qsort);
/* Removing from an array */
cur = ucl_object_fromdouble (1.0);
ucl_array_append (ar, cur);
cur = ucl_array_delete (ar, cur);
assert (ucl_object_todouble (cur) == 1.0);
ucl_object_unref (cur);
cur = ucl_object_fromdouble (2.0);
ucl_array_append (ar, cur);
cur = ucl_array_pop_last (ar);
assert (ucl_object_todouble (cur) == 2.0);
ucl_object_unref (cur);
cur = ucl_object_fromdouble (3.0);
ucl_array_prepend (ar, cur);
cur = ucl_array_pop_first (ar);
assert (ucl_object_todouble (cur) == 3.0);
ucl_object_unref (cur);
ucl_object_insert_key (obj, ar, "key4", 0, false);
cur = ucl_object_frombool (true);
/* Ref object to test refcounts */
ref = ucl_object_ref (cur);
ucl_object_insert_key (obj, cur, "key4", 0, false);
/* Empty strings */
cur = ucl_object_fromstring_common (" ", 0, UCL_STRING_TRIM);
ucl_object_insert_key (obj, cur, "key5", 0, false);
cur = ucl_object_fromstring_common ("", 0, UCL_STRING_ESCAPE);
ucl_object_insert_key (obj, cur, "key6", 0, false);
cur = ucl_object_fromstring_common (" \n", 0, UCL_STRING_ESCAPE);
ucl_object_insert_key (obj, cur, "key7", 0, false);
/* Numbers and booleans */
cur = ucl_object_fromstring_common ("1mb", 0, UCL_STRING_ESCAPE | UCL_STRING_PARSE);
ucl_object_insert_key (obj, cur, "key8", 0, false);
cur = ucl_object_fromstring_common ("3.14", 0, UCL_STRING_PARSE);
ucl_object_insert_key (obj, cur, "key9", 0, false);
cur = ucl_object_fromstring_common ("true", 0, UCL_STRING_PARSE);
ucl_object_insert_key (obj, cur, "key10", 0, false);
cur = ucl_object_fromstring_common (" off ", 0, UCL_STRING_PARSE | UCL_STRING_TRIM);
ucl_object_insert_key (obj, cur, "key11", 0, false);
cur = ucl_object_fromstring_common ("*****@*****.**", 0, UCL_STRING_PARSE_INT);
ucl_object_insert_key (obj, cur, "key12", 0, false);
cur = ucl_object_fromstring_common ("#test", 0, UCL_STRING_PARSE_INT);
ucl_object_insert_key (obj, cur, "key13", 0, false);
cur = ucl_object_frombool (true);
ucl_object_insert_key (obj, cur, "k=3", 0, false);
ucl_object_insert_key (obj, ar1, "key14", 0, false);
cur = ucl_object_new_userdata (ud_dtor, ud_emit, NULL);
ucl_object_insert_key (obj, cur, "key15", 0, false);
/* More tests for keys */
cur = ucl_object_fromlstring ("test", 3);
ucl_object_insert_key (obj, cur, "key16", 0, false);
test = ucl_object_lookup_any (obj, "key100", "key200", "key300", "key16", NULL);
assert (test == cur);
test = ucl_object_lookup_len (obj, "key160", 5);
assert (test == cur);
cur = ucl_object_pop_key (obj, "key16");
assert (test == cur);
test = ucl_object_pop_key (obj, "key16");
assert (test == NULL);
test = ucl_object_lookup_len (obj, "key160", 5);
assert (test == NULL);
/* Objects merging tests */
test_obj = ucl_object_new_full (UCL_OBJECT, 2);
ucl_object_insert_key (test_obj, cur, "key16", 0, true);
ucl_object_merge (obj, test_obj, true);
ucl_object_unref (test_obj);
/* Array merging test */
test_obj = ucl_object_new_full (UCL_ARRAY, 3);
ucl_array_append (test_obj, ucl_object_fromstring ("test"));
ucl_array_merge (test_obj, ar1, false);
ucl_object_insert_key (obj, test_obj, "key17", 0, true);
/* Object deletion */
cur = ucl_object_fromstring ("test");
ucl_object_insert_key (obj, cur, "key18", 0, true);
assert (ucl_object_delete_key (obj, "key18"));
assert (!ucl_object_delete_key (obj, "key18"));
cur = ucl_object_fromlstring ("test", 4);
ucl_object_insert_key (obj, cur, "key18\0\0", 7, true);
assert (ucl_object_lookup_len (obj, "key18\0\0", 7) == cur);
assert (ucl_object_lookup (obj, "key18") == NULL);
assert (ucl_object_lookup_len (obj, "key18\0\1", 7) == NULL);
assert (ucl_object_delete_keyl (obj, "key18\0\0", 7));
/* Comments */
comments = ucl_object_typed_new (UCL_OBJECT);
found = ucl_object_lookup (obj, "key17");
test = ucl_object_lookup (obj, "key16");
ucl_comments_add (comments, found, "# test comment");
assert (ucl_comments_find (comments, found) != NULL);
assert (ucl_comments_find (comments, test) == NULL);
ucl_comments_move (comments, found, test);
assert (ucl_comments_find (comments, found) == NULL);
assert (ucl_comments_find (comments, test) != NULL);
/* Array replace */
ar1 = ucl_object_typed_new (UCL_ARRAY);
cur = ucl_object_fromstring ("test");
cur = ucl_elt_append (cur, ucl_object_fromstring ("test1"));
ucl_array_append (ar1, cur);
test = ucl_array_replace_index (ar1, ucl_object_fromstring ("test2"), 0);
assert (test == cur);
/* Try to find using path */
/* Should exist */
found = ucl_object_lookup_path (obj, "key4.1");
assert (found != NULL && ucl_object_toint (found) == 10);
/* . should be ignored */
found = ucl_object_lookup_path (obj, ".key4.1");
assert (found != NULL && ucl_object_toint (found) == 10);
/* moar dots... */
found = ucl_object_lookup_path (obj, ".key4........1...");
assert (found != NULL && ucl_object_toint (found) == 10);
/* No such index */
found = ucl_object_lookup_path (obj, ".key4.3");
assert (found == NULL);
/* No such key */
found = ucl_object_lookup_path (obj, "key9..key1");
assert (found == NULL);
/* Test iteration */
it = ucl_object_iterate_new (obj);
it_obj = ucl_object_iterate_safe (it, true);
/* key0 = 0.1 */
assert (ucl_object_type (it_obj) == UCL_FLOAT);
it_obj = ucl_object_iterate_safe (it, true);
/* key1 = "" */
assert (ucl_object_type (it_obj) == UCL_STRING);
it_obj = ucl_object_iterate_safe (it, true);
/* key2 = "" */
assert (ucl_object_type (it_obj) == UCL_STRING);
it_obj = ucl_object_iterate_safe (it, true);
/* key3 = "" */
assert (ucl_object_type (it_obj) == UCL_STRING);
it_obj = ucl_object_iterate_safe (it, true);
/* key4 = ([float, int, float], boolean) */
ucl_object_iterate_reset (it, it_obj);
it_obj = ucl_object_iterate_safe (it, true);
assert (ucl_object_type (it_obj) == UCL_FLOAT);
it_obj = ucl_object_iterate_safe (it, true);
assert (ucl_object_type (it_obj) == UCL_INT);
it_obj = ucl_object_iterate_safe (it, true);
assert (ucl_object_type (it_obj) == UCL_FLOAT);
it_obj = ucl_object_iterate_safe (it, true);
assert (ucl_object_type (it_obj) == UCL_BOOLEAN);
ucl_object_iterate_free (it);
fn = ucl_object_emit_memory_funcs (&emitted);
assert (ucl_object_emit_full (obj, UCL_EMIT_CONFIG, fn, comments));
fprintf (out, "%s\n", emitted);
ucl_object_emit_funcs_free (fn);
ucl_object_unref (obj);
ucl_object_unref (comments);
parser = ucl_parser_new (UCL_PARSER_NO_IMPLICIT_ARRAYS);
if (ucl_parser_add_chunk_full (parser, emitted, strlen (emitted),
3, UCL_DUPLICATE_ERROR, UCL_PARSE_UCL)) {
/* Should fail due to duplicate */
assert (0);
}
else {
assert (ucl_parser_get_error (parser) != NULL);
ucl_parser_clear_error (parser);
ucl_parser_free (parser);
parser = ucl_parser_new (0);
ucl_parser_add_chunk_full (parser, emitted, strlen (emitted),
3, UCL_DUPLICATE_MERGE, UCL_PARSE_UCL);
}
assert (ucl_parser_get_column (parser) == 0);
assert (ucl_parser_get_linenum (parser) != 0);
ucl_parser_clear_error (parser);
assert (ucl_parser_get_error_code (parser) == 0);
obj = ucl_parser_get_object (parser);
ucl_parser_free (parser);
ucl_object_free (obj);
if (emitted != NULL) {
free (emitted);
}
fclose (out);
/* Ref should still be accessible */
ref->value.iv = 100500;
ucl_object_unref (ref);
return ret;
}
/* Called when we have connected to the redis server and got keys to check */
static void
rspamd_redis_stat_keys (redisAsyncContext *c, gpointer r, gpointer priv)
{
struct rspamd_redis_stat_cbdata *cbdata = priv;
redisReply *reply = r, *elt;
gchar **pk, *k;
guint i, processed = 0;
if (cbdata->wanna_die) {
return;
}
cbdata->inflight --;
if (c->err == 0 && r != NULL) {
if (reply->type == REDIS_REPLY_ARRAY) {
g_ptr_array_set_size (cbdata->cur_keys, reply->elements);
for (i = 0; i < reply->elements; i ++) {
elt = reply->element[i];
if (elt->type == REDIS_REPLY_STRING) {
pk = (gchar **)&g_ptr_array_index (cbdata->cur_keys, i);
*pk = g_malloc (elt->len + 1);
rspamd_strlcpy (*pk, elt->str, elt->len + 1);
processed ++;
}
}
if (processed) {
for (i = 0; i < cbdata->cur_keys->len; i ++) {
k = (gchar *)g_ptr_array_index (cbdata->cur_keys, i);
if (k) {
redisAsyncCommand (cbdata->redis, rspamd_redis_stat_key,
cbdata,
"HLEN %s",
k);
redisAsyncCommand (cbdata->redis, rspamd_redis_stat_learns,
cbdata,
"HGET %s learns",
k);
cbdata->inflight += 2;
}
}
}
}
/* Set up the required keys */
ucl_object_insert_key (cbdata->cur,
ucl_object_typed_new (UCL_INT), "revision", 0, false);
ucl_object_insert_key (cbdata->cur,
ucl_object_typed_new (UCL_INT), "used", 0, false);
ucl_object_insert_key (cbdata->cur,
ucl_object_typed_new (UCL_INT), "total", 0, false);
ucl_object_insert_key (cbdata->cur,
ucl_object_typed_new (UCL_INT), "size", 0, false);
ucl_object_insert_key (cbdata->cur,
ucl_object_fromstring (cbdata->elt->ctx->stcf->symbol),
"symbol", 0, false);
ucl_object_insert_key (cbdata->cur, ucl_object_fromstring ("redis"),
"type", 0, false);
ucl_object_insert_key (cbdata->cur, ucl_object_fromint (0),
"languages", 0, false);
ucl_object_insert_key (cbdata->cur, ucl_object_fromint (processed),
"users", 0, false);
rspamd_upstream_ok (cbdata->selected);
}
else {
if (c->errstr) {
msg_err ("cannot get keys to gather stat: %s", c->errstr);
}
else {
msg_err ("cannot get keys to gather stat: unknown error");
}
rspamd_upstream_fail (cbdata->selected);
rspamd_redis_async_cbdata_cleanup (cbdata);
}
if (cbdata->inflight == 0) {
rspamd_redis_async_cbdata_cleanup (cbdata);
}
}
int
pkg_ini(const char *path, const char *reposdir, pkg_init_flags flags)
{
struct ucl_parser *p = NULL;
size_t i;
const char *val = NULL;
const char *buf, *walk, *value, *key, *k;
const char *evkey = NULL;
const char *nsname = NULL;
const char *evpipe = NULL;
const ucl_object_t *cur, *object;
ucl_object_t *obj = NULL, *o, *ncfg;
ucl_object_iter_t it = NULL;
struct sbuf *ukey = NULL;
bool fatal_errors = false;
k = NULL;
o = NULL;
pkg_get_myarch(myabi, BUFSIZ);
pkg_get_myarch_legacy(myabi_legacy, BUFSIZ);
if (parsed != false) {
pkg_emit_error("pkg_init() must only be called once");
return (EPKG_FATAL);
}
if (((flags & PKG_INIT_FLAG_USE_IPV4) == PKG_INIT_FLAG_USE_IPV4) &&
((flags & PKG_INIT_FLAG_USE_IPV6) == PKG_INIT_FLAG_USE_IPV6)) {
pkg_emit_error("Invalid flags for pkg_init()");
return (EPKG_FATAL);
}
config = ucl_object_typed_new(UCL_OBJECT);
for (i = 0; i < c_size; i++) {
switch (c[i].type) {
case PKG_STRING:
obj = ucl_object_fromstring_common(
c[i].def != NULL ? c[i].def : "", 0, UCL_STRING_TRIM);
ucl_object_insert_key(config, obj,
c[i].key, strlen(c[i].key), false);
break;
case PKG_INT:
ucl_object_insert_key(config,
ucl_object_fromstring_common(c[i].def, 0, UCL_STRING_PARSE_INT),
c[i].key, strlen(c[i].key), false);
break;
case PKG_BOOL:
ucl_object_insert_key(config,
ucl_object_fromstring_common(c[i].def, 0, UCL_STRING_PARSE_BOOLEAN),
c[i].key, strlen(c[i].key), false);
break;
case PKG_OBJECT:
obj = ucl_object_typed_new(UCL_OBJECT);
if (c[i].def != NULL) {
walk = buf = c[i].def;
while ((buf = strchr(buf, ',')) != NULL) {
key = walk;
value = walk;
while (*value != ',') {
if (*value == '=')
break;
value++;
}
ucl_object_insert_key(obj,
ucl_object_fromstring_common(value + 1, buf - value - 1, UCL_STRING_TRIM),
key, value - key, false);
buf++;
walk = buf;
}
key = walk;
value = walk;
while (*value != ',') {
if (*value == '=')
break;
value++;
}
if (o == NULL)
o = ucl_object_typed_new(UCL_OBJECT);
ucl_object_insert_key(o,
ucl_object_fromstring_common(value + 1, strlen(value + 1), UCL_STRING_TRIM),
key, value - key, false);
}
ucl_object_insert_key(config, obj,
c[i].key, strlen(c[i].key), false);
break;
case PKG_ARRAY:
obj = ucl_object_typed_new(UCL_ARRAY);
if (c[i].def != NULL) {
walk = buf = c[i].def;
while ((buf = strchr(buf, ',')) != NULL) {
ucl_array_append(obj,
ucl_object_fromstring_common(walk, buf - walk, UCL_STRING_TRIM));
buf++;
walk = buf;
}
ucl_array_append(obj,
ucl_object_fromstring_common(walk, strlen(walk), UCL_STRING_TRIM));
}
ucl_object_insert_key(config, obj,
c[i].key, strlen(c[i].key), false);
break;
}
}
if (path == NULL)
path = PREFIX"/etc/pkg.conf";
p = ucl_parser_new(0);
errno = 0;
obj = NULL;
if (!ucl_parser_add_file(p, path)) {
if (errno != ENOENT)
pkg_emit_error("Invalid configuration file: %s", ucl_parser_get_error(p));
} else {
obj = ucl_parser_get_object(p);
}
ncfg = NULL;
while (obj != NULL && (cur = ucl_iterate_object(obj, &it, true))) {
sbuf_init(&ukey);
key = ucl_object_key(cur);
for (i = 0; key[i] != '\0'; i++)
sbuf_putc(ukey, toupper(key[i]));
sbuf_finish(ukey);
object = ucl_object_find_keyl(config, sbuf_data(ukey), sbuf_len(ukey));
if (strncasecmp(sbuf_data(ukey), "PACKAGESITE", sbuf_len(ukey))
== 0 || strncasecmp(sbuf_data(ukey), "PUBKEY",
sbuf_len(ukey)) == 0 || strncasecmp(sbuf_data(ukey),
"MIRROR_TYPE", sbuf_len(ukey)) == 0) {
pkg_emit_error("%s in pkg.conf is no longer "
"supported. Convert to the new repository style."
" See pkg.conf(5)", sbuf_data(ukey));
fatal_errors = true;
continue;
}
/* ignore unknown keys */
if (object == NULL)
continue;
if (object->type != cur->type) {
pkg_emit_error("Malformed key %s, ignoring", key);
continue;
}
if (ncfg == NULL)
ncfg = ucl_object_typed_new(UCL_OBJECT);
ucl_object_insert_key(ncfg, ucl_object_copy(cur), sbuf_data(ukey), sbuf_len(ukey), true);
}
if (fatal_errors) {
ucl_object_unref(ncfg);
ucl_parser_free(p);
return (EPKG_FATAL);
}
if (ncfg != NULL) {
it = NULL;
while (( cur = ucl_iterate_object(ncfg, &it, true))) {
key = ucl_object_key(cur);
ucl_object_replace_key(config, ucl_object_ref(cur), key, strlen(key), true);
}
ucl_object_unref(ncfg);
}
ncfg = NULL;
it = NULL;
while ((cur = ucl_iterate_object(config, &it, true))) {
o = NULL;
key = ucl_object_key(cur);
val = getenv(key);
if (val == NULL)
continue;
switch (cur->type) {
case UCL_STRING:
o = ucl_object_fromstring_common(val, 0, UCL_STRING_TRIM);
break;
case UCL_INT:
o = ucl_object_fromstring_common(val, 0, UCL_STRING_PARSE_INT);
if (o->type != UCL_INT) {
pkg_emit_error("Invalid type for environment "
"variable %s, got %s, while expecting an integer",
key, val);
ucl_object_unref(o);
continue;
}
break;
case UCL_BOOLEAN:
o = ucl_object_fromstring_common(val, 0, UCL_STRING_PARSE_BOOLEAN);
if (o->type != UCL_BOOLEAN) {
pkg_emit_error("Invalid type for environment "
"variable %s, got %s, while expecting a boolean",
key, val);
ucl_object_unref(o);
continue;
}
break;
case UCL_OBJECT:
o = ucl_object_typed_new(UCL_OBJECT);
walk = buf = val;
while ((buf = strchr(buf, ',')) != NULL) {
k = walk;
value = walk;
while (*value != ',') {
if (*value == '=')
break;
value++;
}
ucl_object_insert_key(o,
ucl_object_fromstring_common(value + 1, buf - value - 1, UCL_STRING_TRIM),
k, value - k, false);
buf++;
walk = buf;
}
key = walk;
value = walk;
while (*value != '\0') {
if (*value == '=')
break;
value++;
}
ucl_object_insert_key(o,
ucl_object_fromstring_common(value + 1, strlen(value + 1), UCL_STRING_TRIM),
k, value - k, false);
break;
case UCL_ARRAY:
o = ucl_object_typed_new(UCL_ARRAY);
walk = buf = val;
while ((buf = strchr(buf, ',')) != NULL) {
ucl_array_append(o,
ucl_object_fromstring_common(walk, buf - walk, UCL_STRING_TRIM));
buf++;
walk = buf;
}
ucl_array_append(o,
ucl_object_fromstring_common(walk, strlen(walk), UCL_STRING_TRIM));
break;
default:
/* ignore other types */
break;
}
if (o != NULL) {
if (ncfg == NULL)
ncfg = ucl_object_typed_new(UCL_OBJECT);
ucl_object_insert_key(ncfg, o, key, strlen(key), true);
}
}
if (ncfg != NULL) {
it = NULL;
while (( cur = ucl_iterate_object(ncfg, &it, true))) {
key = ucl_object_key(cur);
ucl_object_replace_key(config, ucl_object_ref(cur), key, strlen(key), true);
}
ucl_object_unref(ncfg);
}
disable_plugins_if_static();
parsed = true;
ucl_object_unref(obj);
ucl_parser_free(p);
if (strcmp(pkg_object_string(pkg_config_get("ABI")), "unknown") == 0) {
pkg_emit_error("Unable to determine ABI");
return (EPKG_FATAL);
}
pkg_debug(1, "%s", "pkg initialized");
/* Start the event pipe */
evpipe = pkg_object_string(pkg_config_get("EVENT_PIPE"));
if (evpipe != NULL)
connect_evpipe(evpipe);
debug_level = pkg_object_int(pkg_config_get("DEBUG_LEVEL"));
it = NULL;
object = ucl_object_find_key(config, "PKG_ENV");
while ((cur = ucl_iterate_object(object, &it, true))) {
evkey = ucl_object_key(cur);
pkg_debug(1, "Setting env var: %s", evkey);
if (evkey != NULL && evkey[0] != '\0')
setenv(evkey, ucl_object_tostring_forced(cur), 1);
}
/* load the repositories */
load_repositories(reposdir, flags);
setenv("HTTP_USER_AGENT", "pkg/"PKGVERSION, 1);
/* bypass resolv.conf with specified NAMESERVER if any */
nsname = pkg_object_string(pkg_config_get("NAMESERVER"));
if (nsname != NULL)
set_nameserver(ucl_object_tostring_forced(o));
return (EPKG_OK);
}
int
merge_mode(char *destination_node, char *data)
{
ucl_object_t *dst_obj = NULL;
ucl_object_t *sub_obj = NULL;
ucl_object_t *old_obj = NULL;
ucl_object_t *tmp_obj = NULL;
int success = 0;
char *dst_frag;
setparser = ucl_parser_new(UCL_PARSER_KEY_LOWERCASE |
UCL_PARSER_NO_IMPLICIT_ARRAYS);
/* Lookup the destination to write to */
dst_obj = get_parent(destination_node);
sub_obj = get_object(destination_node);
if (sub_obj == NULL) {
fprintf(stderr, "Failed to find destination node: %s\n",
destination_node);
return false;
}
if (include_file != NULL) {
/* get UCL to add from file */
set_obj = parse_file(setparser, include_file);
} else if (data == NULL || strcmp(data, "-") == 0) {
/* get UCL to add from stdin */
set_obj = parse_input(setparser, stdin);
} else {
/* User provided data inline */
set_obj = parse_string(setparser, data);
}
if (debug > 0) {
char *rt = NULL, *dt = NULL, *st = NULL;
rt = objtype_as_string(dst_obj);
dt = objtype_as_string(sub_obj);
st = objtype_as_string(set_obj);
fprintf(stderr, "root type: %s, destination type: %s, new type: %s\n",
rt, dt, st);
if (rt != NULL) free(rt);
if (dt != NULL) free(dt);
if (st != NULL) free(st);
fprintf(stderr, "Merging key %s to root: %s\n",
ucl_object_key(sub_obj), ucl_object_key(dst_obj));
}
dst_frag = strrchr(destination_node, input_sepchar);
if (dst_frag == NULL) {
dst_frag = destination_node;
} else {
dst_frag++;
}
/* Add it to the object here */
if (sub_obj == dst_obj && *dst_frag != '\0') {
/* Sub-object does not exist, create a new one */
success = ucl_object_insert_key(dst_obj, set_obj, dst_frag, 0,
true);
} else if (ucl_object_type(sub_obj) == UCL_ARRAY && ucl_object_type(set_obj) == UCL_ARRAY) {
if (debug > 0) {
fprintf(stderr, "Merging array of size %u with array of size %u\n",
sub_obj->len, set_obj->len);
}
success = ucl_array_merge(sub_obj, set_obj, true);
} else if (ucl_object_type(sub_obj) == UCL_ARRAY) {
if (debug > 0) {
fprintf(stderr, "Appending object to array of size %u\n",
sub_obj->len);
}
success = ucl_array_append(sub_obj, ucl_object_ref(set_obj));
} else if (ucl_object_type(sub_obj) == UCL_OBJECT && ucl_object_type(set_obj) == UCL_OBJECT) {
if (debug > 0) {
fprintf(stderr, "Merging object %s with object %s\n",
ucl_object_key(sub_obj), ucl_object_key(set_obj));
}
/* XXX not supported:
* success = ucl_object_merge(sub_obj, set_obj, false, true);
*/
/* Old non-recursive way */
/*
success = ucl_object_merge(sub_obj, set_obj, false);
*/
success = merge_recursive(sub_obj, set_obj, false);
} else if (ucl_object_type(sub_obj) != UCL_OBJECT && ucl_object_type(sub_obj) != UCL_ARRAY) {
/* Create an explicit array */
if (debug > 0) {
fprintf(stderr, "Creating an array and appended the new item\n");
}
tmp_obj = ucl_object_typed_new(UCL_ARRAY);
/*
* Reference and Append the original scalar
* The additional reference is required because the old object will be
* unreferenced as part of the ucl_object_replace_key operation
*/
ucl_array_append(tmp_obj, ucl_object_ref(sub_obj));
/* Reference and Append the new scalar (unref in cleanup()) */
ucl_array_append(tmp_obj, ucl_object_ref(set_obj));
/* Replace the old object with the newly created one */
if (ucl_object_type(dst_obj) == UCL_ARRAY) {
old_obj = ucl_array_replace_index(dst_obj, tmp_obj,
ucl_array_index_of(dst_obj, sub_obj));
success = false;
if (old_obj != NULL) {
ucl_object_unref(old_obj);
success = true;
}
} else {
success = ucl_object_replace_key(dst_obj, tmp_obj,
ucl_object_key(sub_obj), 0, true);
}
} else {
if (debug > 0) {
fprintf(stderr, "Merging object into key %s\n",
ucl_object_key(sub_obj));
}
success = ucl_object_insert_key_merged(dst_obj, ucl_object_ref(set_obj),
ucl_object_key(sub_obj), 0, true);
}
return success;
}
