/*
* Check dependencies for an object
*/
static bool
ucl_schema_validate_dependencies (const ucl_object_t *deps,
const ucl_object_t *obj, struct ucl_schema_error *err,
const ucl_object_t *root,
ucl_object_t *ext_ref)
{
const ucl_object_t *elt, *cur, *cur_dep;
ucl_object_iter_t iter = NULL, piter;
bool ret = true;
while (ret && (cur = ucl_object_iterate (deps, &iter, true)) != NULL) {
elt = ucl_object_lookup (obj, ucl_object_key (cur));
if (elt != NULL) {
/* Need to check dependencies */
if (cur->type == UCL_ARRAY) {
piter = NULL;
while (ret && (cur_dep = ucl_object_iterate (cur, &piter, true)) != NULL) {
if (ucl_object_lookup (obj, ucl_object_tostring (cur_dep)) == NULL) {
ucl_schema_create_error (err, UCL_SCHEMA_MISSING_DEPENDENCY, elt,
"dependency %s is missing for key %s",
ucl_object_tostring (cur_dep), ucl_object_key (cur));
ret = false;
break;
}
}
}
else if (cur->type == UCL_OBJECT) {
ret = ucl_schema_validate (cur, obj, true, err, root, ext_ref);
}
}
}
return ret;
}
static gboolean
rspamd_symbols_cache_process_settings (struct rspamd_task *task,
struct symbols_cache *cache)
{
const ucl_object_t *wl, *cur, *disabled;
struct metric *def;
struct rspamd_symbols_group *gr;
GHashTableIter gr_it;
ucl_object_iter_t it = NULL;
gpointer k, v;
wl = ucl_object_lookup (task->settings, "whitelist");
if (wl != NULL) {
msg_info_task ("<%s> is whitelisted", task->message_id);
task->flags |= RSPAMD_TASK_FLAG_SKIP;
return TRUE;
}
disabled = ucl_object_lookup (task->settings, "symbols_disabled");
if (disabled) {
it = NULL;
while ((cur = ucl_iterate_object (disabled, &it, true)) != NULL) {
rspamd_symbols_cache_disable_symbol (task, cache,
ucl_object_tostring (cur));
}
}
/* Disable groups of symbols */
disabled = ucl_object_lookup (task->settings, "groups_disabled");
def = g_hash_table_lookup (task->cfg->metrics, DEFAULT_METRIC);
if (def && disabled) {
it = NULL;
while ((cur = ucl_iterate_object (disabled, &it, true)) != NULL) {
if (ucl_object_type (cur) == UCL_STRING) {
gr = g_hash_table_lookup (def->groups,
ucl_object_tostring (cur));
if (gr) {
g_hash_table_iter_init (&gr_it, gr->symbols);
while (g_hash_table_iter_next (&gr_it, &k, &v)) {
rspamd_symbols_cache_disable_symbol (task, cache, k);
}
}
}
}
}
return FALSE;
}
/* Called when we get number of elements for a specific key */
static void
rspamd_redis_stat_key (redisAsyncContext *c, gpointer r, gpointer priv)
{
struct rspamd_redis_stat_cbdata *cbdata = priv;
redisReply *reply = r;
ucl_object_t *obj;
glong num = 0;
if (cbdata->wanna_die) {
return;
}
cbdata->inflight --;
if (c->err == 0 && r != NULL) {
if (G_LIKELY (reply->type == REDIS_REPLY_INTEGER)) {
num = reply->integer;
}
else if (reply->type == REDIS_REPLY_STRING) {
rspamd_strtol (reply->str, reply->len, &num);
}
if (num < 0) {
msg_err ("bad learns count: %L", (gint64)num);
num = 0;
}
obj = (ucl_object_t *)ucl_object_lookup (cbdata->cur, "used");
if (obj) {
obj->value.iv += num;
}
obj = (ucl_object_t *)ucl_object_lookup (cbdata->cur, "total");
if (obj) {
obj->value.iv += num;
}
obj = (ucl_object_t *)ucl_object_lookup (cbdata->cur, "size");
if (obj) {
/* Size of key + size of int64_t */
obj->value.iv += num * (sizeof (G_STRINGIFY (G_MAXINT64)) +
sizeof (guint64) + sizeof (gpointer));
}
}
if (cbdata->inflight == 0) {
rspamd_redis_async_cbdata_cleanup (cbdata);
}
}
/* Called when we get number of learns for a specific key */
static void
rspamd_redis_stat_learns (redisAsyncContext *c, gpointer r, gpointer priv)
{
struct rspamd_redis_stat_cbdata *cbdata = priv;
redisReply *reply = r;
ucl_object_t *obj;
gulong num = 0;
if (cbdata->wanna_die) {
return;
}
cbdata->inflight --;
if (c->err == 0 && r != NULL) {
if (G_LIKELY (reply->type == REDIS_REPLY_INTEGER)) {
num = reply->integer;
}
else if (reply->type == REDIS_REPLY_STRING) {
rspamd_strtoul (reply->str, reply->len, &num);
}
obj = (ucl_object_t *)ucl_object_lookup (cbdata->cur, "revision");
if (obj) {
obj->value.iv += num;
}
}
if (cbdata->inflight == 0) {
rspamd_redis_async_cbdata_cleanup (cbdata);
}
}
static gint
rspamd_log_errlog_cmp (const ucl_object_t **o1, const ucl_object_t **o2)
{
const ucl_object_t *ts1, *ts2;
ts1 = ucl_object_lookup (*o1, "ts");
ts2 = ucl_object_lookup (*o2, "ts");
if (ts1 && ts2) {
gdouble t1 = ucl_object_todouble (ts1), t2 = ucl_object_todouble (ts2);
if (t1 > t2) {
return -1;
}
else if (t2 > t1) {
return 1;
}
}
return 0;
}
static void
rspamadm_confighelp_search_word_step (const ucl_object_t *obj,
ucl_object_t *res,
const gchar *str,
gsize len,
GString *path)
{
ucl_object_iter_t it = NULL;
const ucl_object_t *cur, *elt;
const gchar *dot_pos;
while ((cur = ucl_object_iterate (obj, &it, true)) != NULL) {
if (cur->keylen > 0) {
rspamd_printf_gstring (path, ".%*s", (int) cur->keylen, cur->key);
if (rspamd_substring_search_caseless (cur->key,
cur->keylen,
str,
len) != -1) {
ucl_object_insert_key (res, ucl_object_ref (cur),
path->str, path->len, true);
goto fin;
}
}
if (ucl_object_type (cur) == UCL_OBJECT) {
elt = ucl_object_lookup (cur, "data");
if (elt != NULL && ucl_object_type (elt) == UCL_STRING) {
if (rspamd_substring_search_caseless (elt->value.sv,
elt->len,
str,
len) != -1) {
ucl_object_insert_key (res, ucl_object_ref (cur),
path->str, path->len, true);
goto fin;
}
}
rspamadm_confighelp_search_word_step (cur, res, str, len, path);
}
fin:
/* Remove the last component of the path */
dot_pos = strrchr (path->str, '.');
if (dot_pos) {
g_string_erase (path, dot_pos - path->str,
path->len - (dot_pos - path->str));
}
}
}
gpointer
rspamd_redis_init (struct rspamd_stat_ctx *ctx,
struct rspamd_config *cfg, struct rspamd_statfile *st)
{
struct redis_stat_ctx *backend;
struct rspamd_statfile_config *stf = st->stcf;
struct rspamd_redis_stat_elt *st_elt;
const ucl_object_t *obj;
gboolean ret = FALSE;
backend = g_slice_alloc0 (sizeof (*backend));
/* First search in backend configuration */
obj = ucl_object_lookup (st->classifier->cfg->opts, "backend");
if (obj != NULL && ucl_object_type (obj) == UCL_OBJECT) {
ret = rspamd_redis_try_ucl (backend, obj, cfg, stf->symbol);
}
/* Now try statfiles config */
if (!ret) {
ret = rspamd_redis_try_ucl (backend, stf->opts, cfg, stf->symbol);
}
/* Now try classifier config */
if (!ret) {
ret = rspamd_redis_try_ucl (backend, st->classifier->cfg->opts, cfg,
stf->symbol);
}
if (!ret) {
msg_err_config ("cannot init redis backend for %s", stf->symbol);
g_slice_free1 (sizeof (*backend), backend);
return NULL;
}
stf->clcf->flags |= RSPAMD_FLAG_CLASSIFIER_INCREMENTING_BACKEND;
backend->stcf = stf;
st_elt = g_slice_alloc0 (sizeof (*st_elt));
st_elt->ev_base = ctx->ev_base;
st_elt->ctx = backend;
backend->stat_elt = rspamd_stat_ctx_register_async (
rspamd_redis_async_stat_cb,
rspamd_redis_async_stat_fin,
st_elt,
REDIS_STAT_TIMEOUT);
st_elt->async = backend->stat_elt;
return (gpointer)backend;
}
static double
get_specific_action_score (struct rspamd_task *task,
const ucl_object_t *metric,
struct metric_action *action)
{
const ucl_object_t *act, *sact;
const gchar *act_name;
double score;
if (metric) {
act = ucl_object_lookup (metric, "actions");
if (act) {
act_name = rspamd_action_to_str (action->action);
sact = ucl_object_lookup (act, act_name);
if (sact != NULL && ucl_object_todouble_safe (sact, &score)) {
msg_debug_task ("found override score %.2f for action %s in settings",
score, act_name);
return score;
}
}
}
return action->score;
}
static gint
lua_statfile_get_param (lua_State *L)
{
struct rspamd_statfile_config *st = lua_check_statfile (L);
const gchar *param;
const ucl_object_t *value;
param = luaL_checkstring (L, 2);
if (st != NULL && param != NULL) {
value = ucl_object_lookup (st->opts, param);
if (value != NULL) {
lua_pushstring (L, ucl_object_tostring_forced (value));
return 1;
}
}
lua_pushnil (L);
return 1;
}
static gint
lua_classifier_get_param (lua_State *L)
{
struct rspamd_classifier_config *ccf = lua_check_classifier (L);
const gchar *param;
const ucl_object_t *value;
param = luaL_checkstring (L, 2);
if (ccf != NULL && param != NULL) {
value = ucl_object_lookup (ccf->opts, param);
if (value != NULL) {
ucl_object_push_lua (L, value, true);
return 1;
}
}
lua_pushnil (L);
return 1;
}
gint
rspamd_check_action_metric (struct rspamd_task *task,
double score, double *rscore, struct metric *metric)
{
struct metric_action *action, *selected_action = NULL;
double max_score = 0;
const ucl_object_t *ms = NULL;
int i;
if (task->settings) {
ms = ucl_object_lookup (task->settings, metric->name);
}
for (i = METRIC_ACTION_REJECT; i < METRIC_ACTION_MAX; i++) {
double sc;
action = &metric->actions[i];
sc = get_specific_action_score (task, ms, action);
if (isnan (sc)) {
continue;
}
if (score >= sc && sc > max_score) {
selected_action = action;
max_score = sc;
}
if (rscore != NULL && i == METRIC_ACTION_REJECT) {
*rscore = sc;
}
}
if (selected_action) {
return selected_action->action;
}
return METRIC_ACTION_NOACTION;
}
static gboolean
rspamd_redis_cache_try_ucl (struct rspamd_redis_cache_ctx *cache_ctx,
const ucl_object_t *obj,
struct rspamd_config *cfg,
const gchar *symbol)
{
const ucl_object_t *elt, *relt;
elt = ucl_object_lookup_any (obj, "read_servers", "servers", NULL);
if (elt == NULL) {
return FALSE;
}
cache_ctx->read_servers = rspamd_upstreams_create (cfg->ups_ctx);
if (!rspamd_upstreams_from_ucl (cache_ctx->read_servers, elt,
REDIS_DEFAULT_PORT, NULL)) {
msg_err ("statfile %s cannot get read servers configuration",
symbol);
return FALSE;
}
relt = elt;
elt = ucl_object_lookup (obj, "write_servers");
if (elt == NULL) {
/* Use read servers as write ones */
g_assert (relt != NULL);
cache_ctx->write_servers = rspamd_upstreams_create (cfg->ups_ctx);
if (!rspamd_upstreams_from_ucl (cache_ctx->write_servers, relt,
REDIS_DEFAULT_PORT, NULL)) {
msg_err ("statfile %s cannot get write servers configuration",
symbol);
return FALSE;
}
}
else {
cache_ctx->write_servers = rspamd_upstreams_create (cfg->ups_ctx);
if (!rspamd_upstreams_from_ucl (cache_ctx->write_servers, elt,
REDIS_DEFAULT_PORT, NULL)) {
msg_err ("statfile %s cannot get write servers configuration",
symbol);
rspamd_upstreams_destroy (cache_ctx->write_servers);
cache_ctx->write_servers = NULL;
}
}
elt = ucl_object_lookup (obj, "timeout");
if (elt) {
cache_ctx->timeout = ucl_object_todouble (elt);
}
else {
cache_ctx->timeout = REDIS_DEFAULT_TIMEOUT;
}
elt = ucl_object_lookup (obj, "password");
if (elt) {
cache_ctx->password = ucl_object_tostring (elt);
}
else {
cache_ctx->password = NULL;
}
elt = ucl_object_lookup_any (obj, "db", "database", "dbname", NULL);
if (elt) {
cache_ctx->dbname = ucl_object_tostring (elt);
}
else {
cache_ctx->dbname = NULL;
}
elt = ucl_object_lookup_any (obj, "cache_key", "key", NULL);
if (elt == NULL || ucl_object_type (elt) != UCL_STRING) {
cache_ctx->redis_object = DEFAULT_REDIS_KEY;
}
else {
cache_ctx->redis_object = ucl_object_tostring (elt);
}
return TRUE;
}
static void
insert_metric_result (struct rspamd_task *task,
struct metric *metric,
const gchar *symbol,
double flag,
GList * opts,
gboolean single)
{
struct metric_result *metric_res;
struct symbol *s;
gdouble w, *gr_score = NULL;
struct rspamd_symbol_def *sdef;
struct rspamd_symbols_group *gr = NULL;
const ucl_object_t *mobj, *sobj;
metric_res = rspamd_create_metric_result (task, metric->name);
sdef = g_hash_table_lookup (metric->symbols, symbol);
if (sdef == NULL) {
w = 0.0;
}
else {
w = (*sdef->weight_ptr) * flag;
gr = sdef->gr;
if (gr != NULL) {
gr_score = g_hash_table_lookup (metric_res->sym_groups, gr);
if (gr_score == NULL) {
gr_score = rspamd_mempool_alloc (task->task_pool, sizeof (gdouble));
*gr_score = 0;
g_hash_table_insert (metric_res->sym_groups, gr, gr_score);
}
}
}
if (task->settings) {
mobj = ucl_object_lookup (task->settings, metric->name);
if (mobj) {
gdouble corr;
sobj = ucl_object_lookup (mobj, symbol);
if (sobj != NULL && ucl_object_todouble_safe (sobj, &corr)) {
msg_debug ("settings: changed weight of symbol %s from %.2f to %.2f",
symbol, w, corr);
w = corr * flag;
}
}
}
/* XXX: does not take grow factor into account */
if (gr != NULL && gr_score != NULL && gr->max_score > 0.0) {
if (*gr_score >= gr->max_score) {
msg_info_task ("maximum group score %.2f for group %s has been reached,"
" ignoring symbol %s with weight %.2f", gr->max_score,
gr->name, symbol, w);
return;
}
else if (*gr_score + w > gr->max_score) {
w = gr->max_score - *gr_score;
}
*gr_score += w;
}
/* Add metric score */
if ((s = g_hash_table_lookup (metric_res->symbols, symbol)) != NULL) {
if (sdef && (sdef->flags & RSPAMD_SYMBOL_FLAG_ONESHOT)) {
/*
* For one shot symbols we do not need to add them again, so
* we just force single behaviour here
*/
single = TRUE;
}
if (s->options && opts && opts != s->options) {
/* Append new options */
s->options = g_list_concat (s->options, g_list_copy (opts));
/*
* Note that there is no need to add new destructor of GList as elements of appended
* GList are used directly, so just free initial GList
*/
}
else if (opts) {
s->options = g_list_copy (opts);
rspamd_mempool_add_destructor (task->task_pool,
(rspamd_mempool_destruct_t) g_list_free, s->options);
}
if (!single) {
/* Handle grow factor */
if (metric_res->grow_factor && w > 0) {
w *= metric_res->grow_factor;
metric_res->grow_factor *= metric->grow_factor;
}
s->score += w;
metric_res->score += w;
}
else {
if (fabs (s->score) < fabs (w)) {
/* Replace less weight with a bigger one */
metric_res->score = metric_res->score - s->score + w;
s->score = w;
}
}
}
else {
s = rspamd_mempool_alloc (task->task_pool, sizeof (struct symbol));
/* Handle grow factor */
if (metric_res->grow_factor && w > 0) {
w *= metric_res->grow_factor;
metric_res->grow_factor *= metric->grow_factor;
}
else if (w > 0) {
metric_res->grow_factor = metric->grow_factor;
}
s->score = w;
s->name = symbol;
s->def = sdef;
metric_res->score += w;
if (opts) {
s->options = g_list_copy (opts);
rspamd_mempool_add_destructor (task->task_pool,
(rspamd_mempool_destruct_t) g_list_free, s->options);
}
else {
s->options = NULL;
}
g_hash_table_insert (metric_res->symbols, (gpointer) symbol, s);
}
msg_debug ("symbol %s, score %.2f, metric %s, factor: %f",
symbol,
s->score,
metric->name,
w);
}
/*
* Validate object
*/
static bool
ucl_schema_validate_object (const ucl_object_t *schema,
const ucl_object_t *obj, struct ucl_schema_error *err,
const ucl_object_t *root,
ucl_object_t *ext_ref)
{
const ucl_object_t *elt, *prop, *found, *additional_schema = NULL,
*required = NULL, *pat, *pelt;
ucl_object_iter_t iter = NULL, piter = NULL;
bool ret = true, allow_additional = true;
int64_t minmax;
while (ret && (elt = ucl_object_iterate (schema, &iter, true)) != NULL) {
if (elt->type == UCL_OBJECT &&
strcmp (ucl_object_key (elt), "properties") == 0) {
piter = NULL;
while (ret && (prop = ucl_object_iterate (elt, &piter, true)) != NULL) {
found = ucl_object_lookup (obj, ucl_object_key (prop));
if (found) {
ret = ucl_schema_validate (prop, found, true, err, root,
ext_ref);
}
}
}
else if (strcmp (ucl_object_key (elt), "additionalProperties") == 0) {
if (elt->type == UCL_BOOLEAN) {
if (!ucl_object_toboolean (elt)) {
/* Deny additional fields completely */
allow_additional = false;
}
}
else if (elt->type == UCL_OBJECT) {
/* Define validator for additional fields */
additional_schema = elt;
}
else {
ucl_schema_create_error (err, UCL_SCHEMA_INVALID_SCHEMA, elt,
"additionalProperties attribute is invalid in schema");
ret = false;
break;
}
}
else if (strcmp (ucl_object_key (elt), "required") == 0) {
if (elt->type == UCL_ARRAY) {
required = elt;
}
else {
ucl_schema_create_error (err, UCL_SCHEMA_INVALID_SCHEMA, elt,
"required attribute is invalid in schema");
ret = false;
break;
}
}
else if (strcmp (ucl_object_key (elt), "minProperties") == 0
&& ucl_object_toint_safe (elt, &minmax)) {
if (obj->len < minmax) {
ucl_schema_create_error (err, UCL_SCHEMA_CONSTRAINT, obj,
"object has not enough properties: %u, minimum is: %u",
obj->len, (unsigned)minmax);
ret = false;
break;
}
}
else if (strcmp (ucl_object_key (elt), "maxProperties") == 0
&& ucl_object_toint_safe (elt, &minmax)) {
if (obj->len > minmax) {
ucl_schema_create_error (err, UCL_SCHEMA_CONSTRAINT, obj,
"object has too many properties: %u, maximum is: %u",
obj->len, (unsigned)minmax);
ret = false;
break;
}
}
else if (strcmp (ucl_object_key (elt), "patternProperties") == 0) {
piter = NULL;
while (ret && (prop = ucl_object_iterate (elt, &piter, true)) != NULL) {
found = ucl_schema_test_pattern (obj, ucl_object_key (prop));
if (found) {
ret = ucl_schema_validate (prop, found, true, err, root,
ext_ref);
}
}
}
else if (elt->type == UCL_OBJECT &&
strcmp (ucl_object_key (elt), "dependencies") == 0) {
ret = ucl_schema_validate_dependencies (elt, obj, err, root,
ext_ref);
}
}
if (ret) {
/* Additional properties */
if (!allow_additional || additional_schema != NULL) {
/* Check if we have exactly the same properties in schema and object */
iter = NULL;
prop = ucl_object_lookup (schema, "properties");
while ((elt = ucl_object_iterate (obj, &iter, true)) != NULL) {
found = ucl_object_lookup (prop, ucl_object_key (elt));
if (found == NULL) {
/* Try patternProperties */
piter = NULL;
pat = ucl_object_lookup (schema, "patternProperties");
while ((pelt = ucl_object_iterate (pat, &piter, true)) != NULL) {
found = ucl_schema_test_pattern (obj, ucl_object_key (pelt));
if (found != NULL) {
break;
}
}
}
if (found == NULL) {
if (!allow_additional) {
ucl_schema_create_error (err, UCL_SCHEMA_CONSTRAINT, obj,
"object has non-allowed property %s",
ucl_object_key (elt));
ret = false;
break;
}
else if (additional_schema != NULL) {
if (!ucl_schema_validate (additional_schema, elt,
true, err, root, ext_ref)) {
ret = false;
break;
}
}
}
}
}
/* Required properties */
if (required != NULL) {
iter = NULL;
while ((elt = ucl_object_iterate (required, &iter, true)) != NULL) {
if (ucl_object_lookup (obj, ucl_object_tostring (elt)) == NULL) {
ucl_schema_create_error (err, UCL_SCHEMA_MISSING_PROPERTY, obj,
"object has missing property %s",
ucl_object_tostring (elt));
ret = false;
break;
}
}
}
}
return ret;
}
static struct rspamd_osb_tokenizer_config *
rspamd_tokenizer_osb_config_from_ucl (rspamd_mempool_t * pool,
const ucl_object_t *obj)
{
const ucl_object_t *elt;
struct rspamd_osb_tokenizer_config *cf, *def;
guchar *key = NULL;
gsize keylen;
if (pool != NULL) {
cf = rspamd_mempool_alloc0 (pool, sizeof (*cf));
}
else {
cf = g_malloc0 (sizeof (*cf));
}
/* Use default config */
def = rspamd_tokenizer_osb_default_config ();
memcpy (cf, def, sizeof (*cf));
elt = ucl_object_lookup (obj, "hash");
if (elt != NULL && ucl_object_type (elt) == UCL_STRING) {
if (g_ascii_strncasecmp (ucl_object_tostring (elt), "xxh", 3)
== 0) {
cf->ht = RSPAMD_OSB_HASH_XXHASH;
elt = ucl_object_lookup (obj, "seed");
if (elt != NULL && ucl_object_type (elt) == UCL_INT) {
cf->seed = ucl_object_toint (elt);
}
}
else if (g_ascii_strncasecmp (ucl_object_tostring (elt), "sip", 3)
== 0) {
cf->ht = RSPAMD_OSB_HASH_SIPHASH;
elt = ucl_object_lookup (obj, "key");
if (elt != NULL && ucl_object_type (elt) == UCL_STRING) {
key = rspamd_decode_base32 (ucl_object_tostring (elt),
0, &keylen);
if (keylen < sizeof (rspamd_sipkey_t)) {
msg_warn ("siphash key is too short: %z", keylen);
g_free (key);
}
else {
memcpy (cf->sk, key, sizeof (cf->sk));
g_free (key);
}
}
else {
msg_warn_pool ("siphash cannot be used without key");
}
}
}
else {
elt = ucl_object_lookup (obj, "compat");
if (elt != NULL && ucl_object_toboolean (elt)) {
cf->ht = RSPAMD_OSB_HASH_COMPAT;
}
}
elt = ucl_object_lookup (obj, "window");
if (elt != NULL && ucl_object_type (elt) == UCL_INT) {
cf->window_size = ucl_object_toint (elt);
if (cf->window_size > DEFAULT_FEATURE_WINDOW_SIZE * 4) {
msg_err_pool ("too large window size: %d", cf->window_size);
cf->window_size = DEFAULT_FEATURE_WINDOW_SIZE;
}
}
return cf;
}
int
main(int argc, char **argv)
{
const char *fn = NULL;
unsigned char *inbuf;
struct ucl_parser *parser;
int k, ret = 0, r = 0;
ssize_t bufsize;
ucl_object_t *obj = NULL;
const ucl_object_t *par;
FILE *in;
if (argc > 1) {
fn = argv[1];
}
if (fn != NULL) {
in = fopen (fn, "r");
if (in == NULL) {
exit (-errno);
}
}
else {
in = stdin;
}
parser = ucl_parser_new (0);
inbuf = malloc (BUFSIZ);
bufsize = BUFSIZ;
r = 0;
while (!feof (in) && !ferror (in)) {
if (r == bufsize) {
inbuf = realloc (inbuf, bufsize * 2);
bufsize *= 2;
if (inbuf == NULL) {
perror ("realloc");
exit (EXIT_FAILURE);
}
}
r += fread (inbuf + r, 1, bufsize - r, in);
}
if (ferror (in)) {
fprintf (stderr, "Failed to read the input file.\n");
exit (EXIT_FAILURE);
}
ucl_parser_add_chunk (parser, inbuf, r);
fclose (in);
if (ucl_parser_get_error(parser)) {
printf ("Error occured: %s\n", ucl_parser_get_error(parser));
ret = 1;
goto end;
}
obj = ucl_parser_get_object (parser);
if (ucl_parser_get_error (parser)) {
printf ("Error occured: %s\n", ucl_parser_get_error(parser));
ret = 1;
goto end;
}
if (argc > 2) {
for (k = 2; k < argc; k++) {
printf ("search for \"%s\"... ", argv[k]);
par = ucl_object_lookup (obj, argv[k]);
printf ("%sfound\n", (par == NULL )?"not ":"");
ucl_obj_dump (par, 0);
}
}
else {
ucl_obj_dump (obj, 0);
}
end:
if (parser != NULL) {
ucl_parser_free (parser);
}
if (obj != NULL) {
ucl_object_unref (obj);
}
return ret;
}
static gboolean
rspamd_fuzzy_backend_redis_try_ucl (struct rspamd_fuzzy_backend_redis *backend,
const ucl_object_t *obj,
struct rspamd_config *cfg)
{
const ucl_object_t *elt, *relt;
elt = ucl_object_lookup_any (obj, "read_servers", "servers", NULL);
if (elt == NULL) {
return FALSE;
}
backend->read_servers = rspamd_upstreams_create (cfg->ups_ctx);
if (!rspamd_upstreams_from_ucl (backend->read_servers, elt,
REDIS_DEFAULT_PORT, NULL)) {
msg_err_config ("cannot get read servers configuration");
return FALSE;
}
relt = elt;
elt = ucl_object_lookup (obj, "write_servers");
if (elt == NULL) {
/* Use read servers as write ones */
g_assert (relt != NULL);
backend->write_servers = rspamd_upstreams_create (cfg->ups_ctx);
if (!rspamd_upstreams_from_ucl (backend->write_servers, relt,
REDIS_DEFAULT_PORT, NULL)) {
msg_err_config ("cannot get write servers configuration");
return FALSE;
}
}
else {
backend->write_servers = rspamd_upstreams_create (cfg->ups_ctx);
if (!rspamd_upstreams_from_ucl (backend->write_servers, elt,
REDIS_DEFAULT_PORT, NULL)) {
msg_err_config ("cannot get write servers configuration");
rspamd_upstreams_destroy (backend->write_servers);
backend->write_servers = NULL;
}
}
elt = ucl_object_lookup (obj, "prefix");
if (elt == NULL || ucl_object_type (elt) != UCL_STRING) {
backend->redis_object = REDIS_DEFAULT_OBJECT;
}
else {
backend->redis_object = ucl_object_tostring (elt);
}
elt = ucl_object_lookup (obj, "timeout");
if (elt) {
backend->timeout = ucl_object_todouble (elt);
}
else {
backend->timeout = REDIS_DEFAULT_TIMEOUT;
}
elt = ucl_object_lookup (obj, "password");
if (elt) {
backend->password = ucl_object_tostring (elt);
}
else {
backend->password = NULL;
}
elt = ucl_object_lookup_any (obj, "db", "database", "dbname", NULL);
if (elt) {
backend->dbname = ucl_object_tostring (elt);
}
else {
backend->dbname = NULL;
}
return TRUE;
}
void*
rspamd_fuzzy_backend_init_redis (struct rspamd_fuzzy_backend *bk,
const ucl_object_t *obj, struct rspamd_config *cfg, GError **err)
{
struct rspamd_fuzzy_backend_redis *backend;
const ucl_object_t *elt;
gboolean ret = FALSE;
guchar id_hash[rspamd_cryptobox_HASHBYTES];
rspamd_cryptobox_hash_state_t st;
backend = g_slice_alloc0 (sizeof (*backend));
backend->timeout = REDIS_DEFAULT_TIMEOUT;
backend->redis_object = REDIS_DEFAULT_OBJECT;
ret = rspamd_fuzzy_backend_redis_try_ucl (backend, obj, cfg);
/* Now try global redis settings */
if (!ret) {
elt = ucl_object_lookup (cfg->rcl_obj, "redis");
if (elt) {
const ucl_object_t *specific_obj;
specific_obj = ucl_object_lookup_any (elt, "fuzzy", "fuzzy_storage",
NULL);
if (specific_obj) {
ret = rspamd_fuzzy_backend_redis_try_ucl (backend, specific_obj,
cfg);
}
else {
ret = rspamd_fuzzy_backend_redis_try_ucl (backend, elt, cfg);
}
}
}
if (!ret) {
msg_err_config ("cannot init redis backend for fuzzy storage");
g_slice_free1 (sizeof (*backend), backend);
return NULL;
}
REF_INIT_RETAIN (backend, rspamd_fuzzy_backend_redis_dtor);
backend->pool = cfg->redis_pool;
rspamd_cryptobox_hash_init (&st, NULL, 0);
rspamd_cryptobox_hash_update (&st, backend->redis_object,
strlen (backend->redis_object));
if (backend->dbname) {
rspamd_cryptobox_hash_update (&st, backend->dbname,
strlen (backend->dbname));
}
if (backend->password) {
rspamd_cryptobox_hash_update (&st, backend->password,
strlen (backend->password));
}
rspamd_cryptobox_hash_final (&st, id_hash);
backend->id = rspamd_encode_base32 (id_hash, sizeof (id_hash));
return backend;
}
struct rspamd_map*
rspamd_map_add_from_ucl (struct rspamd_config *cfg,
const ucl_object_t *obj,
const gchar *description,
map_cb_t read_callback,
map_fin_cb_t fin_callback,
void **user_data)
{
ucl_object_iter_t it = NULL;
const ucl_object_t *cur, *elt;
struct rspamd_map *map;
struct rspamd_map_backend *bk;
g_assert (obj != NULL);
if (ucl_object_type (obj) == UCL_STRING) {
/* Just a plain string */
return rspamd_map_add (cfg, ucl_object_tostring (obj), NULL,
read_callback, fin_callback, user_data);
}
map = g_slice_alloc0 (sizeof (struct rspamd_map));
map->read_callback = read_callback;
map->fin_callback = fin_callback;
map->user_data = user_data;
map->cfg = cfg;
map->id = g_random_int ();
map->locked =
rspamd_mempool_alloc0_shared (cfg->cfg_pool, sizeof (gint));
map->cache =
rspamd_mempool_alloc0_shared (cfg->cfg_pool, sizeof (*map->cache));
map->backends = g_ptr_array_new ();
map->poll_timeout = cfg->map_timeout;
if (description) {
map->description = g_strdup (description);
}
if (ucl_object_type (obj) == UCL_ARRAY) {
/* Add array of maps as multiple backends */
while ((cur = ucl_object_iterate (obj, &it, true)) != NULL) {
if (ucl_object_type (cur) == UCL_STRING) {
bk = rspamd_map_parse_backend (cfg, ucl_object_tostring (cur));
if (bk != NULL) {
g_ptr_array_add (map->backends, bk);
if (!map->name) {
map->name = g_strdup (ucl_object_tostring (cur));
}
}
}
else {
msg_err_config ("bad map element type: %s",
ucl_object_type_to_string (ucl_object_type (cur)));
}
}
if (map->backends->len == 0) {
msg_err_config ("map has no urls to be loaded");
goto err;
}
}
else if (ucl_object_type (obj) == UCL_OBJECT) {
elt = ucl_object_lookup (obj, "name");
if (elt && ucl_object_type (elt) == UCL_STRING) {
map->name = g_strdup (ucl_object_tostring (elt));
}
elt = ucl_object_lookup (obj, "description");
if (elt && ucl_object_type (elt) == UCL_STRING) {
if (map->description) {
g_free (map->description);
}
map->description = g_strdup (ucl_object_tostring (elt));
}
elt = ucl_object_lookup_any (obj, "timeout", "poll", "poll_time",
"watch_interval", NULL);
if (elt) {
map->poll_timeout = ucl_object_todouble (elt);
}
elt = ucl_object_lookup_any (obj, "upstreams", "url", "urls", NULL);
if (elt == NULL) {
msg_err_config ("map has no urls to be loaded");
goto err;
}
if (ucl_object_type (obj) == UCL_ARRAY) {
/* Add array of maps as multiple backends */
while ((cur = ucl_object_iterate (elt, &it, true)) != NULL) {
if (ucl_object_type (cur) == UCL_STRING) {
bk = rspamd_map_parse_backend (cfg, ucl_object_tostring (cur));
if (bk != NULL) {
g_ptr_array_add (map->backends, bk);
if (!map->name) {
map->name = g_strdup (ucl_object_tostring (cur));
}
}
}
else {
msg_err_config ("bad map element type: %s",
ucl_object_type_to_string (ucl_object_type (cur)));
goto err;
}
}
if (map->backends->len == 0) {
msg_err_config ("map has no urls to be loaded");
goto err;
}
}
else if (ucl_object_type (elt) == UCL_STRING) {
bk = rspamd_map_parse_backend (cfg, ucl_object_tostring (elt));
if (bk != NULL) {
g_ptr_array_add (map->backends, bk);
if (!map->name) {
map->name = g_strdup (ucl_object_tostring (cur));
}
}
}
if (map->backends->len == 0) {
msg_err_config ("map has no urls to be loaded");
goto err;
}
}
rspamd_map_calculate_hash (map);
msg_info_map ("added map from ucl");
cfg->maps = g_list_prepend (cfg->maps, map);
return map;
err:
g_ptr_array_free (map->backends, TRUE);
g_free (map->name);
g_free (map->description);
g_slice_free1 (sizeof (*map), map);
return NULL;
}
static bool
ucl_schema_validate_number (const ucl_object_t *schema,
const ucl_object_t *obj, struct ucl_schema_error *err)
{
const ucl_object_t *elt, *test;
ucl_object_iter_t iter = NULL;
bool ret = true, exclusive = false;
double constraint, val;
const double alpha = 1e-16;
while (ret && (elt = ucl_object_iterate (schema, &iter, true)) != NULL) {
if ((elt->type == UCL_FLOAT || elt->type == UCL_INT) &&
strcmp (ucl_object_key (elt), "multipleOf") == 0) {
constraint = ucl_object_todouble (elt);
if (constraint <= 0) {
ucl_schema_create_error (err, UCL_SCHEMA_INVALID_SCHEMA, elt,
"multipleOf must be greater than zero");
ret = false;
break;
}
val = ucl_object_todouble (obj);
if (fabs (remainder (val, constraint)) > alpha) {
ucl_schema_create_error (err, UCL_SCHEMA_CONSTRAINT, obj,
"number %.4f is not multiple of %.4f, remainder is %.7f",
val, constraint);
ret = false;
break;
}
}
else if ((elt->type == UCL_FLOAT || elt->type == UCL_INT) &&
strcmp (ucl_object_key (elt), "maximum") == 0) {
constraint = ucl_object_todouble (elt);
test = ucl_object_lookup (schema, "exclusiveMaximum");
if (test && test->type == UCL_BOOLEAN) {
exclusive = ucl_object_toboolean (test);
}
val = ucl_object_todouble (obj);
if (val > constraint || (exclusive && val >= constraint)) {
ucl_schema_create_error (err, UCL_SCHEMA_CONSTRAINT, obj,
"number is too big: %.3f, maximum is: %.3f",
val, constraint);
ret = false;
break;
}
}
else if ((elt->type == UCL_FLOAT || elt->type == UCL_INT) &&
strcmp (ucl_object_key (elt), "minimum") == 0) {
constraint = ucl_object_todouble (elt);
test = ucl_object_lookup (schema, "exclusiveMinimum");
if (test && test->type == UCL_BOOLEAN) {
exclusive = ucl_object_toboolean (test);
}
val = ucl_object_todouble (obj);
if (val < constraint || (exclusive && val <= constraint)) {
ucl_schema_create_error (err, UCL_SCHEMA_CONSTRAINT, obj,
"number is too small: %.3f, minimum is: %.3f",
val, constraint);
ret = false;
break;
}
}
}
return ret;
}
gpointer
rspamd_stat_cache_redis_init (struct rspamd_stat_ctx *ctx,
struct rspamd_config *cfg,
struct rspamd_statfile *st,
const ucl_object_t *cf)
{
struct rspamd_redis_cache_ctx *cache_ctx;
struct rspamd_statfile_config *stf = st->stcf;
const ucl_object_t *obj;
gboolean ret = FALSE;
cache_ctx = g_slice_alloc0 (sizeof (*cache_ctx));
/* First search in backend configuration */
obj = ucl_object_lookup (st->classifier->cfg->opts, "backend");
if (obj != NULL && ucl_object_type (obj) == UCL_OBJECT) {
ret = rspamd_redis_cache_try_ucl (cache_ctx, obj, cfg, stf->symbol);
}
/* Now try statfiles config */
if (!ret) {
ret = rspamd_redis_cache_try_ucl (cache_ctx, stf->opts, cfg, stf->symbol);
}
/* Now try classifier config */
if (!ret) {
ret = rspamd_redis_cache_try_ucl (cache_ctx, st->classifier->cfg->opts, cfg,
stf->symbol);
}
/* Now try global redis settings */
if (!ret) {
obj = ucl_object_lookup (cfg->rcl_obj, "redis");
if (obj) {
const ucl_object_t *specific_obj;
specific_obj = ucl_object_lookup (obj, "statistics");
if (specific_obj) {
ret = rspamd_redis_cache_try_ucl (cache_ctx, specific_obj, cfg,
stf->symbol);
}
else {
ret = rspamd_redis_cache_try_ucl (cache_ctx, obj, cfg,
stf->symbol);
}
}
}
if (!ret) {
msg_err_config ("cannot init redis cache for %s", stf->symbol);
g_slice_free1 (sizeof (*cache_ctx), cache_ctx);
return NULL;
}
cache_ctx->stcf = stf;
return (gpointer)cache_ctx;
}
static gboolean
rspamd_symbols_cache_load_items (struct symbols_cache *cache, const gchar *name)
{
struct rspamd_symbols_cache_header *hdr;
struct stat st;
struct ucl_parser *parser;
ucl_object_t *top;
const ucl_object_t *cur, *elt;
ucl_object_iter_t it;
struct cache_item *item, *parent;
const guchar *p;
gint fd;
gpointer map;
fd = open (name, O_RDONLY);
if (fd == -1) {
msg_info_cache ("cannot open file %s, error %d, %s", name,
errno, strerror (errno));
return FALSE;
}
if (fstat (fd, &st) == -1) {
close (fd);
msg_info_cache ("cannot stat file %s, error %d, %s", name,
errno, strerror (errno));
return FALSE;
}
if (st.st_size < (gint)sizeof (*hdr)) {
close (fd);
errno = EINVAL;
msg_info_cache ("cannot use file %s, error %d, %s", name,
errno, strerror (errno));
return FALSE;
}
map = mmap (NULL, st.st_size, PROT_READ, MAP_SHARED, fd, 0);
if (map == MAP_FAILED) {
close (fd);
msg_info_cache ("cannot mmap file %s, error %d, %s", name,
errno, strerror (errno));
return FALSE;
}
close (fd);
hdr = map;
if (memcmp (hdr->magic, rspamd_symbols_cache_magic,
sizeof (rspamd_symbols_cache_magic)) != 0) {
msg_info_cache ("cannot use file %s, bad magic", name);
munmap (map, st.st_size);
return FALSE;
}
parser = ucl_parser_new (0);
p = (const guchar *)(hdr + 1);
if (!ucl_parser_add_chunk (parser, p, st.st_size - sizeof (*hdr))) {
msg_info_cache ("cannot use file %s, cannot parse: %s", name,
ucl_parser_get_error (parser));
munmap (map, st.st_size);
ucl_parser_free (parser);
return FALSE;
}
top = ucl_parser_get_object (parser);
munmap (map, st.st_size);
ucl_parser_free (parser);
if (top == NULL || ucl_object_type (top) != UCL_OBJECT) {
msg_info_cache ("cannot use file %s, bad object", name);
ucl_object_unref (top);
return FALSE;
}
it = ucl_object_iterate_new (top);
while ((cur = ucl_object_iterate_safe (it, true))) {
item = g_hash_table_lookup (cache->items_by_symbol, ucl_object_key (cur));
if (item) {
/* Copy saved info */
/*
* XXX: don't save or load weight, it should be obtained from the
* metric
*/
#if 0
elt = ucl_object_lookup (cur, "weight");
if (elt) {
w = ucl_object_todouble (elt);
if (w != 0) {
item->weight = w;
}
}
#endif
elt = ucl_object_lookup (cur, "time");
if (elt) {
item->avg_time = ucl_object_todouble (elt);
}
elt = ucl_object_lookup (cur, "count");
if (elt) {
item->avg_counter = ucl_object_toint (elt);
}
elt = ucl_object_lookup (cur, "frequency");
if (elt) {
item->frequency = ucl_object_toint (elt);
}
if ((item->type & SYMBOL_TYPE_VIRTUAL) && item->parent != -1) {
g_assert (item->parent < (gint)cache->items_by_id->len);
parent = g_ptr_array_index (cache->items_by_id, item->parent);
if (parent->weight < item->weight) {
parent->weight = item->weight;
}
/*
* We maintain avg_time for virtual symbols equal to the
* parent item avg_time
*/
parent->avg_time = item->avg_time;
parent->avg_counter = item->avg_counter;
}
cache->total_weight += fabs (item->weight);
cache->total_freq += item->frequency;
}
}
ucl_object_iterate_free (it);
ucl_object_unref (top);
return TRUE;
}
static gboolean
rspamd_redis_try_ucl (struct redis_stat_ctx *backend,
const ucl_object_t *obj,
struct rspamd_config *cfg,
const gchar *symbol)
{
const ucl_object_t *elt, *relt, *users_enabled;
const gchar *lua_script;
elt = ucl_object_lookup_any (obj, "read_servers", "servers", NULL);
if (elt == NULL) {
return FALSE;
}
backend->read_servers = rspamd_upstreams_create (cfg->ups_ctx);
if (!rspamd_upstreams_from_ucl (backend->read_servers, elt,
REDIS_DEFAULT_PORT, NULL)) {
msg_err ("statfile %s cannot get read servers configuration",
symbol);
return FALSE;
}
relt = elt;
elt = ucl_object_lookup (obj, "write_servers");
if (elt == NULL) {
/* Use read servers as write ones */
g_assert (relt != NULL);
backend->write_servers = rspamd_upstreams_create (cfg->ups_ctx);
if (!rspamd_upstreams_from_ucl (backend->write_servers, relt,
REDIS_DEFAULT_PORT, NULL)) {
msg_err ("statfile %s cannot get write servers configuration",
symbol);
return FALSE;
}
}
else {
backend->write_servers = rspamd_upstreams_create (cfg->ups_ctx);
if (!rspamd_upstreams_from_ucl (backend->write_servers, elt,
REDIS_DEFAULT_PORT, NULL)) {
msg_err ("statfile %s cannot get write servers configuration",
symbol);
rspamd_upstreams_destroy (backend->write_servers);
backend->write_servers = NULL;
}
}
elt = ucl_object_lookup (obj, "prefix");
if (elt == NULL || ucl_object_type (elt) != UCL_STRING) {
/* Default non-users statistics */
backend->redis_object = REDIS_DEFAULT_OBJECT;
/*
* Make redis backend compatible with sqlite3 backend in users settings
*/
users_enabled = ucl_object_lookup_any (obj, "per_user",
"users_enabled", NULL);
if (users_enabled != NULL) {
if (ucl_object_type (users_enabled) == UCL_BOOLEAN) {
backend->enable_users = ucl_object_toboolean (users_enabled);
backend->cbref_user = -1;
if (backend->enable_users) {
backend->redis_object = REDIS_DEFAULT_USERS_OBJECT;
}
}
else if (ucl_object_type (users_enabled) == UCL_STRING) {
lua_script = ucl_object_tostring (users_enabled);
if (luaL_dostring (cfg->lua_state, lua_script) != 0) {
msg_err_config ("cannot execute lua script for users "
"extraction: %s", lua_tostring (cfg->lua_state, -1));
}
else {
if (lua_type (cfg->lua_state, -1) == LUA_TFUNCTION) {
backend->enable_users = TRUE;
backend->cbref_user = luaL_ref (cfg->lua_state,
LUA_REGISTRYINDEX);
}
else {
msg_err_config ("lua script must return "
"function(task) and not %s",
lua_typename (cfg->lua_state, lua_type (
cfg->lua_state, -1)));
}
}
}
}
else {
backend->enable_users = FALSE;
}
}
else {
/* XXX: sanity check */
backend->redis_object = ucl_object_tostring (elt);
}
elt = ucl_object_lookup (obj, "timeout");
if (elt) {
backend->timeout = ucl_object_todouble (elt);
}
else {
backend->timeout = REDIS_DEFAULT_TIMEOUT;
}
elt = ucl_object_lookup (obj, "password");
if (elt) {
backend->password = ucl_object_tostring (elt);
}
else {
backend->password = NULL;
}
elt = ucl_object_lookup_any (obj, "db", "database", "dbname", NULL);
if (elt) {
backend->dbname = ucl_object_tostring (elt);
}
else {
backend->dbname = NULL;
}
return TRUE;
}
int
main (int argc, char **argv)
{
ucl_object_t *obj, *cur, *ar, *ar1, *ref, *test_obj, *comments;
ucl_object_iter_t it;
const ucl_object_t *found, *it_obj, *test;
struct ucl_emitter_functions *fn;
FILE *out;
unsigned char *emitted;
const char *fname_out = NULL;
struct ucl_parser *parser;
int ret = 0;
switch (argc) {
case 2:
fname_out = argv[1];
break;
}
if (fname_out != NULL) {
out = fopen (fname_out, "w");
if (out == NULL) {
exit (-errno);
}
}
else {
out = stdout;
}
obj = ucl_object_typed_new (UCL_OBJECT);
/* Keys replacing */
cur = ucl_object_fromstring_common ("value1", 0, UCL_STRING_TRIM);
ucl_object_insert_key (obj, cur, "key0", 0, false);
cur = ucl_object_fromdouble (0.1);
assert (ucl_object_replace_key (obj, cur, "key0", 0, false));
/* Create some strings */
cur = ucl_object_fromstring_common (" test string ", 0, UCL_STRING_TRIM);
ucl_object_insert_key (obj, cur, "key1", 0, false);
cur = ucl_object_fromstring_common (" test \nstring\n\r\n\b\t\f\\\" ", 0,
UCL_STRING_TRIM | UCL_STRING_ESCAPE);
ucl_object_insert_key (obj, cur, "key2", 0, false);
cur = ucl_object_fromstring_common (" test string \n", 0, 0);
ucl_object_insert_key (obj, cur, "key3", 0, false);
/* Array of numbers */
ar = ucl_object_typed_new (UCL_ARRAY);
cur = ucl_object_fromint (10);
ucl_array_append (ar, cur);
assert (ucl_array_index_of (ar, cur) == 0);
cur = ucl_object_fromdouble (10.1);
ucl_array_append (ar, cur);
assert (ucl_array_index_of (ar, cur) == 1);
cur = ucl_object_fromdouble (9.999);
ucl_array_prepend (ar, cur);
assert (ucl_array_index_of (ar, cur) == 0);
ar1 = ucl_object_copy (ar);
cur = ucl_object_fromstring ("abc");
ucl_array_prepend (ar1, cur);
cur = ucl_object_fromstring ("cde");
ucl_array_prepend (ar1, cur);
cur = ucl_object_fromstring ("аАаБаВ"); /* UTF8 */
ucl_array_prepend (ar1, cur);
cur = ucl_object_fromstring ("ааБаВ"); /* UTF8 */
ucl_array_prepend (ar1, cur);
/*
* This is ususally broken or fragile as utf collate is far from perfect
cur = ucl_object_fromstring ("баБаВ");
ucl_array_prepend (ar1, cur);
cur = ucl_object_fromstring ("ааБаВ"); // hello to @bapt
ucl_array_prepend (ar1, cur);
*/
cur = ucl_object_fromstring ("№"); /* everybody likes emoji in the code */
ucl_array_prepend (ar1, cur);
ucl_object_array_sort (ar1, ucl_object_compare_qsort);
/* Removing from an array */
cur = ucl_object_fromdouble (1.0);
ucl_array_append (ar, cur);
cur = ucl_array_delete (ar, cur);
assert (ucl_object_todouble (cur) == 1.0);
ucl_object_unref (cur);
cur = ucl_object_fromdouble (2.0);
ucl_array_append (ar, cur);
cur = ucl_array_pop_last (ar);
assert (ucl_object_todouble (cur) == 2.0);
ucl_object_unref (cur);
cur = ucl_object_fromdouble (3.0);
ucl_array_prepend (ar, cur);
cur = ucl_array_pop_first (ar);
assert (ucl_object_todouble (cur) == 3.0);
ucl_object_unref (cur);
ucl_object_insert_key (obj, ar, "key4", 0, false);
cur = ucl_object_frombool (true);
/* Ref object to test refcounts */
ref = ucl_object_ref (cur);
ucl_object_insert_key (obj, cur, "key4", 0, false);
/* Empty strings */
cur = ucl_object_fromstring_common (" ", 0, UCL_STRING_TRIM);
ucl_object_insert_key (obj, cur, "key5", 0, false);
cur = ucl_object_fromstring_common ("", 0, UCL_STRING_ESCAPE);
ucl_object_insert_key (obj, cur, "key6", 0, false);
cur = ucl_object_fromstring_common (" \n", 0, UCL_STRING_ESCAPE);
ucl_object_insert_key (obj, cur, "key7", 0, false);
/* Numbers and booleans */
cur = ucl_object_fromstring_common ("1mb", 0, UCL_STRING_ESCAPE | UCL_STRING_PARSE);
ucl_object_insert_key (obj, cur, "key8", 0, false);
cur = ucl_object_fromstring_common ("3.14", 0, UCL_STRING_PARSE);
ucl_object_insert_key (obj, cur, "key9", 0, false);
cur = ucl_object_fromstring_common ("true", 0, UCL_STRING_PARSE);
ucl_object_insert_key (obj, cur, "key10", 0, false);
cur = ucl_object_fromstring_common (" off ", 0, UCL_STRING_PARSE | UCL_STRING_TRIM);
ucl_object_insert_key (obj, cur, "key11", 0, false);
cur = ucl_object_fromstring_common ("*****@*****.**", 0, UCL_STRING_PARSE_INT);
ucl_object_insert_key (obj, cur, "key12", 0, false);
cur = ucl_object_fromstring_common ("#test", 0, UCL_STRING_PARSE_INT);
ucl_object_insert_key (obj, cur, "key13", 0, false);
cur = ucl_object_frombool (true);
ucl_object_insert_key (obj, cur, "k=3", 0, false);
ucl_object_insert_key (obj, ar1, "key14", 0, false);
cur = ucl_object_new_userdata (ud_dtor, ud_emit, NULL);
ucl_object_insert_key (obj, cur, "key15", 0, false);
/* More tests for keys */
cur = ucl_object_fromlstring ("test", 3);
ucl_object_insert_key (obj, cur, "key16", 0, false);
test = ucl_object_lookup_any (obj, "key100", "key200", "key300", "key16", NULL);
assert (test == cur);
test = ucl_object_lookup_len (obj, "key160", 5);
assert (test == cur);
cur = ucl_object_pop_key (obj, "key16");
assert (test == cur);
test = ucl_object_pop_key (obj, "key16");
assert (test == NULL);
test = ucl_object_lookup_len (obj, "key160", 5);
assert (test == NULL);
/* Objects merging tests */
test_obj = ucl_object_new_full (UCL_OBJECT, 2);
ucl_object_insert_key (test_obj, cur, "key16", 0, true);
ucl_object_merge (obj, test_obj, true);
ucl_object_unref (test_obj);
/* Array merging test */
test_obj = ucl_object_new_full (UCL_ARRAY, 3);
ucl_array_append (test_obj, ucl_object_fromstring ("test"));
ucl_array_merge (test_obj, ar1, false);
ucl_object_insert_key (obj, test_obj, "key17", 0, true);
/* Object deletion */
cur = ucl_object_fromstring ("test");
ucl_object_insert_key (obj, cur, "key18", 0, true);
assert (ucl_object_delete_key (obj, "key18"));
assert (!ucl_object_delete_key (obj, "key18"));
cur = ucl_object_fromlstring ("test", 4);
ucl_object_insert_key (obj, cur, "key18\0\0", 7, true);
assert (ucl_object_lookup_len (obj, "key18\0\0", 7) == cur);
assert (ucl_object_lookup (obj, "key18") == NULL);
assert (ucl_object_lookup_len (obj, "key18\0\1", 7) == NULL);
assert (ucl_object_delete_keyl (obj, "key18\0\0", 7));
/* Comments */
comments = ucl_object_typed_new (UCL_OBJECT);
found = ucl_object_lookup (obj, "key17");
test = ucl_object_lookup (obj, "key16");
ucl_comments_add (comments, found, "# test comment");
assert (ucl_comments_find (comments, found) != NULL);
assert (ucl_comments_find (comments, test) == NULL);
ucl_comments_move (comments, found, test);
assert (ucl_comments_find (comments, found) == NULL);
assert (ucl_comments_find (comments, test) != NULL);
/* Array replace */
ar1 = ucl_object_typed_new (UCL_ARRAY);
cur = ucl_object_fromstring ("test");
cur = ucl_elt_append (cur, ucl_object_fromstring ("test1"));
ucl_array_append (ar1, cur);
test = ucl_array_replace_index (ar1, ucl_object_fromstring ("test2"), 0);
assert (test == cur);
/* Try to find using path */
/* Should exist */
found = ucl_object_lookup_path (obj, "key4.1");
assert (found != NULL && ucl_object_toint (found) == 10);
/* . should be ignored */
found = ucl_object_lookup_path (obj, ".key4.1");
assert (found != NULL && ucl_object_toint (found) == 10);
/* moar dots... */
found = ucl_object_lookup_path (obj, ".key4........1...");
assert (found != NULL && ucl_object_toint (found) == 10);
/* No such index */
found = ucl_object_lookup_path (obj, ".key4.3");
assert (found == NULL);
/* No such key */
found = ucl_object_lookup_path (obj, "key9..key1");
assert (found == NULL);
/* Test iteration */
it = ucl_object_iterate_new (obj);
it_obj = ucl_object_iterate_safe (it, true);
/* key0 = 0.1 */
assert (ucl_object_type (it_obj) == UCL_FLOAT);
it_obj = ucl_object_iterate_safe (it, true);
/* key1 = "" */
assert (ucl_object_type (it_obj) == UCL_STRING);
it_obj = ucl_object_iterate_safe (it, true);
/* key2 = "" */
assert (ucl_object_type (it_obj) == UCL_STRING);
it_obj = ucl_object_iterate_safe (it, true);
/* key3 = "" */
assert (ucl_object_type (it_obj) == UCL_STRING);
it_obj = ucl_object_iterate_safe (it, true);
/* key4 = ([float, int, float], boolean) */
ucl_object_iterate_reset (it, it_obj);
it_obj = ucl_object_iterate_safe (it, true);
assert (ucl_object_type (it_obj) == UCL_FLOAT);
it_obj = ucl_object_iterate_safe (it, true);
assert (ucl_object_type (it_obj) == UCL_INT);
it_obj = ucl_object_iterate_safe (it, true);
assert (ucl_object_type (it_obj) == UCL_FLOAT);
it_obj = ucl_object_iterate_safe (it, true);
assert (ucl_object_type (it_obj) == UCL_BOOLEAN);
ucl_object_iterate_free (it);
fn = ucl_object_emit_memory_funcs (&emitted);
assert (ucl_object_emit_full (obj, UCL_EMIT_CONFIG, fn, comments));
fprintf (out, "%s\n", emitted);
ucl_object_emit_funcs_free (fn);
ucl_object_unref (obj);
ucl_object_unref (comments);
parser = ucl_parser_new (UCL_PARSER_NO_IMPLICIT_ARRAYS);
if (ucl_parser_add_chunk_full (parser, emitted, strlen (emitted),
3, UCL_DUPLICATE_ERROR, UCL_PARSE_UCL)) {
/* Should fail due to duplicate */
assert (0);
}
else {
assert (ucl_parser_get_error (parser) != NULL);
ucl_parser_clear_error (parser);
ucl_parser_free (parser);
parser = ucl_parser_new (0);
ucl_parser_add_chunk_full (parser, emitted, strlen (emitted),
3, UCL_DUPLICATE_MERGE, UCL_PARSE_UCL);
}
assert (ucl_parser_get_column (parser) == 0);
assert (ucl_parser_get_linenum (parser) != 0);
ucl_parser_clear_error (parser);
assert (ucl_parser_get_error_code (parser) == 0);
obj = ucl_parser_get_object (parser);
ucl_parser_free (parser);
ucl_object_free (obj);
if (emitted != NULL) {
free (emitted);
}
fclose (out);
/* Ref should still be accessible */
ref->value.iv = 100500;
ucl_object_unref (ref);
return ret;
}
