/* * Check dependencies for an object */ static bool ucl_schema_validate_dependencies (const ucl_object_t *deps, const ucl_object_t *obj, struct ucl_schema_error *err, const ucl_object_t *root, ucl_object_t *ext_ref) { const ucl_object_t *elt, *cur, *cur_dep; ucl_object_iter_t iter = NULL, piter; bool ret = true; while (ret && (cur = ucl_object_iterate (deps, &iter, true)) != NULL) { elt = ucl_object_lookup (obj, ucl_object_key (cur)); if (elt != NULL) { /* Need to check dependencies */ if (cur->type == UCL_ARRAY) { piter = NULL; while (ret && (cur_dep = ucl_object_iterate (cur, &piter, true)) != NULL) { if (ucl_object_lookup (obj, ucl_object_tostring (cur_dep)) == NULL) { ucl_schema_create_error (err, UCL_SCHEMA_MISSING_DEPENDENCY, elt, "dependency %s is missing for key %s", ucl_object_tostring (cur_dep), ucl_object_key (cur)); ret = false; break; } } } else if (cur->type == UCL_OBJECT) { ret = ucl_schema_validate (cur, obj, true, err, root, ext_ref); } } } return ret; }
static gboolean rspamd_symbols_cache_process_settings (struct rspamd_task *task, struct symbols_cache *cache) { const ucl_object_t *wl, *cur, *disabled; struct metric *def; struct rspamd_symbols_group *gr; GHashTableIter gr_it; ucl_object_iter_t it = NULL; gpointer k, v; wl = ucl_object_lookup (task->settings, "whitelist"); if (wl != NULL) { msg_info_task ("<%s> is whitelisted", task->message_id); task->flags |= RSPAMD_TASK_FLAG_SKIP; return TRUE; } disabled = ucl_object_lookup (task->settings, "symbols_disabled"); if (disabled) { it = NULL; while ((cur = ucl_iterate_object (disabled, &it, true)) != NULL) { rspamd_symbols_cache_disable_symbol (task, cache, ucl_object_tostring (cur)); } } /* Disable groups of symbols */ disabled = ucl_object_lookup (task->settings, "groups_disabled"); def = g_hash_table_lookup (task->cfg->metrics, DEFAULT_METRIC); if (def && disabled) { it = NULL; while ((cur = ucl_iterate_object (disabled, &it, true)) != NULL) { if (ucl_object_type (cur) == UCL_STRING) { gr = g_hash_table_lookup (def->groups, ucl_object_tostring (cur)); if (gr) { g_hash_table_iter_init (&gr_it, gr->symbols); while (g_hash_table_iter_next (&gr_it, &k, &v)) { rspamd_symbols_cache_disable_symbol (task, cache, k); } } } } } return FALSE; }
/* Called when we get number of elements for a specific key */ static void rspamd_redis_stat_key (redisAsyncContext *c, gpointer r, gpointer priv) { struct rspamd_redis_stat_cbdata *cbdata = priv; redisReply *reply = r; ucl_object_t *obj; glong num = 0; if (cbdata->wanna_die) { return; } cbdata->inflight --; if (c->err == 0 && r != NULL) { if (G_LIKELY (reply->type == REDIS_REPLY_INTEGER)) { num = reply->integer; } else if (reply->type == REDIS_REPLY_STRING) { rspamd_strtol (reply->str, reply->len, &num); } if (num < 0) { msg_err ("bad learns count: %L", (gint64)num); num = 0; } obj = (ucl_object_t *)ucl_object_lookup (cbdata->cur, "used"); if (obj) { obj->value.iv += num; } obj = (ucl_object_t *)ucl_object_lookup (cbdata->cur, "total"); if (obj) { obj->value.iv += num; } obj = (ucl_object_t *)ucl_object_lookup (cbdata->cur, "size"); if (obj) { /* Size of key + size of int64_t */ obj->value.iv += num * (sizeof (G_STRINGIFY (G_MAXINT64)) + sizeof (guint64) + sizeof (gpointer)); } } if (cbdata->inflight == 0) { rspamd_redis_async_cbdata_cleanup (cbdata); } }
/* Called when we get number of learns for a specific key */ static void rspamd_redis_stat_learns (redisAsyncContext *c, gpointer r, gpointer priv) { struct rspamd_redis_stat_cbdata *cbdata = priv; redisReply *reply = r; ucl_object_t *obj; gulong num = 0; if (cbdata->wanna_die) { return; } cbdata->inflight --; if (c->err == 0 && r != NULL) { if (G_LIKELY (reply->type == REDIS_REPLY_INTEGER)) { num = reply->integer; } else if (reply->type == REDIS_REPLY_STRING) { rspamd_strtoul (reply->str, reply->len, &num); } obj = (ucl_object_t *)ucl_object_lookup (cbdata->cur, "revision"); if (obj) { obj->value.iv += num; } } if (cbdata->inflight == 0) { rspamd_redis_async_cbdata_cleanup (cbdata); } }
static gint rspamd_log_errlog_cmp (const ucl_object_t **o1, const ucl_object_t **o2) { const ucl_object_t *ts1, *ts2; ts1 = ucl_object_lookup (*o1, "ts"); ts2 = ucl_object_lookup (*o2, "ts"); if (ts1 && ts2) { gdouble t1 = ucl_object_todouble (ts1), t2 = ucl_object_todouble (ts2); if (t1 > t2) { return -1; } else if (t2 > t1) { return 1; } } return 0; }
static void rspamadm_confighelp_search_word_step (const ucl_object_t *obj, ucl_object_t *res, const gchar *str, gsize len, GString *path) { ucl_object_iter_t it = NULL; const ucl_object_t *cur, *elt; const gchar *dot_pos; while ((cur = ucl_object_iterate (obj, &it, true)) != NULL) { if (cur->keylen > 0) { rspamd_printf_gstring (path, ".%*s", (int) cur->keylen, cur->key); if (rspamd_substring_search_caseless (cur->key, cur->keylen, str, len) != -1) { ucl_object_insert_key (res, ucl_object_ref (cur), path->str, path->len, true); goto fin; } } if (ucl_object_type (cur) == UCL_OBJECT) { elt = ucl_object_lookup (cur, "data"); if (elt != NULL && ucl_object_type (elt) == UCL_STRING) { if (rspamd_substring_search_caseless (elt->value.sv, elt->len, str, len) != -1) { ucl_object_insert_key (res, ucl_object_ref (cur), path->str, path->len, true); goto fin; } } rspamadm_confighelp_search_word_step (cur, res, str, len, path); } fin: /* Remove the last component of the path */ dot_pos = strrchr (path->str, '.'); if (dot_pos) { g_string_erase (path, dot_pos - path->str, path->len - (dot_pos - path->str)); } } }
gpointer rspamd_redis_init (struct rspamd_stat_ctx *ctx, struct rspamd_config *cfg, struct rspamd_statfile *st) { struct redis_stat_ctx *backend; struct rspamd_statfile_config *stf = st->stcf; struct rspamd_redis_stat_elt *st_elt; const ucl_object_t *obj; gboolean ret = FALSE; backend = g_slice_alloc0 (sizeof (*backend)); /* First search in backend configuration */ obj = ucl_object_lookup (st->classifier->cfg->opts, "backend"); if (obj != NULL && ucl_object_type (obj) == UCL_OBJECT) { ret = rspamd_redis_try_ucl (backend, obj, cfg, stf->symbol); } /* Now try statfiles config */ if (!ret) { ret = rspamd_redis_try_ucl (backend, stf->opts, cfg, stf->symbol); } /* Now try classifier config */ if (!ret) { ret = rspamd_redis_try_ucl (backend, st->classifier->cfg->opts, cfg, stf->symbol); } if (!ret) { msg_err_config ("cannot init redis backend for %s", stf->symbol); g_slice_free1 (sizeof (*backend), backend); return NULL; } stf->clcf->flags |= RSPAMD_FLAG_CLASSIFIER_INCREMENTING_BACKEND; backend->stcf = stf; st_elt = g_slice_alloc0 (sizeof (*st_elt)); st_elt->ev_base = ctx->ev_base; st_elt->ctx = backend; backend->stat_elt = rspamd_stat_ctx_register_async ( rspamd_redis_async_stat_cb, rspamd_redis_async_stat_fin, st_elt, REDIS_STAT_TIMEOUT); st_elt->async = backend->stat_elt; return (gpointer)backend; }
static double get_specific_action_score (struct rspamd_task *task, const ucl_object_t *metric, struct metric_action *action) { const ucl_object_t *act, *sact; const gchar *act_name; double score; if (metric) { act = ucl_object_lookup (metric, "actions"); if (act) { act_name = rspamd_action_to_str (action->action); sact = ucl_object_lookup (act, act_name); if (sact != NULL && ucl_object_todouble_safe (sact, &score)) { msg_debug_task ("found override score %.2f for action %s in settings", score, act_name); return score; } } } return action->score; }
static gint lua_statfile_get_param (lua_State *L) { struct rspamd_statfile_config *st = lua_check_statfile (L); const gchar *param; const ucl_object_t *value; param = luaL_checkstring (L, 2); if (st != NULL && param != NULL) { value = ucl_object_lookup (st->opts, param); if (value != NULL) { lua_pushstring (L, ucl_object_tostring_forced (value)); return 1; } } lua_pushnil (L); return 1; }
static gint lua_classifier_get_param (lua_State *L) { struct rspamd_classifier_config *ccf = lua_check_classifier (L); const gchar *param; const ucl_object_t *value; param = luaL_checkstring (L, 2); if (ccf != NULL && param != NULL) { value = ucl_object_lookup (ccf->opts, param); if (value != NULL) { ucl_object_push_lua (L, value, true); return 1; } } lua_pushnil (L); return 1; }
gint rspamd_check_action_metric (struct rspamd_task *task, double score, double *rscore, struct metric *metric) { struct metric_action *action, *selected_action = NULL; double max_score = 0; const ucl_object_t *ms = NULL; int i; if (task->settings) { ms = ucl_object_lookup (task->settings, metric->name); } for (i = METRIC_ACTION_REJECT; i < METRIC_ACTION_MAX; i++) { double sc; action = &metric->actions[i]; sc = get_specific_action_score (task, ms, action); if (isnan (sc)) { continue; } if (score >= sc && sc > max_score) { selected_action = action; max_score = sc; } if (rscore != NULL && i == METRIC_ACTION_REJECT) { *rscore = sc; } } if (selected_action) { return selected_action->action; } return METRIC_ACTION_NOACTION; }
static gboolean rspamd_redis_cache_try_ucl (struct rspamd_redis_cache_ctx *cache_ctx, const ucl_object_t *obj, struct rspamd_config *cfg, const gchar *symbol) { const ucl_object_t *elt, *relt; elt = ucl_object_lookup_any (obj, "read_servers", "servers", NULL); if (elt == NULL) { return FALSE; } cache_ctx->read_servers = rspamd_upstreams_create (cfg->ups_ctx); if (!rspamd_upstreams_from_ucl (cache_ctx->read_servers, elt, REDIS_DEFAULT_PORT, NULL)) { msg_err ("statfile %s cannot get read servers configuration", symbol); return FALSE; } relt = elt; elt = ucl_object_lookup (obj, "write_servers"); if (elt == NULL) { /* Use read servers as write ones */ g_assert (relt != NULL); cache_ctx->write_servers = rspamd_upstreams_create (cfg->ups_ctx); if (!rspamd_upstreams_from_ucl (cache_ctx->write_servers, relt, REDIS_DEFAULT_PORT, NULL)) { msg_err ("statfile %s cannot get write servers configuration", symbol); return FALSE; } } else { cache_ctx->write_servers = rspamd_upstreams_create (cfg->ups_ctx); if (!rspamd_upstreams_from_ucl (cache_ctx->write_servers, elt, REDIS_DEFAULT_PORT, NULL)) { msg_err ("statfile %s cannot get write servers configuration", symbol); rspamd_upstreams_destroy (cache_ctx->write_servers); cache_ctx->write_servers = NULL; } } elt = ucl_object_lookup (obj, "timeout"); if (elt) { cache_ctx->timeout = ucl_object_todouble (elt); } else { cache_ctx->timeout = REDIS_DEFAULT_TIMEOUT; } elt = ucl_object_lookup (obj, "password"); if (elt) { cache_ctx->password = ucl_object_tostring (elt); } else { cache_ctx->password = NULL; } elt = ucl_object_lookup_any (obj, "db", "database", "dbname", NULL); if (elt) { cache_ctx->dbname = ucl_object_tostring (elt); } else { cache_ctx->dbname = NULL; } elt = ucl_object_lookup_any (obj, "cache_key", "key", NULL); if (elt == NULL || ucl_object_type (elt) != UCL_STRING) { cache_ctx->redis_object = DEFAULT_REDIS_KEY; } else { cache_ctx->redis_object = ucl_object_tostring (elt); } return TRUE; }
static void insert_metric_result (struct rspamd_task *task, struct metric *metric, const gchar *symbol, double flag, GList * opts, gboolean single) { struct metric_result *metric_res; struct symbol *s; gdouble w, *gr_score = NULL; struct rspamd_symbol_def *sdef; struct rspamd_symbols_group *gr = NULL; const ucl_object_t *mobj, *sobj; metric_res = rspamd_create_metric_result (task, metric->name); sdef = g_hash_table_lookup (metric->symbols, symbol); if (sdef == NULL) { w = 0.0; } else { w = (*sdef->weight_ptr) * flag; gr = sdef->gr; if (gr != NULL) { gr_score = g_hash_table_lookup (metric_res->sym_groups, gr); if (gr_score == NULL) { gr_score = rspamd_mempool_alloc (task->task_pool, sizeof (gdouble)); *gr_score = 0; g_hash_table_insert (metric_res->sym_groups, gr, gr_score); } } } if (task->settings) { mobj = ucl_object_lookup (task->settings, metric->name); if (mobj) { gdouble corr; sobj = ucl_object_lookup (mobj, symbol); if (sobj != NULL && ucl_object_todouble_safe (sobj, &corr)) { msg_debug ("settings: changed weight of symbol %s from %.2f to %.2f", symbol, w, corr); w = corr * flag; } } } /* XXX: does not take grow factor into account */ if (gr != NULL && gr_score != NULL && gr->max_score > 0.0) { if (*gr_score >= gr->max_score) { msg_info_task ("maximum group score %.2f for group %s has been reached," " ignoring symbol %s with weight %.2f", gr->max_score, gr->name, symbol, w); return; } else if (*gr_score + w > gr->max_score) { w = gr->max_score - *gr_score; } *gr_score += w; } /* Add metric score */ if ((s = g_hash_table_lookup (metric_res->symbols, symbol)) != NULL) { if (sdef && (sdef->flags & RSPAMD_SYMBOL_FLAG_ONESHOT)) { /* * For one shot symbols we do not need to add them again, so * we just force single behaviour here */ single = TRUE; } if (s->options && opts && opts != s->options) { /* Append new options */ s->options = g_list_concat (s->options, g_list_copy (opts)); /* * Note that there is no need to add new destructor of GList as elements of appended * GList are used directly, so just free initial GList */ } else if (opts) { s->options = g_list_copy (opts); rspamd_mempool_add_destructor (task->task_pool, (rspamd_mempool_destruct_t) g_list_free, s->options); } if (!single) { /* Handle grow factor */ if (metric_res->grow_factor && w > 0) { w *= metric_res->grow_factor; metric_res->grow_factor *= metric->grow_factor; } s->score += w; metric_res->score += w; } else { if (fabs (s->score) < fabs (w)) { /* Replace less weight with a bigger one */ metric_res->score = metric_res->score - s->score + w; s->score = w; } } } else { s = rspamd_mempool_alloc (task->task_pool, sizeof (struct symbol)); /* Handle grow factor */ if (metric_res->grow_factor && w > 0) { w *= metric_res->grow_factor; metric_res->grow_factor *= metric->grow_factor; } else if (w > 0) { metric_res->grow_factor = metric->grow_factor; } s->score = w; s->name = symbol; s->def = sdef; metric_res->score += w; if (opts) { s->options = g_list_copy (opts); rspamd_mempool_add_destructor (task->task_pool, (rspamd_mempool_destruct_t) g_list_free, s->options); } else { s->options = NULL; } g_hash_table_insert (metric_res->symbols, (gpointer) symbol, s); } msg_debug ("symbol %s, score %.2f, metric %s, factor: %f", symbol, s->score, metric->name, w); }
/* * Validate object */ static bool ucl_schema_validate_object (const ucl_object_t *schema, const ucl_object_t *obj, struct ucl_schema_error *err, const ucl_object_t *root, ucl_object_t *ext_ref) { const ucl_object_t *elt, *prop, *found, *additional_schema = NULL, *required = NULL, *pat, *pelt; ucl_object_iter_t iter = NULL, piter = NULL; bool ret = true, allow_additional = true; int64_t minmax; while (ret && (elt = ucl_object_iterate (schema, &iter, true)) != NULL) { if (elt->type == UCL_OBJECT && strcmp (ucl_object_key (elt), "properties") == 0) { piter = NULL; while (ret && (prop = ucl_object_iterate (elt, &piter, true)) != NULL) { found = ucl_object_lookup (obj, ucl_object_key (prop)); if (found) { ret = ucl_schema_validate (prop, found, true, err, root, ext_ref); } } } else if (strcmp (ucl_object_key (elt), "additionalProperties") == 0) { if (elt->type == UCL_BOOLEAN) { if (!ucl_object_toboolean (elt)) { /* Deny additional fields completely */ allow_additional = false; } } else if (elt->type == UCL_OBJECT) { /* Define validator for additional fields */ additional_schema = elt; } else { ucl_schema_create_error (err, UCL_SCHEMA_INVALID_SCHEMA, elt, "additionalProperties attribute is invalid in schema"); ret = false; break; } } else if (strcmp (ucl_object_key (elt), "required") == 0) { if (elt->type == UCL_ARRAY) { required = elt; } else { ucl_schema_create_error (err, UCL_SCHEMA_INVALID_SCHEMA, elt, "required attribute is invalid in schema"); ret = false; break; } } else if (strcmp (ucl_object_key (elt), "minProperties") == 0 && ucl_object_toint_safe (elt, &minmax)) { if (obj->len < minmax) { ucl_schema_create_error (err, UCL_SCHEMA_CONSTRAINT, obj, "object has not enough properties: %u, minimum is: %u", obj->len, (unsigned)minmax); ret = false; break; } } else if (strcmp (ucl_object_key (elt), "maxProperties") == 0 && ucl_object_toint_safe (elt, &minmax)) { if (obj->len > minmax) { ucl_schema_create_error (err, UCL_SCHEMA_CONSTRAINT, obj, "object has too many properties: %u, maximum is: %u", obj->len, (unsigned)minmax); ret = false; break; } } else if (strcmp (ucl_object_key (elt), "patternProperties") == 0) { piter = NULL; while (ret && (prop = ucl_object_iterate (elt, &piter, true)) != NULL) { found = ucl_schema_test_pattern (obj, ucl_object_key (prop)); if (found) { ret = ucl_schema_validate (prop, found, true, err, root, ext_ref); } } } else if (elt->type == UCL_OBJECT && strcmp (ucl_object_key (elt), "dependencies") == 0) { ret = ucl_schema_validate_dependencies (elt, obj, err, root, ext_ref); } } if (ret) { /* Additional properties */ if (!allow_additional || additional_schema != NULL) { /* Check if we have exactly the same properties in schema and object */ iter = NULL; prop = ucl_object_lookup (schema, "properties"); while ((elt = ucl_object_iterate (obj, &iter, true)) != NULL) { found = ucl_object_lookup (prop, ucl_object_key (elt)); if (found == NULL) { /* Try patternProperties */ piter = NULL; pat = ucl_object_lookup (schema, "patternProperties"); while ((pelt = ucl_object_iterate (pat, &piter, true)) != NULL) { found = ucl_schema_test_pattern (obj, ucl_object_key (pelt)); if (found != NULL) { break; } } } if (found == NULL) { if (!allow_additional) { ucl_schema_create_error (err, UCL_SCHEMA_CONSTRAINT, obj, "object has non-allowed property %s", ucl_object_key (elt)); ret = false; break; } else if (additional_schema != NULL) { if (!ucl_schema_validate (additional_schema, elt, true, err, root, ext_ref)) { ret = false; break; } } } } } /* Required properties */ if (required != NULL) { iter = NULL; while ((elt = ucl_object_iterate (required, &iter, true)) != NULL) { if (ucl_object_lookup (obj, ucl_object_tostring (elt)) == NULL) { ucl_schema_create_error (err, UCL_SCHEMA_MISSING_PROPERTY, obj, "object has missing property %s", ucl_object_tostring (elt)); ret = false; break; } } } } return ret; }
static struct rspamd_osb_tokenizer_config * rspamd_tokenizer_osb_config_from_ucl (rspamd_mempool_t * pool, const ucl_object_t *obj) { const ucl_object_t *elt; struct rspamd_osb_tokenizer_config *cf, *def; guchar *key = NULL; gsize keylen; if (pool != NULL) { cf = rspamd_mempool_alloc0 (pool, sizeof (*cf)); } else { cf = g_malloc0 (sizeof (*cf)); } /* Use default config */ def = rspamd_tokenizer_osb_default_config (); memcpy (cf, def, sizeof (*cf)); elt = ucl_object_lookup (obj, "hash"); if (elt != NULL && ucl_object_type (elt) == UCL_STRING) { if (g_ascii_strncasecmp (ucl_object_tostring (elt), "xxh", 3) == 0) { cf->ht = RSPAMD_OSB_HASH_XXHASH; elt = ucl_object_lookup (obj, "seed"); if (elt != NULL && ucl_object_type (elt) == UCL_INT) { cf->seed = ucl_object_toint (elt); } } else if (g_ascii_strncasecmp (ucl_object_tostring (elt), "sip", 3) == 0) { cf->ht = RSPAMD_OSB_HASH_SIPHASH; elt = ucl_object_lookup (obj, "key"); if (elt != NULL && ucl_object_type (elt) == UCL_STRING) { key = rspamd_decode_base32 (ucl_object_tostring (elt), 0, &keylen); if (keylen < sizeof (rspamd_sipkey_t)) { msg_warn ("siphash key is too short: %z", keylen); g_free (key); } else { memcpy (cf->sk, key, sizeof (cf->sk)); g_free (key); } } else { msg_warn_pool ("siphash cannot be used without key"); } } } else { elt = ucl_object_lookup (obj, "compat"); if (elt != NULL && ucl_object_toboolean (elt)) { cf->ht = RSPAMD_OSB_HASH_COMPAT; } } elt = ucl_object_lookup (obj, "window"); if (elt != NULL && ucl_object_type (elt) == UCL_INT) { cf->window_size = ucl_object_toint (elt); if (cf->window_size > DEFAULT_FEATURE_WINDOW_SIZE * 4) { msg_err_pool ("too large window size: %d", cf->window_size); cf->window_size = DEFAULT_FEATURE_WINDOW_SIZE; } } return cf; }
int main(int argc, char **argv) { const char *fn = NULL; unsigned char *inbuf; struct ucl_parser *parser; int k, ret = 0, r = 0; ssize_t bufsize; ucl_object_t *obj = NULL; const ucl_object_t *par; FILE *in; if (argc > 1) { fn = argv[1]; } if (fn != NULL) { in = fopen (fn, "r"); if (in == NULL) { exit (-errno); } } else { in = stdin; } parser = ucl_parser_new (0); inbuf = malloc (BUFSIZ); bufsize = BUFSIZ; r = 0; while (!feof (in) && !ferror (in)) { if (r == bufsize) { inbuf = realloc (inbuf, bufsize * 2); bufsize *= 2; if (inbuf == NULL) { perror ("realloc"); exit (EXIT_FAILURE); } } r += fread (inbuf + r, 1, bufsize - r, in); } if (ferror (in)) { fprintf (stderr, "Failed to read the input file.\n"); exit (EXIT_FAILURE); } ucl_parser_add_chunk (parser, inbuf, r); fclose (in); if (ucl_parser_get_error(parser)) { printf ("Error occured: %s\n", ucl_parser_get_error(parser)); ret = 1; goto end; } obj = ucl_parser_get_object (parser); if (ucl_parser_get_error (parser)) { printf ("Error occured: %s\n", ucl_parser_get_error(parser)); ret = 1; goto end; } if (argc > 2) { for (k = 2; k < argc; k++) { printf ("search for \"%s\"... ", argv[k]); par = ucl_object_lookup (obj, argv[k]); printf ("%sfound\n", (par == NULL )?"not ":""); ucl_obj_dump (par, 0); } } else { ucl_obj_dump (obj, 0); } end: if (parser != NULL) { ucl_parser_free (parser); } if (obj != NULL) { ucl_object_unref (obj); } return ret; }
static gboolean rspamd_fuzzy_backend_redis_try_ucl (struct rspamd_fuzzy_backend_redis *backend, const ucl_object_t *obj, struct rspamd_config *cfg) { const ucl_object_t *elt, *relt; elt = ucl_object_lookup_any (obj, "read_servers", "servers", NULL); if (elt == NULL) { return FALSE; } backend->read_servers = rspamd_upstreams_create (cfg->ups_ctx); if (!rspamd_upstreams_from_ucl (backend->read_servers, elt, REDIS_DEFAULT_PORT, NULL)) { msg_err_config ("cannot get read servers configuration"); return FALSE; } relt = elt; elt = ucl_object_lookup (obj, "write_servers"); if (elt == NULL) { /* Use read servers as write ones */ g_assert (relt != NULL); backend->write_servers = rspamd_upstreams_create (cfg->ups_ctx); if (!rspamd_upstreams_from_ucl (backend->write_servers, relt, REDIS_DEFAULT_PORT, NULL)) { msg_err_config ("cannot get write servers configuration"); return FALSE; } } else { backend->write_servers = rspamd_upstreams_create (cfg->ups_ctx); if (!rspamd_upstreams_from_ucl (backend->write_servers, elt, REDIS_DEFAULT_PORT, NULL)) { msg_err_config ("cannot get write servers configuration"); rspamd_upstreams_destroy (backend->write_servers); backend->write_servers = NULL; } } elt = ucl_object_lookup (obj, "prefix"); if (elt == NULL || ucl_object_type (elt) != UCL_STRING) { backend->redis_object = REDIS_DEFAULT_OBJECT; } else { backend->redis_object = ucl_object_tostring (elt); } elt = ucl_object_lookup (obj, "timeout"); if (elt) { backend->timeout = ucl_object_todouble (elt); } else { backend->timeout = REDIS_DEFAULT_TIMEOUT; } elt = ucl_object_lookup (obj, "password"); if (elt) { backend->password = ucl_object_tostring (elt); } else { backend->password = NULL; } elt = ucl_object_lookup_any (obj, "db", "database", "dbname", NULL); if (elt) { backend->dbname = ucl_object_tostring (elt); } else { backend->dbname = NULL; } return TRUE; }
void* rspamd_fuzzy_backend_init_redis (struct rspamd_fuzzy_backend *bk, const ucl_object_t *obj, struct rspamd_config *cfg, GError **err) { struct rspamd_fuzzy_backend_redis *backend; const ucl_object_t *elt; gboolean ret = FALSE; guchar id_hash[rspamd_cryptobox_HASHBYTES]; rspamd_cryptobox_hash_state_t st; backend = g_slice_alloc0 (sizeof (*backend)); backend->timeout = REDIS_DEFAULT_TIMEOUT; backend->redis_object = REDIS_DEFAULT_OBJECT; ret = rspamd_fuzzy_backend_redis_try_ucl (backend, obj, cfg); /* Now try global redis settings */ if (!ret) { elt = ucl_object_lookup (cfg->rcl_obj, "redis"); if (elt) { const ucl_object_t *specific_obj; specific_obj = ucl_object_lookup_any (elt, "fuzzy", "fuzzy_storage", NULL); if (specific_obj) { ret = rspamd_fuzzy_backend_redis_try_ucl (backend, specific_obj, cfg); } else { ret = rspamd_fuzzy_backend_redis_try_ucl (backend, elt, cfg); } } } if (!ret) { msg_err_config ("cannot init redis backend for fuzzy storage"); g_slice_free1 (sizeof (*backend), backend); return NULL; } REF_INIT_RETAIN (backend, rspamd_fuzzy_backend_redis_dtor); backend->pool = cfg->redis_pool; rspamd_cryptobox_hash_init (&st, NULL, 0); rspamd_cryptobox_hash_update (&st, backend->redis_object, strlen (backend->redis_object)); if (backend->dbname) { rspamd_cryptobox_hash_update (&st, backend->dbname, strlen (backend->dbname)); } if (backend->password) { rspamd_cryptobox_hash_update (&st, backend->password, strlen (backend->password)); } rspamd_cryptobox_hash_final (&st, id_hash); backend->id = rspamd_encode_base32 (id_hash, sizeof (id_hash)); return backend; }
struct rspamd_map* rspamd_map_add_from_ucl (struct rspamd_config *cfg, const ucl_object_t *obj, const gchar *description, map_cb_t read_callback, map_fin_cb_t fin_callback, void **user_data) { ucl_object_iter_t it = NULL; const ucl_object_t *cur, *elt; struct rspamd_map *map; struct rspamd_map_backend *bk; g_assert (obj != NULL); if (ucl_object_type (obj) == UCL_STRING) { /* Just a plain string */ return rspamd_map_add (cfg, ucl_object_tostring (obj), NULL, read_callback, fin_callback, user_data); } map = g_slice_alloc0 (sizeof (struct rspamd_map)); map->read_callback = read_callback; map->fin_callback = fin_callback; map->user_data = user_data; map->cfg = cfg; map->id = g_random_int (); map->locked = rspamd_mempool_alloc0_shared (cfg->cfg_pool, sizeof (gint)); map->cache = rspamd_mempool_alloc0_shared (cfg->cfg_pool, sizeof (*map->cache)); map->backends = g_ptr_array_new (); map->poll_timeout = cfg->map_timeout; if (description) { map->description = g_strdup (description); } if (ucl_object_type (obj) == UCL_ARRAY) { /* Add array of maps as multiple backends */ while ((cur = ucl_object_iterate (obj, &it, true)) != NULL) { if (ucl_object_type (cur) == UCL_STRING) { bk = rspamd_map_parse_backend (cfg, ucl_object_tostring (cur)); if (bk != NULL) { g_ptr_array_add (map->backends, bk); if (!map->name) { map->name = g_strdup (ucl_object_tostring (cur)); } } } else { msg_err_config ("bad map element type: %s", ucl_object_type_to_string (ucl_object_type (cur))); } } if (map->backends->len == 0) { msg_err_config ("map has no urls to be loaded"); goto err; } } else if (ucl_object_type (obj) == UCL_OBJECT) { elt = ucl_object_lookup (obj, "name"); if (elt && ucl_object_type (elt) == UCL_STRING) { map->name = g_strdup (ucl_object_tostring (elt)); } elt = ucl_object_lookup (obj, "description"); if (elt && ucl_object_type (elt) == UCL_STRING) { if (map->description) { g_free (map->description); } map->description = g_strdup (ucl_object_tostring (elt)); } elt = ucl_object_lookup_any (obj, "timeout", "poll", "poll_time", "watch_interval", NULL); if (elt) { map->poll_timeout = ucl_object_todouble (elt); } elt = ucl_object_lookup_any (obj, "upstreams", "url", "urls", NULL); if (elt == NULL) { msg_err_config ("map has no urls to be loaded"); goto err; } if (ucl_object_type (obj) == UCL_ARRAY) { /* Add array of maps as multiple backends */ while ((cur = ucl_object_iterate (elt, &it, true)) != NULL) { if (ucl_object_type (cur) == UCL_STRING) { bk = rspamd_map_parse_backend (cfg, ucl_object_tostring (cur)); if (bk != NULL) { g_ptr_array_add (map->backends, bk); if (!map->name) { map->name = g_strdup (ucl_object_tostring (cur)); } } } else { msg_err_config ("bad map element type: %s", ucl_object_type_to_string (ucl_object_type (cur))); goto err; } } if (map->backends->len == 0) { msg_err_config ("map has no urls to be loaded"); goto err; } } else if (ucl_object_type (elt) == UCL_STRING) { bk = rspamd_map_parse_backend (cfg, ucl_object_tostring (elt)); if (bk != NULL) { g_ptr_array_add (map->backends, bk); if (!map->name) { map->name = g_strdup (ucl_object_tostring (cur)); } } } if (map->backends->len == 0) { msg_err_config ("map has no urls to be loaded"); goto err; } } rspamd_map_calculate_hash (map); msg_info_map ("added map from ucl"); cfg->maps = g_list_prepend (cfg->maps, map); return map; err: g_ptr_array_free (map->backends, TRUE); g_free (map->name); g_free (map->description); g_slice_free1 (sizeof (*map), map); return NULL; }
static bool ucl_schema_validate_number (const ucl_object_t *schema, const ucl_object_t *obj, struct ucl_schema_error *err) { const ucl_object_t *elt, *test; ucl_object_iter_t iter = NULL; bool ret = true, exclusive = false; double constraint, val; const double alpha = 1e-16; while (ret && (elt = ucl_object_iterate (schema, &iter, true)) != NULL) { if ((elt->type == UCL_FLOAT || elt->type == UCL_INT) && strcmp (ucl_object_key (elt), "multipleOf") == 0) { constraint = ucl_object_todouble (elt); if (constraint <= 0) { ucl_schema_create_error (err, UCL_SCHEMA_INVALID_SCHEMA, elt, "multipleOf must be greater than zero"); ret = false; break; } val = ucl_object_todouble (obj); if (fabs (remainder (val, constraint)) > alpha) { ucl_schema_create_error (err, UCL_SCHEMA_CONSTRAINT, obj, "number %.4f is not multiple of %.4f, remainder is %.7f", val, constraint); ret = false; break; } } else if ((elt->type == UCL_FLOAT || elt->type == UCL_INT) && strcmp (ucl_object_key (elt), "maximum") == 0) { constraint = ucl_object_todouble (elt); test = ucl_object_lookup (schema, "exclusiveMaximum"); if (test && test->type == UCL_BOOLEAN) { exclusive = ucl_object_toboolean (test); } val = ucl_object_todouble (obj); if (val > constraint || (exclusive && val >= constraint)) { ucl_schema_create_error (err, UCL_SCHEMA_CONSTRAINT, obj, "number is too big: %.3f, maximum is: %.3f", val, constraint); ret = false; break; } } else if ((elt->type == UCL_FLOAT || elt->type == UCL_INT) && strcmp (ucl_object_key (elt), "minimum") == 0) { constraint = ucl_object_todouble (elt); test = ucl_object_lookup (schema, "exclusiveMinimum"); if (test && test->type == UCL_BOOLEAN) { exclusive = ucl_object_toboolean (test); } val = ucl_object_todouble (obj); if (val < constraint || (exclusive && val <= constraint)) { ucl_schema_create_error (err, UCL_SCHEMA_CONSTRAINT, obj, "number is too small: %.3f, minimum is: %.3f", val, constraint); ret = false; break; } } } return ret; }
gpointer rspamd_stat_cache_redis_init (struct rspamd_stat_ctx *ctx, struct rspamd_config *cfg, struct rspamd_statfile *st, const ucl_object_t *cf) { struct rspamd_redis_cache_ctx *cache_ctx; struct rspamd_statfile_config *stf = st->stcf; const ucl_object_t *obj; gboolean ret = FALSE; cache_ctx = g_slice_alloc0 (sizeof (*cache_ctx)); /* First search in backend configuration */ obj = ucl_object_lookup (st->classifier->cfg->opts, "backend"); if (obj != NULL && ucl_object_type (obj) == UCL_OBJECT) { ret = rspamd_redis_cache_try_ucl (cache_ctx, obj, cfg, stf->symbol); } /* Now try statfiles config */ if (!ret) { ret = rspamd_redis_cache_try_ucl (cache_ctx, stf->opts, cfg, stf->symbol); } /* Now try classifier config */ if (!ret) { ret = rspamd_redis_cache_try_ucl (cache_ctx, st->classifier->cfg->opts, cfg, stf->symbol); } /* Now try global redis settings */ if (!ret) { obj = ucl_object_lookup (cfg->rcl_obj, "redis"); if (obj) { const ucl_object_t *specific_obj; specific_obj = ucl_object_lookup (obj, "statistics"); if (specific_obj) { ret = rspamd_redis_cache_try_ucl (cache_ctx, specific_obj, cfg, stf->symbol); } else { ret = rspamd_redis_cache_try_ucl (cache_ctx, obj, cfg, stf->symbol); } } } if (!ret) { msg_err_config ("cannot init redis cache for %s", stf->symbol); g_slice_free1 (sizeof (*cache_ctx), cache_ctx); return NULL; } cache_ctx->stcf = stf; return (gpointer)cache_ctx; }
static gboolean rspamd_symbols_cache_load_items (struct symbols_cache *cache, const gchar *name) { struct rspamd_symbols_cache_header *hdr; struct stat st; struct ucl_parser *parser; ucl_object_t *top; const ucl_object_t *cur, *elt; ucl_object_iter_t it; struct cache_item *item, *parent; const guchar *p; gint fd; gpointer map; fd = open (name, O_RDONLY); if (fd == -1) { msg_info_cache ("cannot open file %s, error %d, %s", name, errno, strerror (errno)); return FALSE; } if (fstat (fd, &st) == -1) { close (fd); msg_info_cache ("cannot stat file %s, error %d, %s", name, errno, strerror (errno)); return FALSE; } if (st.st_size < (gint)sizeof (*hdr)) { close (fd); errno = EINVAL; msg_info_cache ("cannot use file %s, error %d, %s", name, errno, strerror (errno)); return FALSE; } map = mmap (NULL, st.st_size, PROT_READ, MAP_SHARED, fd, 0); if (map == MAP_FAILED) { close (fd); msg_info_cache ("cannot mmap file %s, error %d, %s", name, errno, strerror (errno)); return FALSE; } close (fd); hdr = map; if (memcmp (hdr->magic, rspamd_symbols_cache_magic, sizeof (rspamd_symbols_cache_magic)) != 0) { msg_info_cache ("cannot use file %s, bad magic", name); munmap (map, st.st_size); return FALSE; } parser = ucl_parser_new (0); p = (const guchar *)(hdr + 1); if (!ucl_parser_add_chunk (parser, p, st.st_size - sizeof (*hdr))) { msg_info_cache ("cannot use file %s, cannot parse: %s", name, ucl_parser_get_error (parser)); munmap (map, st.st_size); ucl_parser_free (parser); return FALSE; } top = ucl_parser_get_object (parser); munmap (map, st.st_size); ucl_parser_free (parser); if (top == NULL || ucl_object_type (top) != UCL_OBJECT) { msg_info_cache ("cannot use file %s, bad object", name); ucl_object_unref (top); return FALSE; } it = ucl_object_iterate_new (top); while ((cur = ucl_object_iterate_safe (it, true))) { item = g_hash_table_lookup (cache->items_by_symbol, ucl_object_key (cur)); if (item) { /* Copy saved info */ /* * XXX: don't save or load weight, it should be obtained from the * metric */ #if 0 elt = ucl_object_lookup (cur, "weight"); if (elt) { w = ucl_object_todouble (elt); if (w != 0) { item->weight = w; } } #endif elt = ucl_object_lookup (cur, "time"); if (elt) { item->avg_time = ucl_object_todouble (elt); } elt = ucl_object_lookup (cur, "count"); if (elt) { item->avg_counter = ucl_object_toint (elt); } elt = ucl_object_lookup (cur, "frequency"); if (elt) { item->frequency = ucl_object_toint (elt); } if ((item->type & SYMBOL_TYPE_VIRTUAL) && item->parent != -1) { g_assert (item->parent < (gint)cache->items_by_id->len); parent = g_ptr_array_index (cache->items_by_id, item->parent); if (parent->weight < item->weight) { parent->weight = item->weight; } /* * We maintain avg_time for virtual symbols equal to the * parent item avg_time */ parent->avg_time = item->avg_time; parent->avg_counter = item->avg_counter; } cache->total_weight += fabs (item->weight); cache->total_freq += item->frequency; } } ucl_object_iterate_free (it); ucl_object_unref (top); return TRUE; }
static gboolean rspamd_redis_try_ucl (struct redis_stat_ctx *backend, const ucl_object_t *obj, struct rspamd_config *cfg, const gchar *symbol) { const ucl_object_t *elt, *relt, *users_enabled; const gchar *lua_script; elt = ucl_object_lookup_any (obj, "read_servers", "servers", NULL); if (elt == NULL) { return FALSE; } backend->read_servers = rspamd_upstreams_create (cfg->ups_ctx); if (!rspamd_upstreams_from_ucl (backend->read_servers, elt, REDIS_DEFAULT_PORT, NULL)) { msg_err ("statfile %s cannot get read servers configuration", symbol); return FALSE; } relt = elt; elt = ucl_object_lookup (obj, "write_servers"); if (elt == NULL) { /* Use read servers as write ones */ g_assert (relt != NULL); backend->write_servers = rspamd_upstreams_create (cfg->ups_ctx); if (!rspamd_upstreams_from_ucl (backend->write_servers, relt, REDIS_DEFAULT_PORT, NULL)) { msg_err ("statfile %s cannot get write servers configuration", symbol); return FALSE; } } else { backend->write_servers = rspamd_upstreams_create (cfg->ups_ctx); if (!rspamd_upstreams_from_ucl (backend->write_servers, elt, REDIS_DEFAULT_PORT, NULL)) { msg_err ("statfile %s cannot get write servers configuration", symbol); rspamd_upstreams_destroy (backend->write_servers); backend->write_servers = NULL; } } elt = ucl_object_lookup (obj, "prefix"); if (elt == NULL || ucl_object_type (elt) != UCL_STRING) { /* Default non-users statistics */ backend->redis_object = REDIS_DEFAULT_OBJECT; /* * Make redis backend compatible with sqlite3 backend in users settings */ users_enabled = ucl_object_lookup_any (obj, "per_user", "users_enabled", NULL); if (users_enabled != NULL) { if (ucl_object_type (users_enabled) == UCL_BOOLEAN) { backend->enable_users = ucl_object_toboolean (users_enabled); backend->cbref_user = -1; if (backend->enable_users) { backend->redis_object = REDIS_DEFAULT_USERS_OBJECT; } } else if (ucl_object_type (users_enabled) == UCL_STRING) { lua_script = ucl_object_tostring (users_enabled); if (luaL_dostring (cfg->lua_state, lua_script) != 0) { msg_err_config ("cannot execute lua script for users " "extraction: %s", lua_tostring (cfg->lua_state, -1)); } else { if (lua_type (cfg->lua_state, -1) == LUA_TFUNCTION) { backend->enable_users = TRUE; backend->cbref_user = luaL_ref (cfg->lua_state, LUA_REGISTRYINDEX); } else { msg_err_config ("lua script must return " "function(task) and not %s", lua_typename (cfg->lua_state, lua_type ( cfg->lua_state, -1))); } } } } else { backend->enable_users = FALSE; } } else { /* XXX: sanity check */ backend->redis_object = ucl_object_tostring (elt); } elt = ucl_object_lookup (obj, "timeout"); if (elt) { backend->timeout = ucl_object_todouble (elt); } else { backend->timeout = REDIS_DEFAULT_TIMEOUT; } elt = ucl_object_lookup (obj, "password"); if (elt) { backend->password = ucl_object_tostring (elt); } else { backend->password = NULL; } elt = ucl_object_lookup_any (obj, "db", "database", "dbname", NULL); if (elt) { backend->dbname = ucl_object_tostring (elt); } else { backend->dbname = NULL; } return TRUE; }
int main (int argc, char **argv) { ucl_object_t *obj, *cur, *ar, *ar1, *ref, *test_obj, *comments; ucl_object_iter_t it; const ucl_object_t *found, *it_obj, *test; struct ucl_emitter_functions *fn; FILE *out; unsigned char *emitted; const char *fname_out = NULL; struct ucl_parser *parser; int ret = 0; switch (argc) { case 2: fname_out = argv[1]; break; } if (fname_out != NULL) { out = fopen (fname_out, "w"); if (out == NULL) { exit (-errno); } } else { out = stdout; } obj = ucl_object_typed_new (UCL_OBJECT); /* Keys replacing */ cur = ucl_object_fromstring_common ("value1", 0, UCL_STRING_TRIM); ucl_object_insert_key (obj, cur, "key0", 0, false); cur = ucl_object_fromdouble (0.1); assert (ucl_object_replace_key (obj, cur, "key0", 0, false)); /* Create some strings */ cur = ucl_object_fromstring_common (" test string ", 0, UCL_STRING_TRIM); ucl_object_insert_key (obj, cur, "key1", 0, false); cur = ucl_object_fromstring_common (" test \nstring\n\r\n\b\t\f\\\" ", 0, UCL_STRING_TRIM | UCL_STRING_ESCAPE); ucl_object_insert_key (obj, cur, "key2", 0, false); cur = ucl_object_fromstring_common (" test string \n", 0, 0); ucl_object_insert_key (obj, cur, "key3", 0, false); /* Array of numbers */ ar = ucl_object_typed_new (UCL_ARRAY); cur = ucl_object_fromint (10); ucl_array_append (ar, cur); assert (ucl_array_index_of (ar, cur) == 0); cur = ucl_object_fromdouble (10.1); ucl_array_append (ar, cur); assert (ucl_array_index_of (ar, cur) == 1); cur = ucl_object_fromdouble (9.999); ucl_array_prepend (ar, cur); assert (ucl_array_index_of (ar, cur) == 0); ar1 = ucl_object_copy (ar); cur = ucl_object_fromstring ("abc"); ucl_array_prepend (ar1, cur); cur = ucl_object_fromstring ("cde"); ucl_array_prepend (ar1, cur); cur = ucl_object_fromstring ("аАаБаВ"); /* UTF8 */ ucl_array_prepend (ar1, cur); cur = ucl_object_fromstring ("ааБаВ"); /* UTF8 */ ucl_array_prepend (ar1, cur); /* * This is ususally broken or fragile as utf collate is far from perfect cur = ucl_object_fromstring ("баБаВ"); ucl_array_prepend (ar1, cur); cur = ucl_object_fromstring ("ааБаВ"); // hello to @bapt ucl_array_prepend (ar1, cur); */ cur = ucl_object_fromstring ("№"); /* everybody likes emoji in the code */ ucl_array_prepend (ar1, cur); ucl_object_array_sort (ar1, ucl_object_compare_qsort); /* Removing from an array */ cur = ucl_object_fromdouble (1.0); ucl_array_append (ar, cur); cur = ucl_array_delete (ar, cur); assert (ucl_object_todouble (cur) == 1.0); ucl_object_unref (cur); cur = ucl_object_fromdouble (2.0); ucl_array_append (ar, cur); cur = ucl_array_pop_last (ar); assert (ucl_object_todouble (cur) == 2.0); ucl_object_unref (cur); cur = ucl_object_fromdouble (3.0); ucl_array_prepend (ar, cur); cur = ucl_array_pop_first (ar); assert (ucl_object_todouble (cur) == 3.0); ucl_object_unref (cur); ucl_object_insert_key (obj, ar, "key4", 0, false); cur = ucl_object_frombool (true); /* Ref object to test refcounts */ ref = ucl_object_ref (cur); ucl_object_insert_key (obj, cur, "key4", 0, false); /* Empty strings */ cur = ucl_object_fromstring_common (" ", 0, UCL_STRING_TRIM); ucl_object_insert_key (obj, cur, "key5", 0, false); cur = ucl_object_fromstring_common ("", 0, UCL_STRING_ESCAPE); ucl_object_insert_key (obj, cur, "key6", 0, false); cur = ucl_object_fromstring_common (" \n", 0, UCL_STRING_ESCAPE); ucl_object_insert_key (obj, cur, "key7", 0, false); /* Numbers and booleans */ cur = ucl_object_fromstring_common ("1mb", 0, UCL_STRING_ESCAPE | UCL_STRING_PARSE); ucl_object_insert_key (obj, cur, "key8", 0, false); cur = ucl_object_fromstring_common ("3.14", 0, UCL_STRING_PARSE); ucl_object_insert_key (obj, cur, "key9", 0, false); cur = ucl_object_fromstring_common ("true", 0, UCL_STRING_PARSE); ucl_object_insert_key (obj, cur, "key10", 0, false); cur = ucl_object_fromstring_common (" off ", 0, UCL_STRING_PARSE | UCL_STRING_TRIM); ucl_object_insert_key (obj, cur, "key11", 0, false); cur = ucl_object_fromstring_common ("*****@*****.**", 0, UCL_STRING_PARSE_INT); ucl_object_insert_key (obj, cur, "key12", 0, false); cur = ucl_object_fromstring_common ("#test", 0, UCL_STRING_PARSE_INT); ucl_object_insert_key (obj, cur, "key13", 0, false); cur = ucl_object_frombool (true); ucl_object_insert_key (obj, cur, "k=3", 0, false); ucl_object_insert_key (obj, ar1, "key14", 0, false); cur = ucl_object_new_userdata (ud_dtor, ud_emit, NULL); ucl_object_insert_key (obj, cur, "key15", 0, false); /* More tests for keys */ cur = ucl_object_fromlstring ("test", 3); ucl_object_insert_key (obj, cur, "key16", 0, false); test = ucl_object_lookup_any (obj, "key100", "key200", "key300", "key16", NULL); assert (test == cur); test = ucl_object_lookup_len (obj, "key160", 5); assert (test == cur); cur = ucl_object_pop_key (obj, "key16"); assert (test == cur); test = ucl_object_pop_key (obj, "key16"); assert (test == NULL); test = ucl_object_lookup_len (obj, "key160", 5); assert (test == NULL); /* Objects merging tests */ test_obj = ucl_object_new_full (UCL_OBJECT, 2); ucl_object_insert_key (test_obj, cur, "key16", 0, true); ucl_object_merge (obj, test_obj, true); ucl_object_unref (test_obj); /* Array merging test */ test_obj = ucl_object_new_full (UCL_ARRAY, 3); ucl_array_append (test_obj, ucl_object_fromstring ("test")); ucl_array_merge (test_obj, ar1, false); ucl_object_insert_key (obj, test_obj, "key17", 0, true); /* Object deletion */ cur = ucl_object_fromstring ("test"); ucl_object_insert_key (obj, cur, "key18", 0, true); assert (ucl_object_delete_key (obj, "key18")); assert (!ucl_object_delete_key (obj, "key18")); cur = ucl_object_fromlstring ("test", 4); ucl_object_insert_key (obj, cur, "key18\0\0", 7, true); assert (ucl_object_lookup_len (obj, "key18\0\0", 7) == cur); assert (ucl_object_lookup (obj, "key18") == NULL); assert (ucl_object_lookup_len (obj, "key18\0\1", 7) == NULL); assert (ucl_object_delete_keyl (obj, "key18\0\0", 7)); /* Comments */ comments = ucl_object_typed_new (UCL_OBJECT); found = ucl_object_lookup (obj, "key17"); test = ucl_object_lookup (obj, "key16"); ucl_comments_add (comments, found, "# test comment"); assert (ucl_comments_find (comments, found) != NULL); assert (ucl_comments_find (comments, test) == NULL); ucl_comments_move (comments, found, test); assert (ucl_comments_find (comments, found) == NULL); assert (ucl_comments_find (comments, test) != NULL); /* Array replace */ ar1 = ucl_object_typed_new (UCL_ARRAY); cur = ucl_object_fromstring ("test"); cur = ucl_elt_append (cur, ucl_object_fromstring ("test1")); ucl_array_append (ar1, cur); test = ucl_array_replace_index (ar1, ucl_object_fromstring ("test2"), 0); assert (test == cur); /* Try to find using path */ /* Should exist */ found = ucl_object_lookup_path (obj, "key4.1"); assert (found != NULL && ucl_object_toint (found) == 10); /* . should be ignored */ found = ucl_object_lookup_path (obj, ".key4.1"); assert (found != NULL && ucl_object_toint (found) == 10); /* moar dots... */ found = ucl_object_lookup_path (obj, ".key4........1..."); assert (found != NULL && ucl_object_toint (found) == 10); /* No such index */ found = ucl_object_lookup_path (obj, ".key4.3"); assert (found == NULL); /* No such key */ found = ucl_object_lookup_path (obj, "key9..key1"); assert (found == NULL); /* Test iteration */ it = ucl_object_iterate_new (obj); it_obj = ucl_object_iterate_safe (it, true); /* key0 = 0.1 */ assert (ucl_object_type (it_obj) == UCL_FLOAT); it_obj = ucl_object_iterate_safe (it, true); /* key1 = "" */ assert (ucl_object_type (it_obj) == UCL_STRING); it_obj = ucl_object_iterate_safe (it, true); /* key2 = "" */ assert (ucl_object_type (it_obj) == UCL_STRING); it_obj = ucl_object_iterate_safe (it, true); /* key3 = "" */ assert (ucl_object_type (it_obj) == UCL_STRING); it_obj = ucl_object_iterate_safe (it, true); /* key4 = ([float, int, float], boolean) */ ucl_object_iterate_reset (it, it_obj); it_obj = ucl_object_iterate_safe (it, true); assert (ucl_object_type (it_obj) == UCL_FLOAT); it_obj = ucl_object_iterate_safe (it, true); assert (ucl_object_type (it_obj) == UCL_INT); it_obj = ucl_object_iterate_safe (it, true); assert (ucl_object_type (it_obj) == UCL_FLOAT); it_obj = ucl_object_iterate_safe (it, true); assert (ucl_object_type (it_obj) == UCL_BOOLEAN); ucl_object_iterate_free (it); fn = ucl_object_emit_memory_funcs (&emitted); assert (ucl_object_emit_full (obj, UCL_EMIT_CONFIG, fn, comments)); fprintf (out, "%s\n", emitted); ucl_object_emit_funcs_free (fn); ucl_object_unref (obj); ucl_object_unref (comments); parser = ucl_parser_new (UCL_PARSER_NO_IMPLICIT_ARRAYS); if (ucl_parser_add_chunk_full (parser, emitted, strlen (emitted), 3, UCL_DUPLICATE_ERROR, UCL_PARSE_UCL)) { /* Should fail due to duplicate */ assert (0); } else { assert (ucl_parser_get_error (parser) != NULL); ucl_parser_clear_error (parser); ucl_parser_free (parser); parser = ucl_parser_new (0); ucl_parser_add_chunk_full (parser, emitted, strlen (emitted), 3, UCL_DUPLICATE_MERGE, UCL_PARSE_UCL); } assert (ucl_parser_get_column (parser) == 0); assert (ucl_parser_get_linenum (parser) != 0); ucl_parser_clear_error (parser); assert (ucl_parser_get_error_code (parser) == 0); obj = ucl_parser_get_object (parser); ucl_parser_free (parser); ucl_object_free (obj); if (emitted != NULL) { free (emitted); } fclose (out); /* Ref should still be accessible */ ref->value.iv = 100500; ucl_object_unref (ref); return ret; }