static gboolean
check_metric_settings (struct rspamd_task *task, struct metric *metric,
double *score)
{
const ucl_object_t *mobj, *reject, *act;
double val;
if (task->settings == NULL) {
return FALSE;
}
mobj = ucl_object_find_key (task->settings, metric->name);
if (mobj != NULL) {
act = ucl_object_find_key (mobj, "actions");
if (act != NULL) {
reject = ucl_object_find_key (act,
rspamd_action_to_str (METRIC_ACTION_REJECT));
if (reject != NULL && ucl_object_todouble_safe (reject, &val)) {
*score = val;
return TRUE;
}
}
}
return FALSE;
}
/*
* Check dependencies for an object
*/
static bool
ucl_schema_validate_dependencies (const ucl_object_t *deps,
const ucl_object_t *obj, struct ucl_schema_error *err,
const ucl_object_t *root)
{
const ucl_object_t *elt, *cur, *cur_dep;
ucl_object_iter_t iter = NULL, piter;
bool ret = true;
while (ret && (cur = ucl_iterate_object (deps, &iter, true)) != NULL) {
elt = ucl_object_find_key (obj, ucl_object_key (cur));
if (elt != NULL) {
/* Need to check dependencies */
if (cur->type == UCL_ARRAY) {
piter = NULL;
while (ret && (cur_dep = ucl_iterate_object (cur, &piter, true)) != NULL) {
if (ucl_object_find_key (obj, ucl_object_tostring (cur_dep)) == NULL) {
ucl_schema_create_error (err, UCL_SCHEMA_MISSING_DEPENDENCY, elt,
"dependency %s is missing for key %s",
ucl_object_tostring (cur_dep), ucl_object_key (cur));
ret = false;
break;
}
}
}
else if (cur->type == UCL_OBJECT) {
ret = ucl_schema_validate (cur, obj, true, err, root);
}
}
}
return ret;
}
static void
parse_flags (struct fuzzy_rule *rule,
struct rspamd_config *cfg,
const ucl_object_t *val,
gint cb_id)
{
const ucl_object_t *elt;
struct fuzzy_mapping *map;
const gchar *sym = NULL;
if (val->type == UCL_STRING) {
msg_err_config (
"string mappings are deprecated and no longer supported, use new style configuration");
}
else if (val->type == UCL_OBJECT) {
elt = ucl_object_find_key (val, "symbol");
if (elt == NULL || !ucl_object_tostring_safe (elt, &sym)) {
sym = ucl_object_key (val);
}
if (sym != NULL) {
map =
rspamd_mempool_alloc (fuzzy_module_ctx->fuzzy_pool,
sizeof (struct fuzzy_mapping));
map->symbol = sym;
elt = ucl_object_find_key (val, "flag");
if (elt != NULL && ucl_obj_toint_safe (elt, &map->fuzzy_flag)) {
elt = ucl_object_find_key (val, "max_score");
if (elt != NULL) {
map->weight = ucl_obj_todouble (elt);
}
else {
map->weight = rule->max_score;
}
/* Add flag to hash table */
g_hash_table_insert (rule->mappings,
GINT_TO_POINTER (map->fuzzy_flag), map);
rspamd_symbols_cache_add_symbol (cfg->cache,
map->symbol, 0,
NULL, NULL,
SYMBOL_TYPE_VIRTUAL|SYMBOL_TYPE_FINE,
cb_id);
}
else {
msg_err_config ("fuzzy_map parameter has no flag definition");
}
}
else {
msg_err_config ("fuzzy_map parameter has no symbol definition");
}
}
else {
msg_err_config ("fuzzy_map parameter is of an unsupported type");
}
}
const pkg_object *
pkg_object_find(const pkg_object *o, const char *key)
{
if (o == NULL)
return (NULL);
return (ucl_object_find_key(o, key));
}
static ucl_object_t *
dynamic_metric_find_elt (const ucl_object_t *arr, const gchar *name)
{
ucl_object_iter_t it = NULL;
const ucl_object_t *cur, *n;
while ((cur = ucl_iterate_object (arr, &it, true)) != NULL) {
if (cur->type == UCL_OBJECT) {
n = ucl_object_find_key (cur, "name");
if (n && n->type == UCL_STRING &&
strcmp (name, ucl_object_tostring (n)) == 0) {
return (ucl_object_t *)ucl_object_find_key (cur, "value");
}
}
}
return NULL;
}
static double
get_specific_action_score (const ucl_object_t *metric,
struct metric_action *action)
{
const ucl_object_t *act, *sact;
double score;
if (metric) {
act = ucl_object_find_key (metric, "actions");
if (act) {
sact = ucl_object_find_key (act, rspamd_action_to_str (action->action));
if (sact != NULL && ucl_object_todouble_safe (sact, &score)) {
return score;
}
}
}
return action->score;
}
static double
get_specific_action_score (struct rspamd_task *task,
const ucl_object_t *metric,
struct metric_action *action)
{
const ucl_object_t *act, *sact;
const gchar *act_name;
double score;
if (metric) {
act = ucl_object_find_key (metric, "actions");
if (act) {
act_name = rspamd_action_to_str (action->action);
sact = ucl_object_find_key (act, act_name);
if (sact != NULL && ucl_object_todouble_safe (sact, &score)) {
msg_debug_task ("found override score %.2f for action %s in settings",
score, act_name);
return score;
}
}
}
return action->score;
}
gint
rspamd_check_action_metric (struct rspamd_task *task,
double score, double *rscore, struct metric *metric)
{
struct metric_action *action, *selected_action = NULL;
double max_score = 0;
const ucl_object_t *ms = NULL;
int i;
if (task->settings) {
ms = ucl_object_find_key (task->settings, metric->name);
}
for (i = METRIC_ACTION_REJECT; i < METRIC_ACTION_MAX; i++) {
double sc;
action = &metric->actions[i];
sc = get_specific_action_score (ms, action);
if (sc < 0) {
continue;
}
if (score >= sc && sc > max_score) {
selected_action = action;
max_score = sc;
}
if (rscore != NULL && i == METRIC_ACTION_REJECT) {
*rscore = sc;
}
}
if (selected_action) {
return selected_action->action;
}
return METRIC_ACTION_NOACTION;
}
/**
* Add action for specified metric
* @param cfg config file object
* @param metric metric's name
* @param action action's name
* @param value value of symbol
* @return
*/
gboolean
add_dynamic_action (struct rspamd_config *cfg,
const gchar *metric_name,
guint action,
gdouble value)
{
ucl_object_t *metric, *acts;
const gchar *action_name = rspamd_action_to_str (action);
if (cfg->dynamic_conf == NULL) {
msg_info ("dynamic conf is disabled");
return FALSE;
}
metric = dynamic_metric_find_metric (cfg->current_dynamic_conf,
metric_name);
if (metric == NULL) {
metric = new_dynamic_metric (metric_name, cfg->current_dynamic_conf);
}
acts = (ucl_object_t *)ucl_object_find_key (metric, "actions");
if (acts != NULL) {
ucl_object_t *act;
act = dynamic_metric_find_elt (acts, action_name);
if (act) {
act->value.dv = value;
}
else {
new_dynamic_elt (acts, action_name, value);
}
}
apply_dynamic_conf (cfg->current_dynamic_conf, cfg);
return TRUE;
}
/**
* Add symbol for specified metric
* @param cfg config file object
* @param metric metric's name
* @param symbol symbol's name
* @param value value of symbol
* @return
*/
gboolean
add_dynamic_symbol (struct rspamd_config *cfg,
const gchar *metric_name,
const gchar *symbol,
gdouble value)
{
ucl_object_t *metric, *syms;
if (cfg->dynamic_conf == NULL) {
msg_info ("dynamic conf is disabled");
return FALSE;
}
metric = dynamic_metric_find_metric (cfg->current_dynamic_conf,
metric_name);
if (metric == NULL) {
metric = new_dynamic_metric (metric_name, cfg->current_dynamic_conf);
}
syms = (ucl_object_t *)ucl_object_find_key (metric, "symbols");
if (syms != NULL) {
ucl_object_t *sym;
sym = dynamic_metric_find_elt (syms, symbol);
if (sym) {
sym->value.dv = value;
}
else {
new_dynamic_elt (syms, symbol, value);
}
}
apply_dynamic_conf (cfg->current_dynamic_conf, cfg);
return TRUE;
}
int
main(int argc, char **argv)
{
const char *fn = NULL;
unsigned char *inbuf;
struct ucl_parser *parser;
int k, ret = 0, r = 0;
ssize_t bufsize;
ucl_object_t *obj = NULL;
const ucl_object_t *par;
FILE *in;
if (argc > 1) {
fn = argv[1];
}
if (fn != NULL) {
in = fopen (fn, "r");
if (in == NULL) {
exit (-errno);
}
}
else {
in = stdin;
}
parser = ucl_parser_new (0);
inbuf = malloc (BUFSIZ);
bufsize = BUFSIZ;
r = 0;
while (!feof (in) && !ferror (in)) {
if (r == bufsize) {
inbuf = realloc (inbuf, bufsize * 2);
bufsize *= 2;
if (inbuf == NULL) {
perror ("realloc");
exit (EXIT_FAILURE);
}
}
r += fread (inbuf + r, 1, bufsize - r, in);
}
if (ferror (in)) {
fprintf (stderr, "Failed to read the input file.\n");
exit (EXIT_FAILURE);
}
ucl_parser_add_chunk (parser, inbuf, r);
fclose (in);
if (ucl_parser_get_error(parser)) {
printf ("Error occured: %s\n", ucl_parser_get_error(parser));
ret = 1;
goto end;
}
obj = ucl_parser_get_object (parser);
if (ucl_parser_get_error (parser)) {
printf ("Error occured: %s\n", ucl_parser_get_error(parser));
ret = 1;
goto end;
}
if (argc > 2) {
for (k = 2; k < argc; k++) {
printf ("search for \"%s\"... ", argv[k]);
par = ucl_object_find_key (obj, argv[k]);
printf ("%sfound\n", (par == NULL )?"not ":"");
ucl_obj_dump (par, 0);
}
}
else {
ucl_obj_dump (obj, 0);
}
end:
if (parser != NULL) {
ucl_parser_free (parser);
}
if (obj != NULL) {
ucl_object_unref (obj);
}
return ret;
}
gboolean
rspamd_config_is_module_enabled (struct rspamd_config *cfg,
const gchar *module_name)
{
gboolean is_c = FALSE;
struct metric *metric;
const ucl_object_t *conf, *enabled;
GList *cur;
struct rspamd_symbols_group *gr;
metric = cfg->default_metric;
if (g_hash_table_lookup (cfg->c_modules, module_name)) {
is_c = TRUE;
}
if (g_hash_table_lookup (cfg->explicit_modules, module_name) != NULL) {
/* Always load module */
return TRUE;
}
if (is_c) {
gboolean found = FALSE;
cur = g_list_first (cfg->filters);
while (cur) {
if (strcmp (cur->data, module_name) == 0) {
found = TRUE;
break;
}
cur = g_list_next (cur);
}
if (!found) {
msg_info_config ("internal module %s is disable in `filters` line",
module_name);
return FALSE;
}
}
conf = ucl_object_find_key (cfg->rcl_obj, module_name);
if (conf == NULL) {
msg_info_config ("%s module %s is enabled but has not been configured",
is_c ? "internal" : "lua", module_name);
if (!is_c) {
msg_info_config ("%s disabling unconfigured lua module", module_name);
return FALSE;
}
}
else {
enabled = ucl_object_find_key (conf, "enabled");
if (enabled && ucl_object_type (enabled) == UCL_BOOLEAN) {
if (!ucl_object_toboolean (enabled)) {
msg_info_config ("%s module %s is disabled in the configuration",
is_c ? "internal" : "lua", module_name);
return FALSE;
}
}
}
/* Now we check symbols group */
gr = g_hash_table_lookup (metric->groups, module_name);
if (gr) {
if (gr->disabled) {
msg_info_config ("%s module %s is disabled in the configuration as "
"its group has been disabled",
is_c ? "internal" : "lua", module_name);
return FALSE;
}
}
return TRUE;
}
void
rspamd_stat_init (struct rspamd_config *cfg, struct event_base *ev_base)
{
GList *cur, *curst;
struct rspamd_classifier_config *clf;
struct rspamd_statfile_config *stf;
struct rspamd_stat_backend *bk;
struct rspamd_statfile *st;
struct rspamd_classifier *cl;
const ucl_object_t *cache_obj = NULL, *cache_name_obj;
const gchar *cache_name = NULL;
if (stat_ctx == NULL) {
stat_ctx = g_slice_alloc0 (sizeof (*stat_ctx));
}
stat_ctx->backends_subrs = stat_backends;
stat_ctx->backends_count = G_N_ELEMENTS (stat_backends);
stat_ctx->classifiers_subrs = stat_classifiers;
stat_ctx->classifiers_count = G_N_ELEMENTS (stat_classifiers);
stat_ctx->tokenizers_subrs = stat_tokenizers;
stat_ctx->tokenizers_count = G_N_ELEMENTS (stat_tokenizers);
stat_ctx->caches_subrs = stat_caches;
stat_ctx->caches_count = G_N_ELEMENTS (stat_caches);
stat_ctx->cfg = cfg;
stat_ctx->statfiles = g_ptr_array_new ();
stat_ctx->classifiers = g_ptr_array_new ();
stat_ctx->async_elts = g_queue_new ();
stat_ctx->ev_base = ev_base;
REF_RETAIN (stat_ctx->cfg);
/* Create statfiles from the classifiers */
cur = cfg->classifiers;
while (cur) {
clf = cur->data;
bk = rspamd_stat_get_backend (clf->backend);
if (bk == NULL) {
msg_err_config ("cannot get backend of type %s, so disable classifier"
" %s completely", clf->backend, clf->name);
cur = g_list_next (cur);
continue;
}
/* XXX:
* Here we get the first classifier tokenizer config as the only one
* We NO LONGER support multiple tokenizers per rspamd instance
*/
if (stat_ctx->tkcf == NULL) {
stat_ctx->tokenizer = rspamd_stat_get_tokenizer (clf->tokenizer->name);
g_assert (stat_ctx->tokenizer != NULL);
stat_ctx->tkcf = stat_ctx->tokenizer->get_config (cfg->cfg_pool,
clf->tokenizer, NULL);
}
cl = g_slice_alloc0 (sizeof (*cl));
cl->cfg = clf;
cl->ctx = stat_ctx;
cl->statfiles_ids = g_array_new (FALSE, FALSE, sizeof (gint));
cl->subrs = rspamd_stat_get_classifier (clf->classifier);
g_assert (cl->subrs != NULL);
cl->subrs->init_func (cfg->cfg_pool, cl);
/* Init classifier cache */
cache_name = NULL;
if (clf->opts) {
cache_obj = ucl_object_find_key (clf->opts, "cache");
cache_name_obj = NULL;
if (cache_obj) {
cache_name_obj = ucl_object_find_any_key (cache_obj,
"name", "type", NULL);
}
if (cache_name_obj) {
cache_name = ucl_object_tostring (cache_name_obj);
}
}
if (cache_name == NULL) {
/* We assume that learn cache is the same as backend */
cache_name = clf->backend;
}
curst = clf->statfiles;
while (curst) {
stf = curst->data;
st = g_slice_alloc0 (sizeof (*st));
st->classifier = cl;
st->stcf = stf;
st->backend = bk;
st->bkcf = bk->init (stat_ctx, cfg, st);
msg_debug_config ("added backend %s for symbol %s",
bk->name, stf->symbol);
/* XXX: bad hack to pass statfiles configuration to cache */
if (cl->cache == NULL) {
cl->cache = rspamd_stat_get_cache (cache_name);
g_assert (cl->cache != NULL);
cl->cachecf = cl->cache->init (stat_ctx, cfg, st, cache_obj);
if (cl->cachecf == NULL) {
msg_err_config ("error adding cache %s for symbol %s",
cl->cache->name, stf->symbol);
cl->cache = NULL;
}
else {
msg_debug_config ("added cache %s for symbol %s",
cl->cache->name, stf->symbol);
}
}
if (st->bkcf == NULL) {
msg_err_config ("cannot init backend %s for statfile %s",
clf->backend, stf->symbol);
g_slice_free1 (sizeof (*st), st);
}
else {
st->id = stat_ctx->statfiles->len;
g_ptr_array_add (stat_ctx->statfiles, st);
g_array_append_val (cl->statfiles_ids, st->id);
}
curst = curst->next;
}
g_ptr_array_add (stat_ctx->classifiers, cl);
cur = cur->next;
}
}
/**
* Apply configuration to the specified configuration
* @param conf_metrics
* @param cfg
*/
static void
apply_dynamic_conf (const ucl_object_t *top, struct rspamd_config *cfg)
{
gint test_act;
const ucl_object_t *cur_elt, *cur_nm, *it_val;
ucl_object_iter_t it = NULL;
struct metric *real_metric;
struct metric_action *cur_action;
struct rspamd_symbol_def *s;
while ((cur_elt = ucl_iterate_object (top, &it, true))) {
if (ucl_object_type (cur_elt) != UCL_OBJECT) {
msg_err ("loaded json array element is not an object");
continue;
}
cur_nm = ucl_object_find_key (cur_elt, "metric");
if (!cur_nm || ucl_object_type (cur_nm) != UCL_STRING) {
msg_err (
"loaded json metric object element has no 'metric' attribute");
continue;
}
real_metric = g_hash_table_lookup (cfg->metrics,
ucl_object_tostring (cur_nm));
if (real_metric == NULL) {
msg_warn ("cannot find metric %s", ucl_object_tostring (cur_nm));
continue;
}
cur_nm = ucl_object_find_key (cur_elt, "symbols");
/* Parse symbols */
if (cur_nm && ucl_object_type (cur_nm) == UCL_ARRAY) {
ucl_object_iter_t nit = NULL;
while ((it_val = ucl_iterate_object (cur_nm, &nit, true))) {
if (ucl_object_find_key (it_val, "name") &&
ucl_object_find_key (it_val, "value")) {
const ucl_object_t *n =
ucl_object_find_key (it_val, "name");
const ucl_object_t *v =
ucl_object_find_key (it_val, "value");
if((s = g_hash_table_lookup (real_metric->symbols,
ucl_object_tostring (n))) != NULL) {
*s->weight_ptr = ucl_object_todouble (v);
}
}
else {
msg_info (
"json symbol object has no mandatory 'name' and 'value' attributes");
}
}
}
else {
ucl_object_t *arr;
arr = ucl_object_typed_new (UCL_ARRAY);
ucl_object_insert_key ((ucl_object_t *)cur_elt, arr, "symbols",
sizeof ("symbols") - 1, false);
}
cur_nm = ucl_object_find_key (cur_elt, "actions");
/* Parse actions */
if (cur_nm && ucl_object_type (cur_nm) == UCL_ARRAY) {
ucl_object_iter_t nit = NULL;
while ((it_val = ucl_iterate_object (cur_nm, &nit, true))) {
if (ucl_object_find_key (it_val, "name") &&
ucl_object_find_key (it_val, "value")) {
if (!rspamd_action_from_str (ucl_object_tostring (
ucl_object_find_key (it_val, "name")), &test_act)) {
msg_err ("unknown action: %s",
ucl_object_tostring (ucl_object_find_key (it_val,
"name")));
continue;
}
cur_action = &real_metric->actions[test_act];
cur_action->action = test_act;
cur_action->score =
ucl_object_todouble (ucl_object_find_key (it_val,
"value"));
}
else {
msg_info (
"json action object has no mandatory 'name' and 'value' attributes");
}
}
}
else {
ucl_object_t *arr;
arr = ucl_object_typed_new (UCL_ARRAY);
ucl_object_insert_key ((ucl_object_t *)cur_elt, arr, "actions",
sizeof ("actions") - 1, false);
}
}
}
int
pkg_ini(const char *path, const char *reposdir, pkg_init_flags flags)
{
struct ucl_parser *p = NULL;
size_t i;
const char *val = NULL;
const char *buf, *walk, *value, *key, *k;
const char *evkey = NULL;
const char *nsname = NULL;
const char *evpipe = NULL;
const ucl_object_t *cur, *object;
ucl_object_t *obj = NULL, *o, *ncfg;
ucl_object_iter_t it = NULL;
struct sbuf *ukey = NULL;
bool fatal_errors = false;
k = NULL;
o = NULL;
pkg_get_myarch(myabi, BUFSIZ);
pkg_get_myarch_legacy(myabi_legacy, BUFSIZ);
if (parsed != false) {
pkg_emit_error("pkg_init() must only be called once");
return (EPKG_FATAL);
}
if (((flags & PKG_INIT_FLAG_USE_IPV4) == PKG_INIT_FLAG_USE_IPV4) &&
((flags & PKG_INIT_FLAG_USE_IPV6) == PKG_INIT_FLAG_USE_IPV6)) {
pkg_emit_error("Invalid flags for pkg_init()");
return (EPKG_FATAL);
}
config = ucl_object_typed_new(UCL_OBJECT);
for (i = 0; i < c_size; i++) {
switch (c[i].type) {
case PKG_STRING:
obj = ucl_object_fromstring_common(
c[i].def != NULL ? c[i].def : "", 0, UCL_STRING_TRIM);
ucl_object_insert_key(config, obj,
c[i].key, strlen(c[i].key), false);
break;
case PKG_INT:
ucl_object_insert_key(config,
ucl_object_fromstring_common(c[i].def, 0, UCL_STRING_PARSE_INT),
c[i].key, strlen(c[i].key), false);
break;
case PKG_BOOL:
ucl_object_insert_key(config,
ucl_object_fromstring_common(c[i].def, 0, UCL_STRING_PARSE_BOOLEAN),
c[i].key, strlen(c[i].key), false);
break;
case PKG_OBJECT:
obj = ucl_object_typed_new(UCL_OBJECT);
if (c[i].def != NULL) {
walk = buf = c[i].def;
while ((buf = strchr(buf, ',')) != NULL) {
key = walk;
value = walk;
while (*value != ',') {
if (*value == '=')
break;
value++;
}
ucl_object_insert_key(obj,
ucl_object_fromstring_common(value + 1, buf - value - 1, UCL_STRING_TRIM),
key, value - key, false);
buf++;
walk = buf;
}
key = walk;
value = walk;
while (*value != ',') {
if (*value == '=')
break;
value++;
}
if (o == NULL)
o = ucl_object_typed_new(UCL_OBJECT);
ucl_object_insert_key(o,
ucl_object_fromstring_common(value + 1, strlen(value + 1), UCL_STRING_TRIM),
key, value - key, false);
}
ucl_object_insert_key(config, obj,
c[i].key, strlen(c[i].key), false);
break;
case PKG_ARRAY:
obj = ucl_object_typed_new(UCL_ARRAY);
if (c[i].def != NULL) {
walk = buf = c[i].def;
while ((buf = strchr(buf, ',')) != NULL) {
ucl_array_append(obj,
ucl_object_fromstring_common(walk, buf - walk, UCL_STRING_TRIM));
buf++;
walk = buf;
}
ucl_array_append(obj,
ucl_object_fromstring_common(walk, strlen(walk), UCL_STRING_TRIM));
}
ucl_object_insert_key(config, obj,
c[i].key, strlen(c[i].key), false);
break;
}
}
if (path == NULL)
path = PREFIX"/etc/pkg.conf";
p = ucl_parser_new(0);
errno = 0;
obj = NULL;
if (!ucl_parser_add_file(p, path)) {
if (errno != ENOENT)
pkg_emit_error("Invalid configuration file: %s", ucl_parser_get_error(p));
} else {
obj = ucl_parser_get_object(p);
}
ncfg = NULL;
while (obj != NULL && (cur = ucl_iterate_object(obj, &it, true))) {
sbuf_init(&ukey);
key = ucl_object_key(cur);
for (i = 0; key[i] != '\0'; i++)
sbuf_putc(ukey, toupper(key[i]));
sbuf_finish(ukey);
object = ucl_object_find_keyl(config, sbuf_data(ukey), sbuf_len(ukey));
if (strncasecmp(sbuf_data(ukey), "PACKAGESITE", sbuf_len(ukey))
== 0 || strncasecmp(sbuf_data(ukey), "PUBKEY",
sbuf_len(ukey)) == 0 || strncasecmp(sbuf_data(ukey),
"MIRROR_TYPE", sbuf_len(ukey)) == 0) {
pkg_emit_error("%s in pkg.conf is no longer "
"supported. Convert to the new repository style."
" See pkg.conf(5)", sbuf_data(ukey));
fatal_errors = true;
continue;
}
/* ignore unknown keys */
if (object == NULL)
continue;
if (object->type != cur->type) {
pkg_emit_error("Malformed key %s, ignoring", key);
continue;
}
if (ncfg == NULL)
ncfg = ucl_object_typed_new(UCL_OBJECT);
ucl_object_insert_key(ncfg, ucl_object_copy(cur), sbuf_data(ukey), sbuf_len(ukey), true);
}
if (fatal_errors) {
ucl_object_unref(ncfg);
ucl_parser_free(p);
return (EPKG_FATAL);
}
if (ncfg != NULL) {
it = NULL;
while (( cur = ucl_iterate_object(ncfg, &it, true))) {
key = ucl_object_key(cur);
ucl_object_replace_key(config, ucl_object_ref(cur), key, strlen(key), true);
}
ucl_object_unref(ncfg);
}
ncfg = NULL;
it = NULL;
while ((cur = ucl_iterate_object(config, &it, true))) {
o = NULL;
key = ucl_object_key(cur);
val = getenv(key);
if (val == NULL)
continue;
switch (cur->type) {
case UCL_STRING:
o = ucl_object_fromstring_common(val, 0, UCL_STRING_TRIM);
break;
case UCL_INT:
o = ucl_object_fromstring_common(val, 0, UCL_STRING_PARSE_INT);
if (o->type != UCL_INT) {
pkg_emit_error("Invalid type for environment "
"variable %s, got %s, while expecting an integer",
key, val);
ucl_object_unref(o);
continue;
}
break;
case UCL_BOOLEAN:
o = ucl_object_fromstring_common(val, 0, UCL_STRING_PARSE_BOOLEAN);
if (o->type != UCL_BOOLEAN) {
pkg_emit_error("Invalid type for environment "
"variable %s, got %s, while expecting a boolean",
key, val);
ucl_object_unref(o);
continue;
}
break;
case UCL_OBJECT:
o = ucl_object_typed_new(UCL_OBJECT);
walk = buf = val;
while ((buf = strchr(buf, ',')) != NULL) {
k = walk;
value = walk;
while (*value != ',') {
if (*value == '=')
break;
value++;
}
ucl_object_insert_key(o,
ucl_object_fromstring_common(value + 1, buf - value - 1, UCL_STRING_TRIM),
k, value - k, false);
buf++;
walk = buf;
}
key = walk;
value = walk;
while (*value != '\0') {
if (*value == '=')
break;
value++;
}
ucl_object_insert_key(o,
ucl_object_fromstring_common(value + 1, strlen(value + 1), UCL_STRING_TRIM),
k, value - k, false);
break;
case UCL_ARRAY:
o = ucl_object_typed_new(UCL_ARRAY);
walk = buf = val;
while ((buf = strchr(buf, ',')) != NULL) {
ucl_array_append(o,
ucl_object_fromstring_common(walk, buf - walk, UCL_STRING_TRIM));
buf++;
walk = buf;
}
ucl_array_append(o,
ucl_object_fromstring_common(walk, strlen(walk), UCL_STRING_TRIM));
break;
default:
/* ignore other types */
break;
}
if (o != NULL) {
if (ncfg == NULL)
ncfg = ucl_object_typed_new(UCL_OBJECT);
ucl_object_insert_key(ncfg, o, key, strlen(key), true);
}
}
if (ncfg != NULL) {
it = NULL;
while (( cur = ucl_iterate_object(ncfg, &it, true))) {
key = ucl_object_key(cur);
ucl_object_replace_key(config, ucl_object_ref(cur), key, strlen(key), true);
}
ucl_object_unref(ncfg);
}
disable_plugins_if_static();
parsed = true;
ucl_object_unref(obj);
ucl_parser_free(p);
if (strcmp(pkg_object_string(pkg_config_get("ABI")), "unknown") == 0) {
pkg_emit_error("Unable to determine ABI");
return (EPKG_FATAL);
}
pkg_debug(1, "%s", "pkg initialized");
/* Start the event pipe */
evpipe = pkg_object_string(pkg_config_get("EVENT_PIPE"));
if (evpipe != NULL)
connect_evpipe(evpipe);
debug_level = pkg_object_int(pkg_config_get("DEBUG_LEVEL"));
it = NULL;
object = ucl_object_find_key(config, "PKG_ENV");
while ((cur = ucl_iterate_object(object, &it, true))) {
evkey = ucl_object_key(cur);
pkg_debug(1, "Setting env var: %s", evkey);
if (evkey != NULL && evkey[0] != '\0')
setenv(evkey, ucl_object_tostring_forced(cur), 1);
}
/* load the repositories */
load_repositories(reposdir, flags);
setenv("HTTP_USER_AGENT", "pkg/"PKGVERSION, 1);
/* bypass resolv.conf with specified NAMESERVER if any */
nsname = pkg_object_string(pkg_config_get("NAMESERVER"));
if (nsname != NULL)
set_nameserver(ucl_object_tostring_forced(o));
return (EPKG_OK);
}
const pkg_object *
pkg_config_get(const char *key) {
return (ucl_object_find_key(config, key));
}
gboolean
rspamd_symbols_cache_process_symbols (struct rspamd_task * task,
struct symbols_cache *cache)
{
struct cache_item *item = NULL;
struct cache_savepoint *checkpoint;
gint i;
g_assert (cache != NULL);
if (task->checkpoint == NULL) {
checkpoint = rspamd_mempool_alloc0 (task->task_pool, sizeof (*checkpoint));
/* Bit 0: check started, Bit 1: check finished */
checkpoint->processed_bits = rspamd_mempool_alloc0 (task->task_pool,
NBYTES (cache->used_items) * 2);
checkpoint->waitq = g_ptr_array_new ();
rspamd_mempool_add_destructor (task->task_pool,
rspamd_ptr_array_free_hard, checkpoint->waitq);
task->checkpoint = checkpoint;
rspamd_create_metric_result (task, DEFAULT_METRIC);
if (task->settings) {
const ucl_object_t *wl;
wl = ucl_object_find_key (task->settings, "whitelist");
if (wl != NULL) {
msg_info ("<%s> is whitelisted", task->message_id);
task->flags |= RSPAMD_TASK_FLAG_SKIP;
return TRUE;
}
}
}
else {
checkpoint = task->checkpoint;
}
msg_debug ("symbols processing stage at pass: %d", checkpoint->pass);
if (checkpoint->pass == 0) {
/*
* On the first pass we check symbols that do not have dependencies
* If we figure out symbol that has no dependencies satisfied, then
* we just save it for another pass
*/
for (i = 0; i < (gint)cache->used_items; i ++) {
if (rspamd_symbols_cache_metric_limit (task, checkpoint)) {
msg_info ("<%s> has already scored more than %.2f, so do not "
"plan any more checks", task->message_id,
checkpoint->rs->score);
return TRUE;
}
item = g_ptr_array_index (cache->items_by_order, i);
if (!isset (checkpoint->processed_bits, item->id * 2)) {
if (!rspamd_symbols_cache_check_deps (task, cache, item,
checkpoint)) {
msg_debug ("blocked execution of %d unless deps are resolved",
item->id);
g_ptr_array_add (checkpoint->waitq, item);
continue;
}
rspamd_symbols_cache_check_symbol (task, cache, item, checkpoint);
}
}
checkpoint->pass ++;
}
else {
/* We just go through the blocked symbols and check if they are ready */
for (i = 0; i < (gint)checkpoint->waitq->len; i ++) {
item = g_ptr_array_index (checkpoint->waitq, i);
if (!isset (checkpoint->processed_bits, item->id * 2)) {
if (!rspamd_symbols_cache_check_deps (task, cache, item,
checkpoint)) {
continue;
}
rspamd_symbols_cache_check_symbol (task, cache, item, checkpoint);
}
}
}
return TRUE;
}
gint
regexp_module_config (struct rspamd_config *cfg)
{
struct regexp_module_item *cur_item;
const ucl_object_t *sec, *value, *elt;
ucl_object_iter_t it = NULL;
gint res = TRUE, id;
if (!rspamd_config_is_module_enabled (cfg, "regexp")) {
return TRUE;
}
sec = ucl_object_find_key (cfg->rcl_obj, "regexp");
if (sec == NULL) {
msg_err_config ("regexp module enabled, but no rules are defined");
return TRUE;
}
regexp_module_ctx->max_size = 0;
while ((value = ucl_iterate_object (sec, &it, true)) != NULL) {
if (g_ascii_strncasecmp (ucl_object_key (value), "max_size",
sizeof ("max_size") - 1) == 0) {
regexp_module_ctx->max_size = ucl_obj_toint (value);
rspamd_mime_expression_set_re_limit (regexp_module_ctx->max_size);
}
else if (g_ascii_strncasecmp (ucl_object_key (value), "max_threads",
sizeof ("max_threads") - 1) == 0) {
msg_warn_config ("regexp module is now single threaded, max_threads is ignored");
}
else if (value->type == UCL_STRING) {
cur_item = rspamd_mempool_alloc0 (regexp_module_ctx->regexp_pool,
sizeof (struct regexp_module_item));
cur_item->symbol = ucl_object_key (value);
if (!read_regexp_expression (regexp_module_ctx->regexp_pool,
cur_item, ucl_object_key (value),
ucl_obj_tostring (value), cfg)) {
res = FALSE;
}
else {
rspamd_symbols_cache_add_symbol (cfg->cache,
cur_item->symbol,
0,
process_regexp_item,
cur_item,
SYMBOL_TYPE_NORMAL, -1);
}
}
else if (value->type == UCL_USERDATA) {
/* Just a lua function */
cur_item = rspamd_mempool_alloc0 (regexp_module_ctx->regexp_pool,
sizeof (struct regexp_module_item));
cur_item->symbol = ucl_object_key (value);
cur_item->lua_function = ucl_object_toclosure (value);
rspamd_symbols_cache_add_symbol (cfg->cache,
cur_item->symbol,
0,
process_regexp_item,
cur_item,
SYMBOL_TYPE_NORMAL, -1);
}
else if (value->type == UCL_OBJECT) {
const gchar *description = NULL, *group = NULL,
*metric = DEFAULT_METRIC;
gdouble score = 0.0;
gboolean one_shot = FALSE, is_lua = FALSE, valid_expression = TRUE;
/* We have some lua table, extract its arguments */
elt = ucl_object_find_key (value, "callback");
if (elt == NULL || elt->type != UCL_USERDATA) {
/* Try plain regexp expression */
elt = ucl_object_find_any_key (value, "regexp", "re", NULL);
if (elt != NULL && ucl_object_type (elt) == UCL_STRING) {
cur_item = rspamd_mempool_alloc0 (regexp_module_ctx->regexp_pool,
sizeof (struct regexp_module_item));
cur_item->symbol = ucl_object_key (value);
if (!read_regexp_expression (regexp_module_ctx->regexp_pool,
cur_item, ucl_object_key (value),
ucl_obj_tostring (elt), cfg)) {
res = FALSE;
}
else {
valid_expression = TRUE;
}
}
else {
msg_err_config (
"no callback/expression defined for regexp symbol: "
"%s", ucl_object_key (value));
}
}
else {
is_lua = TRUE;
cur_item = rspamd_mempool_alloc0 (
regexp_module_ctx->regexp_pool,
sizeof (struct regexp_module_item));
cur_item->symbol = ucl_object_key (value);
cur_item->lua_function = ucl_object_toclosure (value);
}
if (cur_item && (is_lua || valid_expression)) {
id = rspamd_symbols_cache_add_symbol (cfg->cache,
cur_item->symbol,
0,
process_regexp_item,
cur_item,
SYMBOL_TYPE_NORMAL, -1);
elt = ucl_object_find_key (value, "condition");
if (elt != NULL && ucl_object_type (elt) == UCL_USERDATA) {
struct ucl_lua_funcdata *conddata;
conddata = ucl_object_toclosure (elt);
rspamd_symbols_cache_add_condition (cfg->cache, id,
conddata->L, conddata->idx);
}
elt = ucl_object_find_key (value, "metric");
if (elt) {
metric = ucl_object_tostring (elt);
}
elt = ucl_object_find_key (value, "description");
if (elt) {
description = ucl_object_tostring (elt);
}
elt = ucl_object_find_key (value, "group");
if (elt) {
group = ucl_object_tostring (elt);
}
elt = ucl_object_find_key (value, "score");
if (elt) {
score = ucl_object_todouble (elt);
}
elt = ucl_object_find_key (value, "one_shot");
if (elt) {
one_shot = ucl_object_toboolean (elt);
}
rspamd_config_add_metric_symbol (cfg, metric, cur_item->symbol,
score, description, group, one_shot, FALSE);
}
}
else {
msg_warn_config ("unknown type of attribute %s for regexp module",
ucl_object_key (value));
}
}
return res;
}
static void
rspamadm_configdump (gint argc, gchar **argv)
{
GOptionContext *context;
GError *error = NULL;
const gchar *confdir;
const ucl_object_t *obj, *cur;
struct rspamd_config *cfg = rspamd_main->cfg;
gboolean ret = TRUE;
worker_t **pworker;
gchar **sec;
context = g_option_context_new (
"keypair - create encryption keys");
g_option_context_set_summary (context,
"Summary:\n Rspamd administration utility version "
RVERSION
"\n Release id: "
RID);
g_option_context_add_main_entries (context, entries, NULL);
if (!g_option_context_parse (context, &argc, &argv, &error)) {
fprintf (stderr, "option parsing failed: %s\n", error->message);
g_error_free (error);
exit (1);
}
if (config == NULL) {
if ((confdir = g_hash_table_lookup (ucl_vars, "CONFDIR")) == NULL) {
confdir = RSPAMD_CONFDIR;
}
config = g_strdup_printf ("%s%c%s", confdir, G_DIR_SEPARATOR,
"rspamd.conf");
}
pworker = &workers[0];
while (*pworker) {
/* Init string quarks */
(void) g_quark_from_static_string ((*pworker)->name);
pworker++;
}
cfg->cache = rspamd_symbols_cache_new (cfg);
cfg->compiled_modules = modules;
cfg->compiled_workers = workers;
cfg->cfg_name = config;
if (!rspamd_config_read (cfg, cfg->cfg_name, NULL,
config_logger, rspamd_main, ucl_vars)) {
ret = FALSE;
}
else {
/* Do post-load actions */
rspamd_lua_post_load_config (cfg);
if (!rspamd_init_filters (rspamd_main->cfg, FALSE)) {
ret = FALSE;
}
if (ret) {
ret = rspamd_config_post_load (cfg, FALSE);
}
}
if (ret) {
/* Output configuration */
if (!sections) {
rspamadm_dump_section_obj (cfg->rcl_obj);
}
else {
sec = sections;
while (*sec != NULL) {
obj = ucl_object_find_key (cfg->rcl_obj, *sec);
if (!obj) {
rspamd_printf ("Section %s NOT FOUND\n", *sec);
}
else {
LL_FOREACH (obj, cur) {
if (!json && !compact) {
rspamd_printf ("*** Section %s ***\n", *sec);
}
rspamadm_dump_section_obj (cur);
if (!json && !compact) {
rspamd_printf ("*** End of section %s ***\n", *sec);
}
else {
rspamd_printf ("\n");
}
}
}
sec ++;
}
g_strfreev (sections);
}
}
if (!ret) {
exit (EXIT_FAILURE);
}
exit (EXIT_SUCCESS);
}
/**
* Entry point.
*/
int main(int argc, char **argv)
{
// Parse program options
char *config_file = NULL;
bool server = false;
bool collector = false;
bool callibrator = false;
bool status_only = false;
int log_option = 0;
char c;
while ((c = getopt(argc, argv, "hc:sdflr")) != EOF) {
switch (c) {
case 'h': {
show_help(argv[0]);
return 1;
}
case 'c': config_file = strdup(optarg); break;
case 's': status_only = true; break;
case 'd': server = true; break;
case 'l': collector = true; break;
case 'r': callibrator = true; break;
case 'f': log_option |= LOG_PERROR; break;
default: {
fprintf(stderr, "ERROR: Invalid option %c!\n", c);
show_help(argv[0]);
return 1;
}
}
}
if (config_file == NULL)
config_file = strdup("/etc/koruza.cfg");
// Load the configuration file
struct ucl_parser *parser = ucl_parser_new(UCL_PARSER_KEY_LOWERCASE);
ucl_object_t *config = NULL;
ucl_object_t *obj = NULL;
int ret_value = 0;
if (!parser) {
fprintf(stderr, "ERROR: Failed to initialize configuration parser!\n");
return 2;
}
if (!ucl_parser_add_file(parser, config_file)) {
fprintf(stderr, "ERROR: Failed to parse configuration file '%s'!\n", config_file);
fprintf(stderr, "ERROR: %s\n", ucl_parser_get_error(parser));
ret_value = 2;
goto cleanup_exit;
} else {
config = ucl_parser_get_object(parser);
}
if (server) {
obj = ucl_object_find_key(config, "server");
if (!obj) {
fprintf(stderr, "ERROR: Missing server configuration!\n");
ret_value = 2;
goto cleanup_exit;
}
start_server(obj, log_option);
} else if (collector) {
start_collector(config, log_option);
} else if (callibrator) {
start_callibrator(config, log_option);
} else {
start_controller(config, status_only);
}
cleanup_exit:
// Cleanup and exit
if (config)
ucl_object_free(config);
if (parser)
ucl_parser_free(parser);
return ret_value;
}
/*
* Validate object
*/
static bool
ucl_schema_validate_object (const ucl_object_t *schema,
const ucl_object_t *obj, struct ucl_schema_error *err,
const ucl_object_t *root)
{
const ucl_object_t *elt, *prop, *found, *additional_schema = NULL,
*required = NULL, *pat, *pelt;
ucl_object_iter_t iter = NULL, piter = NULL;
bool ret = true, allow_additional = true;
int64_t minmax;
while (ret && (elt = ucl_iterate_object (schema, &iter, true)) != NULL) {
if (elt->type == UCL_OBJECT &&
strcmp (ucl_object_key (elt), "properties") == 0) {
piter = NULL;
while (ret && (prop = ucl_iterate_object (elt, &piter, true)) != NULL) {
found = ucl_object_find_key (obj, ucl_object_key (prop));
if (found) {
ret = ucl_schema_validate (prop, found, true, err, root);
}
}
}
else if (strcmp (ucl_object_key (elt), "additionalProperties") == 0) {
if (elt->type == UCL_BOOLEAN) {
if (!ucl_object_toboolean (elt)) {
/* Deny additional fields completely */
allow_additional = false;
}
}
else if (elt->type == UCL_OBJECT) {
/* Define validator for additional fields */
additional_schema = elt;
}
else {
ucl_schema_create_error (err, UCL_SCHEMA_INVALID_SCHEMA, elt,
"additionalProperties attribute is invalid in schema");
ret = false;
break;
}
}
else if (strcmp (ucl_object_key (elt), "required") == 0) {
if (elt->type == UCL_ARRAY) {
required = elt;
}
else {
ucl_schema_create_error (err, UCL_SCHEMA_INVALID_SCHEMA, elt,
"required attribute is invalid in schema");
ret = false;
break;
}
}
else if (strcmp (ucl_object_key (elt), "minProperties") == 0
&& ucl_object_toint_safe (elt, &minmax)) {
if (obj->len < minmax) {
ucl_schema_create_error (err, UCL_SCHEMA_CONSTRAINT, obj,
"object has not enough properties: %u, minimum is: %u",
obj->len, (unsigned)minmax);
ret = false;
break;
}
}
else if (strcmp (ucl_object_key (elt), "maxProperties") == 0
&& ucl_object_toint_safe (elt, &minmax)) {
if (obj->len > minmax) {
ucl_schema_create_error (err, UCL_SCHEMA_CONSTRAINT, obj,
"object has too many properties: %u, maximum is: %u",
obj->len, (unsigned)minmax);
ret = false;
break;
}
}
else if (strcmp (ucl_object_key (elt), "patternProperties") == 0) {
piter = NULL;
while (ret && (prop = ucl_iterate_object (elt, &piter, true)) != NULL) {
found = ucl_schema_test_pattern (obj, ucl_object_key (prop));
if (found) {
ret = ucl_schema_validate (prop, found, true, err, root);
}
}
}
else if (elt->type == UCL_OBJECT &&
strcmp (ucl_object_key (elt), "dependencies") == 0) {
ret = ucl_schema_validate_dependencies (elt, obj, err, root);
}
}
if (ret) {
/* Additional properties */
if (!allow_additional || additional_schema != NULL) {
/* Check if we have exactly the same properties in schema and object */
iter = NULL;
prop = ucl_object_find_key (schema, "properties");
while ((elt = ucl_iterate_object (obj, &iter, true)) != NULL) {
found = ucl_object_find_key (prop, ucl_object_key (elt));
if (found == NULL) {
/* Try patternProperties */
piter = NULL;
pat = ucl_object_find_key (schema, "patternProperties");
while ((pelt = ucl_iterate_object (pat, &piter, true)) != NULL) {
found = ucl_schema_test_pattern (obj, ucl_object_key (pelt));
if (found != NULL) {
break;
}
}
}
if (found == NULL) {
if (!allow_additional) {
ucl_schema_create_error (err, UCL_SCHEMA_CONSTRAINT, obj,
"object has non-allowed property %s",
ucl_object_key (elt));
ret = false;
break;
}
else if (additional_schema != NULL) {
if (!ucl_schema_validate (additional_schema, elt, true, err, root)) {
ret = false;
break;
}
}
}
}
}
/* Required properties */
if (required != NULL) {
iter = NULL;
while ((elt = ucl_iterate_object (required, &iter, true)) != NULL) {
if (ucl_object_find_key (obj, ucl_object_tostring (elt)) == NULL) {
ucl_schema_create_error (err, UCL_SCHEMA_MISSING_PROPERTY, obj,
"object has missing property %s",
ucl_object_tostring (elt));
ret = false;
break;
}
}
}
}
return ret;
}
static int
parse_modules(struct rtpp_cfg_stable *csp, const ucl_object_t *wop)
{
ucl_object_iter_t it_conf;
const ucl_object_t *obj_file;
const char *cf_key;
const ucl_object_t *obj_key;
int ecode, success;
void *confp;
const conf_helper_map *fent, *map;
struct rtpp_module_conf *mcp;
char mpath[PATH_MAX + 1];
const char *cp, *mp;
struct rtpp_module_if *mif;
it_conf = ucl_object_iterate_new(wop);
if (it_conf == NULL)
return (-1);
ecode = 0;
while ((obj_file = ucl_object_iterate_safe(it_conf, true)) != NULL) {
cf_key = ucl_object_key(obj_file);
RTPP_LOG(csp->glog, RTPP_LOG_DBUG, "\tmodule: %s", cf_key);
obj_key = ucl_object_find_key(obj_file, "load");
if (obj_key == NULL) {
cp = rtpp_module_dsop_canonic(cf_key, mpath, sizeof(mpath));
if (cp == NULL) {
RTPP_LOG(csp->glog, RTPP_LOG_ERR, "Error: Unable to find load parameter in module: %s", cf_key);
ecode = -1;
goto e0;
}
} else {
if (obj_key->type != UCL_STRING) {
RTPP_LOG(csp->glog, RTPP_LOG_ERR, "Error: \"load\" parameter in %s has a wrong type, string is expected", cf_key);
ecode = -1;
goto e0;
}
mp = ucl_object_tostring(obj_key);
cp = realpath(mp, mpath);
if (cp == NULL) {
RTPP_ELOG(csp->glog, RTPP_LOG_ERR, "realpath() failed: %s", mp);
ecode = -1;
goto e0;
}
}
mif = rtpp_module_if_ctor(cp);
if (mif == NULL) {
RTPP_LOG(csp->glog, RTPP_LOG_ERR, "dymanic module constructor has failed: %s", cp);
ecode = -1;
goto e0;
}
if (CALL_METHOD(mif, load, csp, csp->glog) != 0) {
RTPP_LOG(csp->glog, RTPP_LOG_ERR, "%p->load() method has failed: %s", mif, cp);
goto e1;
}
if (CALL_METHOD(mif, get_mconf, &mcp) < 0) {
RTPP_LOG(csp->glog, RTPP_LOG_ERR, "%p->get_mconf() method has failed: %s", mif, cp);
goto e1;
}
fent = NULL;
if (mcp != NULL) {
map = mcp->conf_map;
confp = mcp->conf_data;
} else {
map = default_module_map;
confp = NULL;
}
success = conf_helper_mapper(csp->glog, obj_file, map, confp, &fent);
if (!success) {
RTPP_LOG(csp->glog, RTPP_LOG_ERR, "Config parsing issue in section %s",
cf_key);
if (fent != NULL && fent->conf_key != NULL) {
RTPP_LOG(csp->glog, RTPP_LOG_ERR, "\tparameter %s", fent->conf_key);
}
goto e1;
}
if (CALL_METHOD(mif, config) < 0) {
RTPP_LOG(csp->glog, RTPP_LOG_ERR, "%p->config() method has failed: %s", mif, cp);
goto e1;
}
rtpp_list_append(csp->modules_cf, mif);
continue;
e1:
ecode = -1;
CALL_SMETHOD(mif->rcnt, decref);
goto e0;
}
e0:
if (ucl_object_iter_chk_excpn(it_conf)) {
RTPP_LOG(csp->glog, RTPP_LOG_ERR, "UCL has failed with an internal error");
ecode = -1;
}
ucl_object_iterate_free(it_conf);
return (ecode);
}
int
main(int argc, char **argv)
{
const char *fn = NULL;
char inbuf[8192];
struct ucl_parser *parser;
int k, ret = 0, r = 0;
ucl_object_t *obj = NULL;
const ucl_object_t *par;
FILE *in;
if (argc > 1) {
fn = argv[1];
}
if (fn != NULL) {
in = fopen (fn, "r");
if (in == NULL) {
exit (-errno);
}
}
else {
in = stdin;
}
parser = ucl_parser_new (0);
while (!feof (in) && r < (int)sizeof (inbuf)) {
r += fread (inbuf + r, 1, sizeof (inbuf) - r, in);
}
ucl_parser_add_chunk (parser, inbuf, r);
fclose (in);
if (ucl_parser_get_error(parser)) {
printf ("Error occured: %s\n", ucl_parser_get_error(parser));
ret = 1;
goto end;
}
obj = ucl_parser_get_object (parser);
if (ucl_parser_get_error (parser)) {
printf ("Error occured: %s\n", ucl_parser_get_error(parser));
ret = 1;
goto end;
}
if (argc > 2) {
for (k = 2; k < argc; k++) {
printf ("search for \"%s\"... ", argv[k]);
par = ucl_object_find_key (obj, argv[k]);
printf ("%sfound\n", (par == NULL )?"not ":"");
ucl_obj_dump (par, 0);
}
}
else {
ucl_obj_dump (obj, 0);
}
end:
if (parser != NULL) {
ucl_parser_free (parser);
}
if (obj != NULL) {
ucl_object_unref (obj);
}
return ret;
}
static void
add_repo(const ucl_object_t *obj, struct pkg_repo *r, const char *rname, pkg_init_flags flags)
{
const ucl_object_t *cur, *enabled;
ucl_object_iter_t it = NULL;
bool enable = true;
const char *url = NULL, *pubkey = NULL, *mirror_type = NULL;
const char *signature_type = NULL, *fingerprints = NULL;
const char *key;
const char *type = NULL;
int use_ipvx = 0;
pkg_debug(1, "PkgConfig: parsing repository object %s", rname);
enabled = ucl_object_find_key(obj, "enabled");
if (enabled == NULL)
enabled = ucl_object_find_key(obj, "ENABLED");
if (enabled != NULL) {
enable = ucl_object_toboolean(enabled);
if (!enable && r == NULL) {
pkg_debug(1, "PkgConfig: skipping disabled repo %s", rname);
return;
}
else if (!enable && r != NULL) {
/*
* We basically want to remove the existing repo r and
* forget all stuff parsed
*/
pkg_debug(1, "PkgConfig: disabling repo %s", rname);
HASH_DEL(repos, r);
pkg_repo_free(r);
return;
}
}
while ((cur = ucl_iterate_object(obj, &it, true))) {
key = ucl_object_key(cur);
if (key == NULL)
continue;
if (strcasecmp(key, "url") == 0) {
if (cur->type != UCL_STRING) {
pkg_emit_error("Expecting a string for the "
"'%s' key of the '%s' repo",
key, rname);
return;
}
url = ucl_object_tostring(cur);
} else if (strcasecmp(key, "pubkey") == 0) {
if (cur->type != UCL_STRING) {
pkg_emit_error("Expecting a string for the "
"'%s' key of the '%s' repo",
key, rname);
return;
}
pubkey = ucl_object_tostring(cur);
} else if (strcasecmp(key, "mirror_type") == 0) {
if (cur->type != UCL_STRING) {
pkg_emit_error("Expecting a string for the "
"'%s' key of the '%s' repo",
key, rname);
return;
}
mirror_type = ucl_object_tostring(cur);
} else if (strcasecmp(key, "signature_type") == 0) {
if (cur->type != UCL_STRING) {
pkg_emit_error("Expecting a string for the "
"'%s' key of the '%s' repo",
key, rname);
return;
}
signature_type = ucl_object_tostring(cur);
} else if (strcasecmp(key, "fingerprints") == 0) {
if (cur->type != UCL_STRING) {
pkg_emit_error("Expecting a string for the "
"'%s' key of the '%s' repo",
key, rname);
return;
}
fingerprints = ucl_object_tostring(cur);
} else if (strcasecmp(key, "type") == 0) {
if (cur->type != UCL_STRING) {
pkg_emit_error("Expecting a string for the "
"'%s' key of the '%s' repo",
key, rname);
return;
}
type = ucl_object_tostring(cur);
} else if (strcasecmp(key, "ip_version") == 0) {
if (cur->type != UCL_INT) {
pkg_emit_error("Expecting a integer for the "
"'%s' key of the '%s' repo",
key, rname);
return;
}
use_ipvx = ucl_object_toint(cur);
if (use_ipvx != 4 && use_ipvx != 6)
use_ipvx = 0;
}
}
if (r == NULL && url == NULL) {
pkg_debug(1, "No repo and no url for %s", rname);
return;
}
if (r == NULL)
r = pkg_repo_new(rname, url, type);
else
pkg_repo_overwrite(r, rname, url, type);
if (signature_type != NULL) {
if (strcasecmp(signature_type, "pubkey") == 0)
r->signature_type = SIG_PUBKEY;
else if (strcasecmp(signature_type, "fingerprints") == 0)
r->signature_type = SIG_FINGERPRINT;
else
r->signature_type = SIG_NONE;
}
if (fingerprints != NULL) {
free(r->fingerprints);
r->fingerprints = strdup(fingerprints);
}
if (pubkey != NULL) {
free(r->pubkey);
r->pubkey = strdup(pubkey);
}
r->enable = enable;
if (mirror_type != NULL) {
if (strcasecmp(mirror_type, "srv") == 0)
r->mirror_type = SRV;
else if (strcasecmp(mirror_type, "http") == 0)
r->mirror_type = HTTP;
else
r->mirror_type = NOMIRROR;
}
if ((flags & PKG_INIT_FLAG_USE_IPV4) == PKG_INIT_FLAG_USE_IPV4)
use_ipvx = 4;
else if ((flags & PKG_INIT_FLAG_USE_IPV6) == PKG_INIT_FLAG_USE_IPV6)
use_ipvx = 6;
if (use_ipvx != 4 && use_ipvx != 6)
use_ipvx = pkg_object_int(pkg_config_get("IP_VERSION"));
if (use_ipvx == 4)
r->flags = REPO_FLAGS_USE_IPV4;
else if (use_ipvx == 6)
r->flags = REPO_FLAGS_USE_IPV6;
}
static bool
ucl_schema_validate_number (const ucl_object_t *schema,
const ucl_object_t *obj, struct ucl_schema_error *err)
{
const ucl_object_t *elt, *test;
ucl_object_iter_t iter = NULL;
bool ret = true, exclusive = false;
double constraint, val;
const double alpha = 1e-16;
while (ret && (elt = ucl_iterate_object (schema, &iter, true)) != NULL) {
if ((elt->type == UCL_FLOAT || elt->type == UCL_INT) &&
strcmp (ucl_object_key (elt), "multipleOf") == 0) {
constraint = ucl_object_todouble (elt);
if (constraint <= 0) {
ucl_schema_create_error (err, UCL_SCHEMA_INVALID_SCHEMA, elt,
"multipleOf must be greater than zero");
ret = false;
break;
}
val = ucl_object_todouble (obj);
if (fabs (remainder (val, constraint)) > alpha) {
ucl_schema_create_error (err, UCL_SCHEMA_CONSTRAINT, obj,
"number %.4f is not multiple of %.4f, remainder is %.7f",
val, constraint);
ret = false;
break;
}
}
else if ((elt->type == UCL_FLOAT || elt->type == UCL_INT) &&
strcmp (ucl_object_key (elt), "maximum") == 0) {
constraint = ucl_object_todouble (elt);
test = ucl_object_find_key (schema, "exclusiveMaximum");
if (test && test->type == UCL_BOOLEAN) {
exclusive = ucl_object_toboolean (test);
}
val = ucl_object_todouble (obj);
if (val > constraint || (exclusive && val >= constraint)) {
ucl_schema_create_error (err, UCL_SCHEMA_CONSTRAINT, obj,
"number is too big: %.3f, maximum is: %.3f",
val, constraint);
ret = false;
break;
}
}
else if ((elt->type == UCL_FLOAT || elt->type == UCL_INT) &&
strcmp (ucl_object_key (elt), "minimum") == 0) {
constraint = ucl_object_todouble (elt);
test = ucl_object_find_key (schema, "exclusiveMinimum");
if (test && test->type == UCL_BOOLEAN) {
exclusive = ucl_object_toboolean (test);
}
val = ucl_object_todouble (obj);
if (val < constraint || (exclusive && val <= constraint)) {
ucl_schema_create_error (err, UCL_SCHEMA_CONSTRAINT, obj,
"number is too small: %.3f, minimum is: %.3f",
val, constraint);
ret = false;
break;
}
}
}
return ret;
}
static gint
fuzzy_parse_rule (struct rspamd_config *cfg, const ucl_object_t *obj, gint cb_id)
{
const ucl_object_t *value, *cur;
struct fuzzy_rule *rule;
ucl_object_iter_t it = NULL;
const char *k = NULL;
if (obj->type != UCL_OBJECT) {
msg_err_config ("invalid rule definition");
return -1;
}
rule = fuzzy_rule_new (fuzzy_module_ctx->default_symbol,
fuzzy_module_ctx->fuzzy_pool);
if ((value = ucl_object_find_key (obj, "mime_types")) != NULL) {
it = NULL;
while ((cur = ucl_iterate_object (value, &it, value->type == UCL_ARRAY))
!= NULL) {
rule->mime_types = g_list_concat (rule->mime_types,
parse_mime_types (ucl_obj_tostring (cur)));
}
}
if (rule->mime_types != NULL) {
rspamd_mempool_add_destructor (fuzzy_module_ctx->fuzzy_pool,
(rspamd_mempool_destruct_t)g_list_free, rule->mime_types);
}
if ((value = ucl_object_find_key (obj, "max_score")) != NULL) {
rule->max_score = ucl_obj_todouble (value);
}
if ((value = ucl_object_find_key (obj, "symbol")) != NULL) {
rule->symbol = ucl_obj_tostring (value);
}
if ((value = ucl_object_find_key (obj, "read_only")) != NULL) {
rule->read_only = ucl_obj_toboolean (value);
}
if ((value = ucl_object_find_key (obj, "skip_unknown")) != NULL) {
rule->skip_unknown = ucl_obj_toboolean (value);
}
if ((value = ucl_object_find_key (obj, "servers")) != NULL) {
rule->servers = rspamd_upstreams_create ();
rspamd_mempool_add_destructor (fuzzy_module_ctx->fuzzy_pool,
(rspamd_mempool_destruct_t)rspamd_upstreams_destroy,
rule->servers);
rspamd_upstreams_from_ucl (rule->servers, value, DEFAULT_PORT, NULL);
}
if ((value = ucl_object_find_key (obj, "fuzzy_map")) != NULL) {
it = NULL;
while ((cur = ucl_iterate_object (value, &it, true)) != NULL) {
parse_flags (rule, cfg, cur, cb_id);
}
}
if ((value = ucl_object_find_key (obj, "encryption_key")) != NULL) {
/* Create key from user's input */
k = ucl_object_tostring (value);
if (k == NULL || (rule->peer_key =
rspamd_http_connection_make_peer_key (k)) == NULL) {
msg_err_config ("bad encryption key value: %s",
k);
return -1;
}
rule->local_key = rspamd_http_connection_gen_key ();
}
if ((value = ucl_object_find_key (obj, "fuzzy_key")) != NULL) {
/* Create key from user's input */
k = ucl_object_tostring (value);
}
/* Setup keys */
if (k == NULL) {
/* Use some default key for all ops */
k = "rspamd";
}
rule->hash_key = g_string_sized_new (BLAKE2B_KEYBYTES);
blake2 (rule->hash_key->str, k, NULL, BLAKE2B_KEYBYTES, strlen (k), 0);
rule->hash_key->len = BLAKE2B_KEYBYTES;
if ((value = ucl_object_find_key (obj, "fuzzy_shingles_key")) != NULL) {
k = ucl_object_tostring (value);
}
if (k == NULL) {
k = "rspamd";
}
rule->shingles_key = g_string_sized_new (16);
blake2 (rule->shingles_key->str, k, NULL, 16, strlen (k), 0);
rule->shingles_key->len = 16;
if (rspamd_upstreams_count (rule->servers) == 0) {
msg_err_config ("no servers defined for fuzzy rule with symbol: %s",
rule->symbol);
return -1;
}
else {
fuzzy_module_ctx->fuzzy_rules = g_list_prepend (
fuzzy_module_ctx->fuzzy_rules,
rule);
if (rule->symbol != fuzzy_module_ctx->default_symbol) {
rspamd_symbols_cache_add_symbol (cfg->cache, rule->symbol,
0,
NULL, NULL,
SYMBOL_TYPE_VIRTUAL|SYMBOL_TYPE_FINE,
cb_id);
}
}
rspamd_mempool_add_destructor (fuzzy_module_ctx->fuzzy_pool, fuzzy_free_rule,
rule);
return 0;
}
static gboolean
rspamd_symbols_cache_load_items (struct symbols_cache *cache, const gchar *name)
{
struct rspamd_symbols_cache_header *hdr;
struct stat st;
struct ucl_parser *parser;
ucl_object_t *top;
const ucl_object_t *cur, *elt;
ucl_object_iter_t it;
struct cache_item *item, *parent;
const guchar *p;
gint fd;
gpointer map;
double w;
fd = open (name, O_RDONLY);
if (fd == -1) {
msg_info ("cannot open file %s, error %d, %s", name,
errno, strerror (errno));
return FALSE;
}
if (fstat (fd, &st) == -1) {
close (fd);
msg_info ("cannot stat file %s, error %d, %s", name,
errno, strerror (errno));
return FALSE;
}
if (st.st_size < (gint)sizeof (*hdr)) {
close (fd);
errno = EINVAL;
msg_info ("cannot use file %s, error %d, %s", name,
errno, strerror (errno));
return FALSE;
}
map = mmap (NULL, st.st_size, PROT_READ, MAP_SHARED, fd, 0);
if (map == MAP_FAILED) {
close (fd);
msg_info ("cannot mmap file %s, error %d, %s", name,
errno, strerror (errno));
return FALSE;
}
close (fd);
hdr = map;
if (memcmp (hdr->magic, rspamd_symbols_cache_magic,
sizeof (rspamd_symbols_cache_magic)) != 0) {
msg_info ("cannot use file %s, bad magic", name);
munmap (map, st.st_size);
return FALSE;
}
parser = ucl_parser_new (0);
p = (const guchar *)(hdr + 1);
if (!ucl_parser_add_chunk (parser, p, st.st_size - sizeof (*hdr))) {
msg_info ("cannot use file %s, cannot parse: %s", name,
ucl_parser_get_error (parser));
munmap (map, st.st_size);
ucl_parser_free (parser);
return FALSE;
}
top = ucl_parser_get_object (parser);
munmap (map, st.st_size);
ucl_parser_free (parser);
if (top == NULL || ucl_object_type (top) != UCL_OBJECT) {
msg_info ("cannot use file %s, bad object", name);
ucl_object_unref (top);
return FALSE;
}
it = ucl_object_iterate_new (top);
while ((cur = ucl_object_iterate_safe (it, true))) {
item = g_hash_table_lookup (cache->items_by_symbol, ucl_object_key (cur));
if (item) {
/* Copy saved info */
elt = ucl_object_find_key (cur, "weight");
if (elt) {
w = ucl_object_todouble (elt);
if (w != 0) {
item->weight = w;
}
}
elt = ucl_object_find_key (cur, "time");
if (elt) {
item->avg_time = ucl_object_todouble (elt);
}
elt = ucl_object_find_key (cur, "count");
if (elt) {
item->avg_counter = ucl_object_toint (elt);
}
elt = ucl_object_find_key (cur, "frequency");
if (elt) {
item->frequency = ucl_object_toint (elt);
}
if (item->type == SYMBOL_TYPE_VIRTUAL && item->parent != -1) {
g_assert (item->parent < (gint)cache->items_by_id->len);
parent = g_ptr_array_index (cache->items_by_id, item->parent);
if (parent->weight < item->weight) {
parent->weight = item->weight;
}
/*
* We maintain avg_time for virtual symbols equal to the
* parent item avg_time
*/
parent->avg_time = item->avg_time;
parent->avg_counter = item->avg_counter;
}
if (fabs (item->weight) > cache->max_weight) {
cache->max_weight = fabs (item->weight);
}
cache->total_freq += item->frequency;
}
}
ucl_object_iterate_free (it);
ucl_object_unref (top);
return TRUE;
}
static void
insert_metric_result (struct rspamd_task *task,
struct metric *metric,
const gchar *symbol,
double flag,
GList * opts,
gboolean single)
{
struct metric_result *metric_res;
struct symbol *s;
gdouble w, *gr_score = NULL;
struct rspamd_symbol_def *sdef;
struct rspamd_symbols_group *gr = NULL;
const ucl_object_t *mobj, *sobj;
metric_res = rspamd_create_metric_result (task, metric->name);
sdef = g_hash_table_lookup (metric->symbols, symbol);
if (sdef == NULL) {
w = 0.0;
}
else {
w = (*sdef->weight_ptr) * flag;
gr = sdef->gr;
if (gr != NULL) {
gr_score = g_hash_table_lookup (metric_res->sym_groups, gr);
if (gr_score == NULL) {
gr_score = rspamd_mempool_alloc (task->task_pool, sizeof (gdouble));
*gr_score = 0;
g_hash_table_insert (metric_res->sym_groups, gr, gr_score);
}
}
}
if (task->settings) {
mobj = ucl_object_find_key (task->settings, metric->name);
if (mobj) {
gdouble corr;
sobj = ucl_object_find_key (mobj, symbol);
if (sobj != NULL && ucl_object_todouble_safe (sobj, &corr)) {
msg_debug ("settings: changed weight of symbol %s from %.2f to %.2f",
symbol, w, corr);
w = corr * flag;
}
}
}
/* XXX: does not take grow factor into account */
if (gr != NULL && gr_score != NULL && gr->max_score > 0.0) {
if (*gr_score >= gr->max_score) {
msg_info ("maximum group score %.2f for group %s has been reached,"
" ignoring symbol %s with weight %.2f", gr->max_score,
gr->name, symbol, w);
return;
}
else if (*gr_score + w > gr->max_score) {
w = gr->max_score - *gr_score;
}
*gr_score += w;
}
/* Add metric score */
if ((s = g_hash_table_lookup (metric_res->symbols, symbol)) != NULL) {
if (sdef && sdef->one_shot) {
/*
* For one shot symbols we do not need to add them again, so
* we just force single behaviour here
*/
single = TRUE;
}
if (s->options && opts && opts != s->options) {
/* Append new options */
s->options = g_list_concat (s->options, g_list_copy (opts));
/*
* Note that there is no need to add new destructor of GList as elements of appended
* GList are used directly, so just free initial GList
*/
}
else if (opts) {
s->options = g_list_copy (opts);
rspamd_mempool_add_destructor (task->task_pool,
(rspamd_mempool_destruct_t) g_list_free, s->options);
}
if (!single) {
/* Handle grow factor */
if (metric_res->grow_factor && w > 0) {
w *= metric_res->grow_factor;
metric_res->grow_factor *= metric->grow_factor;
}
s->score += w;
metric_res->score += w;
}
else {
if (fabs (s->score) < fabs (w)) {
/* Replace less weight with a bigger one */
metric_res->score = metric_res->score - s->score + w;
s->score = w;
}
}
}
else {
s = rspamd_mempool_alloc (task->task_pool, sizeof (struct symbol));
/* Handle grow factor */
if (metric_res->grow_factor && w > 0) {
w *= metric_res->grow_factor;
metric_res->grow_factor *= metric->grow_factor;
}
else if (w > 0) {
metric_res->grow_factor = metric->grow_factor;
}
s->score = w;
s->name = symbol;
s->def = sdef;
metric_res->score += w;
if (opts) {
s->options = g_list_copy (opts);
rspamd_mempool_add_destructor (task->task_pool,
(rspamd_mempool_destruct_t) g_list_free, s->options);
}
else {
s->options = NULL;
}
g_hash_table_insert (metric_res->symbols, (gpointer) symbol, s);
}
debug_task ("symbol %s, score %.2f, metric %s, factor: %f",
symbol,
s->score,
metric->name,
w);
}
static struct rspamd_osb_tokenizer_config *
rspamd_tokenizer_osb_config_from_ucl (rspamd_mempool_t * pool,
const ucl_object_t *obj)
{
const ucl_object_t *elt;
struct rspamd_osb_tokenizer_config *cf, *def;
guchar *key = NULL;
gsize keylen;
if (pool != NULL) {
cf = rspamd_mempool_alloc (pool, sizeof (*cf));
}
else {
cf = g_slice_alloc (sizeof (*cf));
}
/* Use default config */
def = rspamd_tokenizer_osb_default_config ();
memcpy (cf, def, sizeof (*cf));
elt = ucl_object_find_key (obj, "hash");
if (elt != NULL && ucl_object_type (elt) == UCL_STRING) {
if (g_ascii_strncasecmp (ucl_object_tostring (elt), "xxh", 3)
== 0) {
cf->ht = RSPAMD_OSB_HASH_XXHASH;
elt = ucl_object_find_key (obj, "seed");
if (elt != NULL && ucl_object_type (elt) == UCL_INT) {
cf->seed = ucl_object_toint (elt);
}
}
else if (g_ascii_strncasecmp (ucl_object_tostring (elt), "sip", 3)
== 0) {
cf->ht = RSPAMD_OSB_HASH_SIPHASH;
elt = ucl_object_find_key (obj, "key");
if (elt != NULL && ucl_object_type (elt) == UCL_STRING) {
key = rspamd_decode_base32 (ucl_object_tostring (elt),
0, &keylen);
if (keylen < sizeof (rspamd_sipkey_t)) {
msg_warn ("siphash key is too short: %s", keylen);
g_free (key);
}
else {
memcpy (cf->sk, key, sizeof (cf->sk));
g_free (key);
}
}
else {
msg_warn_pool ("siphash cannot be used without key");
}
}
}
else {
elt = ucl_object_find_key (obj, "compat");
if (elt != NULL && ucl_object_toboolean (elt)) {
cf->ht = RSPAMD_OSB_HASH_COMPAT;
}
}
elt = ucl_object_find_key (obj, "window");
if (elt != NULL && ucl_object_type (elt) == UCL_INT) {
cf->window_size = ucl_object_toint (elt);
if (cf->window_size > DEFAULT_FEATURE_WINDOW_SIZE * 4) {
msg_err_pool ("too large window size: %d", cf->window_size);
cf->window_size = DEFAULT_FEATURE_WINDOW_SIZE;
}
}
return cf;
}