static PyObject *
_internal_load_ucl (char *uclstr)
{
PyObject *ret;
struct ucl_parser *parser = ucl_parser_new (UCL_PARSER_NO_TIME);
bool r = ucl_parser_add_string(parser, uclstr, 0);
if (r) {
if (ucl_parser_get_error (parser)) {
PyErr_SetString(PyExc_ValueError, ucl_parser_get_error(parser));
ucl_parser_free(parser);
ret = NULL;
goto return_with_parser;
} else {
ucl_object_t *uclobj = ucl_parser_get_object(parser);
ret = _iterate_valid_ucl(uclobj);
ucl_object_unref(uclobj);
goto return_with_parser;
}
}
else {
PyErr_SetString(PyExc_ValueError, ucl_parser_get_error (parser));
ret = NULL;
goto return_with_parser;
}
return_with_parser:
ucl_parser_free(parser);
return ret;
}
static int
read_conf_file(const char *confpath, pkg_conf_file_t conftype)
{
struct ucl_parser *p;
ucl_object_t *obj = NULL;
p = ucl_parser_new(0);
if (!ucl_parser_add_file(p, confpath)) {
if (errno != ENOENT)
errx(EXIT_FAILURE, "Unable to parse configuration "
"file %s: %s", confpath, ucl_parser_get_error(p));
ucl_parser_free(p);
/* no configuration present */
return (1);
}
obj = ucl_parser_get_object(p);
if (obj->type != UCL_OBJECT)
warnx("Invalid configuration format, ignoring the "
"configuration file %s", confpath);
else {
if (conftype == CONFFILE_PKG)
config_parse(obj, conftype);
else if (conftype == CONFFILE_REPO)
parse_repo_file(obj);
}
ucl_object_unref(obj);
ucl_parser_free(p);
return (0);
}
static gint
rspamd_control_finish_handler (struct rspamd_http_connection *conn,
struct rspamd_http_message *msg)
{
struct ucl_parser *parser;
ucl_object_t *obj;
rspamd_fstring_t *out;
const gchar *body;
gsize body_len;
struct rspamadm_control_cbdata *cbdata = conn->ud;
body = rspamd_http_message_get_body (msg, &body_len);
parser = ucl_parser_new (0);
if (!body || !ucl_parser_add_chunk (parser, body, body_len)) {
rspamd_fprintf (stderr, "cannot parse server's reply: %s\n",
ucl_parser_get_error (parser));
ucl_parser_free (parser);
}
else {
obj = ucl_parser_get_object (parser);
out = rspamd_fstring_new ();
if (json) {
rspamd_ucl_emit_fstring (obj, UCL_EMIT_JSON, &out);
}
else if (compact) {
rspamd_ucl_emit_fstring (obj, UCL_EMIT_JSON_COMPACT, &out);
}
else {
if (strcmp (cbdata->path, "/fuzzystat") == 0) {
rspamadm_execute_lua_ucl_subr (cbdata->L,
cbdata->argc,
cbdata->argv,
obj,
rspamadm_script_fuzzy_stat);
rspamd_fstring_free (out);
ucl_object_unref (obj);
ucl_parser_free (parser);
return 0;
}
else {
rspamd_ucl_emit_fstring (obj, UCL_EMIT_CONFIG, &out);
}
}
rspamd_fprintf (stdout, "%V", out);
rspamd_fstring_free (out);
ucl_object_unref (obj);
ucl_parser_free (parser);
}
return 0;
}
static void
rspamd_ucl_fin_cb (rspamd_mempool_t * pool, struct map_cb_data *data)
{
struct rspamd_ucl_map_cbdata *cbdata = data->cur_data, *prev =
data->prev_data;
ucl_object_t *obj;
struct ucl_parser *parser;
guint32 checksum;
ucl_object_iter_t it = NULL;
const ucl_object_t *cur;
struct rspamd_config *cfg = data->map->cfg;
if (prev != NULL) {
if (prev->buf != NULL) {
g_string_free (prev->buf, TRUE);
}
g_free (prev);
}
if (cbdata == NULL) {
msg_err_config ("map fin error: new data is NULL");
return;
}
checksum = XXH64 (cbdata->buf->str, cbdata->buf->len, 0);
if (data->map->checksum != checksum) {
/* New data available */
parser = ucl_parser_new (0);
if (!ucl_parser_add_chunk (parser, cbdata->buf->str,
cbdata->buf->len)) {
msg_err_config ("cannot parse map %s: %s",
data->map->uri,
ucl_parser_get_error (parser));
ucl_parser_free (parser);
}
else {
obj = ucl_parser_get_object (parser);
ucl_parser_free (parser);
it = NULL;
while ((cur = ucl_iterate_object (obj, &it, true))) {
ucl_object_replace_key (cbdata->cfg->rcl_obj, (ucl_object_t *)cur,
cur->key, cur->keylen, false);
}
ucl_object_unref (obj);
data->map->checksum = checksum;
}
}
else {
msg_info_config ("do not reload map %s, checksum is the same: %d",
data->map->uri,
checksum);
}
}
static gint
rspamd_client_finish_handler (struct rspamd_http_connection *conn,
struct rspamd_http_message *msg)
{
struct rspamd_client_request *req =
(struct rspamd_client_request *)conn->ud;
struct rspamd_client_connection *c;
struct ucl_parser *parser;
GError *err;
c = req->conn;
if (!c->req_sent) {
c->req_sent = TRUE;
rspamd_http_connection_reset (c->http_conn);
rspamd_http_connection_read_message (c->http_conn,
c->req,
c->fd,
&c->timeout,
c->ev_base);
return 0;
}
else {
if (rspamd_http_message_get_body (msg, NULL) == NULL || msg->code != 200) {
err = g_error_new (RCLIENT_ERROR, msg->code, "HTTP error: %d, %.*s",
msg->code,
(gint)msg->status->len, msg->status->str);
req->cb (c, msg, c->server_name->str, NULL, req->input, req->ud, err);
g_error_free (err);
return 0;
}
parser = ucl_parser_new (0);
if (!ucl_parser_add_chunk (parser, msg->body_buf.begin, msg->body_buf.len)) {
err = g_error_new (RCLIENT_ERROR, msg->code, "Cannot parse UCL: %s",
ucl_parser_get_error (parser));
ucl_parser_free (parser);
req->cb (c, msg, c->server_name->str, NULL, req->input, req->ud, err);
g_error_free (err);
return 0;
}
req->cb (c, msg, c->server_name->str, ucl_parser_get_object (
parser), req->input, req->ud, NULL);
ucl_parser_free (parser);
}
return 0;
}
void
cleanup()
{
if (parser != NULL) {
ucl_parser_free(parser);
}
if (setparser != NULL) {
ucl_parser_free(setparser);
}
if (root_obj != NULL) {
ucl_object_unref(root_obj);
}
if (set_obj != NULL) {
ucl_object_unref(set_obj);
}
}
static void
load_repo_file(const char *repofile, pkg_init_flags flags)
{
struct ucl_parser *p;
ucl_object_t *obj = NULL;
const char *myarch = NULL;
const char *myarch_legacy = NULL;
p = ucl_parser_new(0);
myarch = pkg_object_string(pkg_config_get("ABI"));
ucl_parser_register_variable (p, "ABI", myarch);
myarch_legacy = pkg_object_string(pkg_config_get("ALTABI"));
ucl_parser_register_variable (p, "ALTABI", myarch_legacy);
pkg_debug(1, "PKgConfig: loading %s", repofile);
if (!ucl_parser_add_file(p, repofile)) {
pkg_emit_error("Error parsing: %s: %s", repofile,
ucl_parser_get_error(p));
ucl_parser_free(p);
return;
}
obj = ucl_parser_get_object(p);
if (obj == NULL)
return;
if (obj->type == UCL_OBJECT)
walk_repo_obj(obj, repofile, flags);
ucl_object_unref(obj);
}
static ucl_object_t *
keyword_open_schema(void)
{
struct ucl_parser *parser;
static const char keyword_schema_str[] = ""
"{"
" type = object;"
" properties {"
" actions = { "
" type = array; "
" items = { type = string }; "
" uniqueItems: true "
" }; "
" attributes = { "
" type = object; "
" properties { "
" owner = { type = string }; "
" group = { type = string }; "
" mode = { oneOf: [ { type = integer }, { type = string } ] }; "
" }"
" }; "
" pre-install = { type = string }; "
" post-install = { type = string }; "
" pre-deinstall = { type = string }; "
" post-deinstall = { type = string }; "
" pre-upgrade = { type = string }; "
" post-upgrade = { type = string }; "
" }"
"}";
if (keyword_schema != NULL)
return (keyword_schema);
parser = ucl_parser_new(0);
if (!ucl_parser_add_chunk(parser, keyword_schema_str,
sizeof(keyword_schema_str) -1)) {
pkg_emit_error("Cannot parse schema for keywords: %s",
ucl_parser_get_error(parser));
ucl_parser_free(parser);
return (NULL);
}
keyword_schema = ucl_parser_get_object(parser);
ucl_parser_free(parser);
return (keyword_schema);
}
/***
* @function rspamd_cryptobox_keypair.load(file)
* Loads public key from UCL file
* @param {string} file filename to load
* @return {cryptobox_keypair} new keypair
*/
static gint
lua_cryptobox_keypair_load (lua_State *L)
{
struct rspamd_cryptobox_keypair *kp, **pkp;
const gchar *filename;
struct ucl_parser *parser;
ucl_object_t *obj;
filename = luaL_checkstring (L, 1);
if (filename != NULL) {
parser = ucl_parser_new (0);
if (!ucl_parser_add_file (parser, filename)) {
msg_err ("cannot open keypair from file: %s, %s",
filename,
ucl_parser_get_error (parser));
ucl_parser_free (parser);
lua_pushnil (L);
}
else {
obj = ucl_parser_get_object (parser);
kp = rspamd_keypair_from_ucl (obj);
ucl_parser_free (parser);
if (kp == NULL) {
msg_err ("cannot open keypair from file: %s",
filename);
ucl_object_unref (obj);
lua_pushnil (L);
}
else {
pkp = lua_newuserdata (L, sizeof (gpointer));
*pkp = kp;
rspamd_lua_setclass (L, "rspamd{cryptobox_keypair}", -1);
ucl_object_unref (obj);
}
}
}
else {
return luaL_error (L, "bad input arguments");
}
return 1;
}
/***
* @function rspamd_cryptobox_keypair.create(ucl_data)
* Loads public key from UCL data
* @param {string} ucl_data ucl to load
* @return {cryptobox_keypair} new keypair
*/
static gint
lua_cryptobox_keypair_create (lua_State *L)
{
struct rspamd_cryptobox_keypair *kp, **pkp;
const gchar *buf;
gsize len;
struct ucl_parser *parser;
ucl_object_t *obj;
buf = luaL_checklstring (L, 1, &len);
if (buf != NULL) {
parser = ucl_parser_new (0);
if (!ucl_parser_add_chunk (parser, buf, len)) {
msg_err ("cannot open keypair from data: %s",
ucl_parser_get_error (parser));
ucl_parser_free (parser);
lua_pushnil (L);
}
else {
obj = ucl_parser_get_object (parser);
kp = rspamd_keypair_from_ucl (obj);
ucl_parser_free (parser);
if (kp == NULL) {
msg_err ("cannot load keypair from data");
ucl_object_unref (obj);
lua_pushnil (L);
}
else {
pkp = lua_newuserdata (L, sizeof (gpointer));
*pkp = kp;
rspamd_lua_setclass (L, "rspamd{cryptobox_keypair}", -1);
ucl_object_unref (obj);
}
}
}
else {
luaL_error (L, "bad input arguments");
}
return 1;
}
static void
load_repo_file(const char *repofile)
{
struct ucl_parser *p;
ucl_object_t *obj = NULL;
bool fallback = false;
const char *myarch = NULL;
p = ucl_parser_new(0);
myarch = pkg_object_string(pkg_config_get("ABI"));
ucl_parser_register_variable (p, "ABI", myarch);
pkg_debug(1, "PKgConfig: loading %s", repofile);
if (!ucl_parser_add_file(p, repofile)) {
pkg_emit_error("Error parsing: %s: %s", repofile,
ucl_parser_get_error(p));
if (errno == ENOENT) {
ucl_parser_free(p);
return;
}
fallback = true;
}
if (fallback) {
obj = yaml_to_ucl(repofile, NULL, 0);
if (obj == NULL)
return;
}
if (fallback) {
pkg_emit_error("%s file is using a deprecated format. "
"Please replace it with the following:\n"
"====== BEGIN %s ======\n"
"%s"
"\n====== END %s ======\n",
repofile, repofile,
ucl_object_emit(obj, UCL_EMIT_YAML),
repofile);
}
obj = ucl_parser_get_object(p);
if (obj->type == UCL_OBJECT)
walk_repo_obj(obj, repofile);
ucl_object_free(obj);
}
int
main(int argc, char **argv, char **envp)
{
char const *error;
ucl_object_t *obj;
ucl_object_t const *kobj;
struct ucl_parser *parser;
int status = 0;
parseopt(argc, argv);
parser = ucl_parser_new(UCL_PARSER_NO_IMPLICIT_ARRAYS | UCL_PARSER_ZEROCOPY);
if (parser == NULL) {
err(EX_OSERR, "cannot create parser");
}
addvars(parser, envp);
parse_file(parser);
if ((error = ucl_parser_get_error(parser))) {
errx(EX_DATAERR, "cannot load %s: %s", file, error);
}
obj = ucl_parser_get_object(parser);
if (key != NULL && strcmp(key, ".") != 0)
kobj = ucl_object_lookup_path(obj, key);
else
kobj = obj;
if (env) {
int nlen = 0;
char *s;
if (name != NULL)
nlen = strlcpy(envkey, name, MAX_KEY_LEN);
else if ((s = strrchr(key, '.')) != NULL)
nlen = strlcpy(envkey, s + 1, MAX_KEY_LEN);
if (kobj != NULL)
expand(nlen, kobj);
status = run(argv + cmdarg);
} else if (kobj != NULL) {
print(kobj);
}
ucl_object_unref(obj);
ucl_parser_free(parser);
exit(status);
}
static gint
rspamd_client_finish_handler (struct rspamd_http_connection *conn,
struct rspamd_http_message *msg)
{
struct rspamd_client_request *req =
(struct rspamd_client_request *)conn->ud;
struct rspamd_client_connection *c;
struct ucl_parser *parser;
GError *err;
const rspamd_ftok_t *tok;
c = req->conn;
if (!c->req_sent) {
c->req_sent = TRUE;
rspamd_http_connection_reset (c->http_conn);
rspamd_http_connection_read_message (c->http_conn,
c->req,
c->fd,
&c->timeout,
c->ev_base);
return 0;
}
else {
if (rspamd_http_message_get_body (msg, NULL) == NULL || msg->code != 200) {
err = g_error_new (RCLIENT_ERROR, msg->code, "HTTP error: %d, %.*s",
msg->code,
(gint)msg->status->len, msg->status->str);
req->cb (c, msg, c->server_name->str, NULL, req->input, req->ud, err);
g_error_free (err);
return 0;
}
tok = rspamd_http_message_find_header (msg, "compression");
if (tok) {
/* Need to uncompress */
rspamd_ftok_t t;
t.begin = "zstd";
t.len = 4;
if (rspamd_ftok_casecmp (tok, &t) == 0) {
ZSTD_DStream *zstream;
ZSTD_inBuffer zin;
ZSTD_outBuffer zout;
guchar *out;
gsize outlen, r;
zstream = ZSTD_createDStream ();
ZSTD_initDStream (zstream);
zin.pos = 0;
zin.src = msg->body_buf.begin;
zin.size = msg->body_buf.len;
if ((outlen = ZSTD_getDecompressedSize (zin.src, zin.size)) == 0) {
outlen = ZSTD_DStreamOutSize ();
}
out = g_malloc (outlen);
zout.dst = out;
zout.pos = 0;
zout.size = outlen;
while (zin.pos < zin.size) {
r = ZSTD_decompressStream (zstream, &zout, &zin);
if (ZSTD_isError (r)) {
err = g_error_new (RCLIENT_ERROR, 500,
"Decompression error: %s",
ZSTD_getErrorName (r));
req->cb (c, msg, c->server_name->str, NULL,
req->input, req->ud, err);
g_error_free (err);
ZSTD_freeDStream (zstream);
g_free (out);
return 0;
}
if (zout.pos == zout.size) {
/* We need to extend output buffer */
zout.size = zout.size * 1.5 + 1.0;
zout.dst = g_realloc (zout.dst, zout.size);
}
}
ZSTD_freeDStream (zstream);
parser = ucl_parser_new (0);
if (!ucl_parser_add_chunk (parser, zout.dst, zout.pos)) {
err = g_error_new (RCLIENT_ERROR, msg->code, "Cannot parse UCL: %s",
ucl_parser_get_error (parser));
ucl_parser_free (parser);
req->cb (c, msg, c->server_name->str, NULL, req->input, req->ud, err);
g_error_free (err);
g_free (zout.dst);
return 0;
}
g_free (zout.dst);
}
else {
err = g_error_new (RCLIENT_ERROR, 500,
"Invalid compression method");
req->cb (c, msg, c->server_name->str, NULL,
req->input, req->ud, err);
g_error_free (err);
return 0;
}
}
else {
parser = ucl_parser_new (0);
if (!ucl_parser_add_chunk (parser, msg->body_buf.begin, msg->body_buf.len)) {
err = g_error_new (RCLIENT_ERROR, msg->code, "Cannot parse UCL: %s",
ucl_parser_get_error (parser));
ucl_parser_free (parser);
req->cb (c, msg, c->server_name->str, NULL, req->input, req->ud, err);
g_error_free (err);
return 0;
}
}
req->cb (c, msg, c->server_name->str, ucl_parser_get_object (
parser), req->input, req->ud, NULL);
ucl_parser_free (parser);
}
return 0;
}
int
pkg_ini(const char *path, const char *reposdir, pkg_init_flags flags)
{
struct ucl_parser *p = NULL;
size_t i;
const char *val = NULL;
const char *buf, *walk, *value, *key, *k;
const char *evkey = NULL;
const char *nsname = NULL;
const char *evpipe = NULL;
const ucl_object_t *cur, *object;
ucl_object_t *obj = NULL, *o, *ncfg;
ucl_object_iter_t it = NULL;
struct sbuf *ukey = NULL;
bool fatal_errors = false;
k = NULL;
o = NULL;
pkg_get_myarch(myabi, BUFSIZ);
pkg_get_myarch_legacy(myabi_legacy, BUFSIZ);
if (parsed != false) {
pkg_emit_error("pkg_init() must only be called once");
return (EPKG_FATAL);
}
if (((flags & PKG_INIT_FLAG_USE_IPV4) == PKG_INIT_FLAG_USE_IPV4) &&
((flags & PKG_INIT_FLAG_USE_IPV6) == PKG_INIT_FLAG_USE_IPV6)) {
pkg_emit_error("Invalid flags for pkg_init()");
return (EPKG_FATAL);
}
config = ucl_object_typed_new(UCL_OBJECT);
for (i = 0; i < c_size; i++) {
switch (c[i].type) {
case PKG_STRING:
obj = ucl_object_fromstring_common(
c[i].def != NULL ? c[i].def : "", 0, UCL_STRING_TRIM);
ucl_object_insert_key(config, obj,
c[i].key, strlen(c[i].key), false);
break;
case PKG_INT:
ucl_object_insert_key(config,
ucl_object_fromstring_common(c[i].def, 0, UCL_STRING_PARSE_INT),
c[i].key, strlen(c[i].key), false);
break;
case PKG_BOOL:
ucl_object_insert_key(config,
ucl_object_fromstring_common(c[i].def, 0, UCL_STRING_PARSE_BOOLEAN),
c[i].key, strlen(c[i].key), false);
break;
case PKG_OBJECT:
obj = ucl_object_typed_new(UCL_OBJECT);
if (c[i].def != NULL) {
walk = buf = c[i].def;
while ((buf = strchr(buf, ',')) != NULL) {
key = walk;
value = walk;
while (*value != ',') {
if (*value == '=')
break;
value++;
}
ucl_object_insert_key(obj,
ucl_object_fromstring_common(value + 1, buf - value - 1, UCL_STRING_TRIM),
key, value - key, false);
buf++;
walk = buf;
}
key = walk;
value = walk;
while (*value != ',') {
if (*value == '=')
break;
value++;
}
if (o == NULL)
o = ucl_object_typed_new(UCL_OBJECT);
ucl_object_insert_key(o,
ucl_object_fromstring_common(value + 1, strlen(value + 1), UCL_STRING_TRIM),
key, value - key, false);
}
ucl_object_insert_key(config, obj,
c[i].key, strlen(c[i].key), false);
break;
case PKG_ARRAY:
obj = ucl_object_typed_new(UCL_ARRAY);
if (c[i].def != NULL) {
walk = buf = c[i].def;
while ((buf = strchr(buf, ',')) != NULL) {
ucl_array_append(obj,
ucl_object_fromstring_common(walk, buf - walk, UCL_STRING_TRIM));
buf++;
walk = buf;
}
ucl_array_append(obj,
ucl_object_fromstring_common(walk, strlen(walk), UCL_STRING_TRIM));
}
ucl_object_insert_key(config, obj,
c[i].key, strlen(c[i].key), false);
break;
}
}
if (path == NULL)
path = PREFIX"/etc/pkg.conf";
p = ucl_parser_new(0);
errno = 0;
obj = NULL;
if (!ucl_parser_add_file(p, path)) {
if (errno != ENOENT)
pkg_emit_error("Invalid configuration file: %s", ucl_parser_get_error(p));
} else {
obj = ucl_parser_get_object(p);
}
ncfg = NULL;
while (obj != NULL && (cur = ucl_iterate_object(obj, &it, true))) {
sbuf_init(&ukey);
key = ucl_object_key(cur);
for (i = 0; key[i] != '\0'; i++)
sbuf_putc(ukey, toupper(key[i]));
sbuf_finish(ukey);
object = ucl_object_find_keyl(config, sbuf_data(ukey), sbuf_len(ukey));
if (strncasecmp(sbuf_data(ukey), "PACKAGESITE", sbuf_len(ukey))
== 0 || strncasecmp(sbuf_data(ukey), "PUBKEY",
sbuf_len(ukey)) == 0 || strncasecmp(sbuf_data(ukey),
"MIRROR_TYPE", sbuf_len(ukey)) == 0) {
pkg_emit_error("%s in pkg.conf is no longer "
"supported. Convert to the new repository style."
" See pkg.conf(5)", sbuf_data(ukey));
fatal_errors = true;
continue;
}
/* ignore unknown keys */
if (object == NULL)
continue;
if (object->type != cur->type) {
pkg_emit_error("Malformed key %s, ignoring", key);
continue;
}
if (ncfg == NULL)
ncfg = ucl_object_typed_new(UCL_OBJECT);
ucl_object_insert_key(ncfg, ucl_object_copy(cur), sbuf_data(ukey), sbuf_len(ukey), true);
}
if (fatal_errors) {
ucl_object_unref(ncfg);
ucl_parser_free(p);
return (EPKG_FATAL);
}
if (ncfg != NULL) {
it = NULL;
while (( cur = ucl_iterate_object(ncfg, &it, true))) {
key = ucl_object_key(cur);
ucl_object_replace_key(config, ucl_object_ref(cur), key, strlen(key), true);
}
ucl_object_unref(ncfg);
}
ncfg = NULL;
it = NULL;
while ((cur = ucl_iterate_object(config, &it, true))) {
o = NULL;
key = ucl_object_key(cur);
val = getenv(key);
if (val == NULL)
continue;
switch (cur->type) {
case UCL_STRING:
o = ucl_object_fromstring_common(val, 0, UCL_STRING_TRIM);
break;
case UCL_INT:
o = ucl_object_fromstring_common(val, 0, UCL_STRING_PARSE_INT);
if (o->type != UCL_INT) {
pkg_emit_error("Invalid type for environment "
"variable %s, got %s, while expecting an integer",
key, val);
ucl_object_unref(o);
continue;
}
break;
case UCL_BOOLEAN:
o = ucl_object_fromstring_common(val, 0, UCL_STRING_PARSE_BOOLEAN);
if (o->type != UCL_BOOLEAN) {
pkg_emit_error("Invalid type for environment "
"variable %s, got %s, while expecting a boolean",
key, val);
ucl_object_unref(o);
continue;
}
break;
case UCL_OBJECT:
o = ucl_object_typed_new(UCL_OBJECT);
walk = buf = val;
while ((buf = strchr(buf, ',')) != NULL) {
k = walk;
value = walk;
while (*value != ',') {
if (*value == '=')
break;
value++;
}
ucl_object_insert_key(o,
ucl_object_fromstring_common(value + 1, buf - value - 1, UCL_STRING_TRIM),
k, value - k, false);
buf++;
walk = buf;
}
key = walk;
value = walk;
while (*value != '\0') {
if (*value == '=')
break;
value++;
}
ucl_object_insert_key(o,
ucl_object_fromstring_common(value + 1, strlen(value + 1), UCL_STRING_TRIM),
k, value - k, false);
break;
case UCL_ARRAY:
o = ucl_object_typed_new(UCL_ARRAY);
walk = buf = val;
while ((buf = strchr(buf, ',')) != NULL) {
ucl_array_append(o,
ucl_object_fromstring_common(walk, buf - walk, UCL_STRING_TRIM));
buf++;
walk = buf;
}
ucl_array_append(o,
ucl_object_fromstring_common(walk, strlen(walk), UCL_STRING_TRIM));
break;
default:
/* ignore other types */
break;
}
if (o != NULL) {
if (ncfg == NULL)
ncfg = ucl_object_typed_new(UCL_OBJECT);
ucl_object_insert_key(ncfg, o, key, strlen(key), true);
}
}
if (ncfg != NULL) {
it = NULL;
while (( cur = ucl_iterate_object(ncfg, &it, true))) {
key = ucl_object_key(cur);
ucl_object_replace_key(config, ucl_object_ref(cur), key, strlen(key), true);
}
ucl_object_unref(ncfg);
}
disable_plugins_if_static();
parsed = true;
ucl_object_unref(obj);
ucl_parser_free(p);
if (strcmp(pkg_object_string(pkg_config_get("ABI")), "unknown") == 0) {
pkg_emit_error("Unable to determine ABI");
return (EPKG_FATAL);
}
pkg_debug(1, "%s", "pkg initialized");
/* Start the event pipe */
evpipe = pkg_object_string(pkg_config_get("EVENT_PIPE"));
if (evpipe != NULL)
connect_evpipe(evpipe);
debug_level = pkg_object_int(pkg_config_get("DEBUG_LEVEL"));
it = NULL;
object = ucl_object_find_key(config, "PKG_ENV");
while ((cur = ucl_iterate_object(object, &it, true))) {
evkey = ucl_object_key(cur);
pkg_debug(1, "Setting env var: %s", evkey);
if (evkey != NULL && evkey[0] != '\0')
setenv(evkey, ucl_object_tostring_forced(cur), 1);
}
/* load the repositories */
load_repositories(reposdir, flags);
setenv("HTTP_USER_AGENT", "pkg/"PKGVERSION, 1);
/* bypass resolv.conf with specified NAMESERVER if any */
nsname = pkg_object_string(pkg_config_get("NAMESERVER"));
if (nsname != NULL)
set_nameserver(ucl_object_tostring_forced(o));
return (EPKG_OK);
}
int
rtpp_cfile_process(struct rtpp_cfg_stable *csp)
{
struct ucl_parser *parser;
ucl_object_t *conf_root;
ucl_object_iter_t it_conf;
const ucl_object_t *obj_file;
const char *cf_key;
int fd, ecode;
ecode = 0;
fd = open(csp->cfile, O_RDONLY);
if (fd < 0) {
RTPP_ELOG(csp->glog, RTPP_LOG_ERR, "open failed: %s", csp->cfile);
ecode = -1;
goto e0;
}
parser = ucl_parser_new(UCL_PARSER_NO_FILEVARS);
if (parser == NULL) {
RTPP_LOG(csp->glog, RTPP_LOG_ERR, "ucl_parser_new() failed");
ecode = -1;
goto e1;
}
ucl_parser_add_fd(parser, fd);
conf_root = ucl_parser_get_object(parser);
if (conf_root == NULL) {
RTPP_LOG(csp->glog, RTPP_LOG_ERR, "ucl_parser_get_object() failed");
ecode = -1;
goto e2;
}
if (ucl_parser_get_error(parser)) {
RTPP_LOG(csp->glog, RTPP_LOG_ERR, "Parse Error occured: %s", ucl_parser_get_error(parser));
ecode = -1;
goto e3;
}
it_conf = ucl_object_iterate_new(conf_root);
if (it_conf == NULL) {
RTPP_LOG(csp->glog, RTPP_LOG_ERR, "ucl_object_iterate_new() failed");
ecode = -1;
goto e3;
}
while ((obj_file = ucl_object_iterate_safe(it_conf, true)) != NULL) {
cf_key = ucl_object_key(obj_file);
RTPP_LOG(csp->glog, RTPP_LOG_DBUG, "Entry: %s", cf_key);
if (strcasecmp(cf_key, "modules") == 0) {
if (parse_modules(csp, obj_file) < 0) {
RTPP_LOG(csp->glog, RTPP_LOG_ERR, "parse_modules() failed");
ecode = -1;
goto e4;
}
}
}
if (ucl_object_iter_chk_excpn(it_conf)) {
ecode = -1;
}
e4:
ucl_object_iterate_free(it_conf);
e3:
ucl_object_unref(conf_root);
e2:
ucl_parser_free(parser);
e1:
close(fd);
e0:
return (ecode);
}
static gboolean
rspamd_symbols_cache_load_items (struct symbols_cache *cache, const gchar *name)
{
struct rspamd_symbols_cache_header *hdr;
struct stat st;
struct ucl_parser *parser;
ucl_object_t *top;
const ucl_object_t *cur, *elt;
ucl_object_iter_t it;
struct cache_item *item, *parent;
const guchar *p;
gint fd;
gpointer map;
double w;
fd = open (name, O_RDONLY);
if (fd == -1) {
msg_info ("cannot open file %s, error %d, %s", name,
errno, strerror (errno));
return FALSE;
}
if (fstat (fd, &st) == -1) {
close (fd);
msg_info ("cannot stat file %s, error %d, %s", name,
errno, strerror (errno));
return FALSE;
}
if (st.st_size < (gint)sizeof (*hdr)) {
close (fd);
errno = EINVAL;
msg_info ("cannot use file %s, error %d, %s", name,
errno, strerror (errno));
return FALSE;
}
map = mmap (NULL, st.st_size, PROT_READ, MAP_SHARED, fd, 0);
if (map == MAP_FAILED) {
close (fd);
msg_info ("cannot mmap file %s, error %d, %s", name,
errno, strerror (errno));
return FALSE;
}
close (fd);
hdr = map;
if (memcmp (hdr->magic, rspamd_symbols_cache_magic,
sizeof (rspamd_symbols_cache_magic)) != 0) {
msg_info ("cannot use file %s, bad magic", name);
munmap (map, st.st_size);
return FALSE;
}
parser = ucl_parser_new (0);
p = (const guchar *)(hdr + 1);
if (!ucl_parser_add_chunk (parser, p, st.st_size - sizeof (*hdr))) {
msg_info ("cannot use file %s, cannot parse: %s", name,
ucl_parser_get_error (parser));
munmap (map, st.st_size);
ucl_parser_free (parser);
return FALSE;
}
top = ucl_parser_get_object (parser);
munmap (map, st.st_size);
ucl_parser_free (parser);
if (top == NULL || ucl_object_type (top) != UCL_OBJECT) {
msg_info ("cannot use file %s, bad object", name);
ucl_object_unref (top);
return FALSE;
}
it = ucl_object_iterate_new (top);
while ((cur = ucl_object_iterate_safe (it, true))) {
item = g_hash_table_lookup (cache->items_by_symbol, ucl_object_key (cur));
if (item) {
/* Copy saved info */
elt = ucl_object_find_key (cur, "weight");
if (elt) {
w = ucl_object_todouble (elt);
if (w != 0) {
item->weight = w;
}
}
elt = ucl_object_find_key (cur, "time");
if (elt) {
item->avg_time = ucl_object_todouble (elt);
}
elt = ucl_object_find_key (cur, "count");
if (elt) {
item->avg_counter = ucl_object_toint (elt);
}
elt = ucl_object_find_key (cur, "frequency");
if (elt) {
item->frequency = ucl_object_toint (elt);
}
if (item->type == SYMBOL_TYPE_VIRTUAL && item->parent != -1) {
g_assert (item->parent < (gint)cache->items_by_id->len);
parent = g_ptr_array_index (cache->items_by_id, item->parent);
if (parent->weight < item->weight) {
parent->weight = item->weight;
}
/*
* We maintain avg_time for virtual symbols equal to the
* parent item avg_time
*/
parent->avg_time = item->avg_time;
parent->avg_counter = item->avg_counter;
}
if (fabs (item->weight) > cache->max_weight) {
cache->max_weight = fabs (item->weight);
}
cache->total_freq += item->frequency;
}
}
ucl_object_iterate_free (it);
ucl_object_unref (top);
return TRUE;
}
/**
* Entry point.
*/
int main(int argc, char **argv)
{
// Parse program options
char *config_file = NULL;
bool server = false;
bool collector = false;
bool callibrator = false;
bool status_only = false;
int log_option = 0;
char c;
while ((c = getopt(argc, argv, "hc:sdflr")) != EOF) {
switch (c) {
case 'h': {
show_help(argv[0]);
return 1;
}
case 'c': config_file = strdup(optarg); break;
case 's': status_only = true; break;
case 'd': server = true; break;
case 'l': collector = true; break;
case 'r': callibrator = true; break;
case 'f': log_option |= LOG_PERROR; break;
default: {
fprintf(stderr, "ERROR: Invalid option %c!\n", c);
show_help(argv[0]);
return 1;
}
}
}
if (config_file == NULL)
config_file = strdup("/etc/koruza.cfg");
// Load the configuration file
struct ucl_parser *parser = ucl_parser_new(UCL_PARSER_KEY_LOWERCASE);
ucl_object_t *config = NULL;
ucl_object_t *obj = NULL;
int ret_value = 0;
if (!parser) {
fprintf(stderr, "ERROR: Failed to initialize configuration parser!\n");
return 2;
}
if (!ucl_parser_add_file(parser, config_file)) {
fprintf(stderr, "ERROR: Failed to parse configuration file '%s'!\n", config_file);
fprintf(stderr, "ERROR: %s\n", ucl_parser_get_error(parser));
ret_value = 2;
goto cleanup_exit;
} else {
config = ucl_parser_get_object(parser);
}
if (server) {
obj = ucl_object_find_key(config, "server");
if (!obj) {
fprintf(stderr, "ERROR: Missing server configuration!\n");
ret_value = 2;
goto cleanup_exit;
}
start_server(obj, log_option);
} else if (collector) {
start_collector(config, log_option);
} else if (callibrator) {
start_callibrator(config, log_option);
} else {
start_controller(config, status_only);
}
cleanup_exit:
// Cleanup and exit
if (config)
ucl_object_free(config);
if (parser)
ucl_parser_free(parser);
return ret_value;
}
void
json_config_fin_cb (rspamd_mempool_t * pool, struct map_cb_data *data)
{
struct config_json_buf *jb;
ucl_object_t *top;
struct ucl_parser *parser;
if (data->prev_data) {
jb = data->prev_data;
/* Clean prev data */
if (jb->buf) {
g_free (jb->buf);
}
g_free (jb);
}
/* Now parse json */
if (data->cur_data) {
jb = data->cur_data;
}
else {
msg_err ("no data read");
return;
}
if (jb->buf == NULL) {
msg_err ("no data read");
return;
}
/* NULL terminate current buf */
*jb->pos = '\0';
parser = ucl_parser_new (0);
if (!ucl_parser_add_chunk (parser, jb->buf, jb->pos - jb->buf)) {
msg_err ("cannot load json data: parse error %s",
ucl_parser_get_error (parser));
ucl_parser_free (parser);
return;
}
top = ucl_parser_get_object (parser);
ucl_parser_free (parser);
if (ucl_object_type (top) != UCL_ARRAY) {
ucl_object_unref (top);
msg_err ("loaded json is not an array");
return;
}
jb->cfg->current_dynamic_conf = NULL;
ucl_object_unref (jb->obj);
jb->obj = top;
/*
* Note about thread safety: we are updating values that are gdoubles so it is not atomic in general case
* but on the other hand all that data is used only in the main thread, so why it is *likely* safe
* to do this task in this way without explicit lock.
*/
apply_dynamic_conf (jb->obj, jb->cfg);
jb->cfg->current_dynamic_conf = jb->obj;
}
static void
rspamadm_statconvert (gint argc, gchar **argv, const struct rspamadm_command *cmd)
{
GOptionContext *context;
GError *error = NULL;
ucl_object_t *obj;
context = g_option_context_new (
"statconvert - converts statistics from sqlite3 to redis");
g_option_context_set_summary (context,
"Summary:\n Rspamd administration utility version "
RVERSION
"\n Release id: "
RID);
g_option_context_add_main_entries (context, entries, NULL);
g_option_context_set_ignore_unknown_options (context, TRUE);
if (!g_option_context_parse (context, &argc, &argv, &error)) {
rspamd_fprintf (stderr, "option parsing failed: %s\n", error->message);
g_error_free (error);
exit (1);
}
if (config_file) {
/* Load config file, assuming that it has all information required */
struct ucl_parser *parser;
parser = ucl_parser_new (0);
rspamd_ucl_add_conf_variables (parser, ucl_vars);
if (!ucl_parser_add_file (parser, config_file)) {
msg_err ("ucl parser error: %s", ucl_parser_get_error (parser));
ucl_parser_free (parser);
exit (EXIT_FAILURE);
}
obj = ucl_parser_get_object (parser);
ucl_parser_free (parser);
}
else {
/* We need to get all information from the command line */
ucl_object_t *classifier, *statfile_ham, *statfile_spam, *tmp, *redis;
/* Check arguments sanity */
if (spam_db == NULL) {
msg_err ("No spam-db specified");
exit (EXIT_FAILURE);
}
if (ham_db == NULL) {
msg_err ("No ham-db specified");
exit (EXIT_FAILURE);
}
if (redis_host == NULL) {
msg_err ("No redis-host specified");
exit (EXIT_FAILURE);
}
if (symbol_ham == NULL) {
msg_err ("No symbol-ham specified");
exit (EXIT_FAILURE);
}
if (symbol_spam == NULL) {
msg_err ("No symbol-spam specified");
exit (EXIT_FAILURE);
}
obj = ucl_object_typed_new (UCL_OBJECT);
classifier = ucl_object_typed_new (UCL_OBJECT);
ucl_object_insert_key (obj, classifier, "classifier", 0, false);
/* Now we need to create "bayes" key in it */
tmp = ucl_object_typed_new (UCL_OBJECT);
ucl_object_insert_key (classifier, tmp, "bayes", 0, false);
classifier = tmp;
ucl_object_insert_key (classifier, ucl_object_fromstring ("sqlite3"),
"backend", 0, false);
if (cache_db != NULL) {
ucl_object_t *cache;
cache = ucl_object_typed_new (UCL_OBJECT);
ucl_object_insert_key (cache, ucl_object_fromstring ("sqlite3"),
"type", 0, false);
ucl_object_insert_key (cache, ucl_object_fromstring (cache_db),
"file", 0, false);
ucl_object_insert_key (classifier, cache, "cache", 0, false);
}
statfile_ham = ucl_object_typed_new (UCL_OBJECT);
ucl_object_insert_key (statfile_ham, ucl_object_fromstring (symbol_ham),
"symbol", 0, false);
ucl_object_insert_key (statfile_ham, ucl_object_frombool (false),
"spam", 0, false);
ucl_object_insert_key (statfile_ham, ucl_object_fromstring (ham_db),
"db", 0, false);
statfile_spam = ucl_object_typed_new (UCL_OBJECT);
ucl_object_insert_key (statfile_spam, ucl_object_fromstring (symbol_spam),
"symbol", 0, false);
ucl_object_insert_key (statfile_spam, ucl_object_frombool (true),
"spam", 0, false);
ucl_object_insert_key (statfile_spam, ucl_object_fromstring (spam_db),
"db", 0, false);
DL_APPEND (statfile_ham, statfile_spam);
ucl_object_insert_key (classifier, statfile_ham,
"statfile", 0, false);
/* Deal with redis */
redis = ucl_object_typed_new (UCL_OBJECT);
ucl_object_insert_key (obj, redis, "redis", 0, false);
ucl_object_insert_key (redis, ucl_object_fromstring (redis_host),
"servers", 0, false);
if (redis_db) {
ucl_object_insert_key (redis, ucl_object_fromstring (redis_db),
"dbname", 0, false);
}
if (redis_password) {
ucl_object_insert_key (redis, ucl_object_fromstring (redis_password),
"password", 0, false);
}
}
ucl_object_insert_key (obj, ucl_object_frombool (reset_previous),
"reset_previous", 0, false);
if (expire != 0) {
ucl_object_insert_key (obj, ucl_object_fromdouble (expire),
"expire", 0, false);
}
rspamadm_execute_lua_ucl_subr (argc,
argv,
obj,
"stat_convert",
TRUE);
ucl_object_unref (obj);
}
int
main(int argc, char **argv)
{
const char *fn = NULL;
unsigned char *inbuf;
struct ucl_parser *parser;
int k, ret = 0, r = 0;
ssize_t bufsize;
ucl_object_t *obj = NULL;
const ucl_object_t *par;
FILE *in;
if (argc > 1) {
fn = argv[1];
}
if (fn != NULL) {
in = fopen (fn, "r");
if (in == NULL) {
exit (-errno);
}
}
else {
in = stdin;
}
parser = ucl_parser_new (0);
inbuf = malloc (BUFSIZ);
bufsize = BUFSIZ;
r = 0;
while (!feof (in) && !ferror (in)) {
if (r == bufsize) {
inbuf = realloc (inbuf, bufsize * 2);
bufsize *= 2;
if (inbuf == NULL) {
perror ("realloc");
exit (EXIT_FAILURE);
}
}
r += fread (inbuf + r, 1, bufsize - r, in);
}
if (ferror (in)) {
fprintf (stderr, "Failed to read the input file.\n");
exit (EXIT_FAILURE);
}
ucl_parser_add_chunk (parser, inbuf, r);
fclose (in);
if (ucl_parser_get_error(parser)) {
printf ("Error occured: %s\n", ucl_parser_get_error(parser));
ret = 1;
goto end;
}
obj = ucl_parser_get_object (parser);
if (ucl_parser_get_error (parser)) {
printf ("Error occured: %s\n", ucl_parser_get_error(parser));
ret = 1;
goto end;
}
if (argc > 2) {
for (k = 2; k < argc; k++) {
printf ("search for \"%s\"... ", argv[k]);
par = ucl_object_find_key (obj, argv[k]);
printf ("%sfound\n", (par == NULL )?"not ":"");
ucl_obj_dump (par, 0);
}
}
else {
ucl_obj_dump (obj, 0);
}
end:
if (parser != NULL) {
ucl_parser_free (parser);
}
if (obj != NULL) {
ucl_object_unref (obj);
}
return ret;
}
static void
rspamadm_signtool (gint argc, gchar **argv)
{
GOptionContext *context;
GError *error = NULL;
struct ucl_parser *parser;
ucl_object_t *top;
struct rspamd_cryptobox_pubkey *pk;
struct rspamd_cryptobox_keypair *kp;
gsize fsize, flen;
gint i;
context = g_option_context_new (
"keypair - create encryption keys");
g_option_context_set_summary (context,
"Summary:\n Rspamd administration utility version "
RVERSION
"\n Release id: "
RID);
g_option_context_add_main_entries (context, entries, NULL);
if (!g_option_context_parse (context, &argc, &argv, &error)) {
rspamd_fprintf (stderr, "option parsing failed: %s\n", error->message);
g_error_free (error);
exit (1);
}
if (openssl) {
mode = RSPAMD_CRYPTOBOX_MODE_NIST;
}
if (verify && (!pubkey && !pubkey_file)) {
rspamd_fprintf (stderr, "no pubkey for verification\n");
exit (1);
}
else if (!verify && (!keypair_file)) {
rspamd_fprintf (stderr, "no keypair for signing\n");
exit (1);
}
if (verify) {
g_assert (pubkey || pubkey_file);
if (pubkey_file) {
gint fd;
gchar *map;
struct stat st;
fd = open (pubkey_file, O_RDONLY);
if (fd == -1) {
rspamd_fprintf (stderr, "cannot open %s: %s\n", pubkey_file,
strerror (errno));
exit (errno);
}
g_assert (fstat (fd, &st) != -1);
fsize = st.st_size;
flen = fsize;
map = mmap (NULL, fsize, PROT_READ, MAP_SHARED, fd, 0);
close (fd);
if (map == MAP_FAILED) {
rspamd_fprintf (stderr, "cannot read %s: %s\n", pubkey_file,
strerror (errno));
exit (errno);
}
/* XXX: assume base32 pubkey now */
while (flen > 0 && g_ascii_isspace (map[flen - 1])) {
flen --;
}
pk = rspamd_pubkey_from_base32 (map, flen,
RSPAMD_KEYPAIR_SIGN, mode);
if (pk == NULL) {
rspamd_fprintf (stderr, "bad size %s: %ud, %ud expected\n", flen,
rspamd_cryptobox_pk_sig_bytes (mode));
exit (errno);
}
munmap (map, fsize);
}
else {
pk = rspamd_pubkey_from_base32 (pubkey, strlen (pubkey),
RSPAMD_KEYPAIR_SIGN, mode);
if (pk == NULL) {
rspamd_fprintf (stderr, "bad size %s: %ud, %ud expected\n",
strlen (pubkey),
rspamd_cryptobox_pk_sig_bytes (mode));
exit (errno);
}
}
for (i = 1; i < argc; i++) {
/* XXX: support cmd line signature */
if (!rspamadm_verify_file (argv[i], rspamd_pubkey_get_pk (pk, NULL))) {
exit (EXIT_FAILURE);
}
}
g_free (pk);
}
else {
g_assert (keypair_file != NULL);
parser = ucl_parser_new (0);
if (!ucl_parser_add_file (parser, keypair_file) ||
(top = ucl_parser_get_object (parser)) == NULL) {
rspamd_fprintf (stderr, "cannot load keypair: %s\n",
ucl_parser_get_error (parser));
exit (EINVAL);
}
ucl_parser_free (parser);
kp = rspamd_keypair_from_ucl (top);
if (kp == NULL || rspamd_keypair_type (kp) != RSPAMD_KEYPAIR_SIGN) {
rspamd_fprintf (stderr, "invalid or unsuitable for signing key\n");
exit (EXIT_FAILURE);
}
for (i = 1; i < argc; i++) {
/* XXX: support cmd line signature */
if (!rspamadm_sign_file (argv[i], kp)) {
rspamd_keypair_unref (kp);
exit (EXIT_FAILURE);
}
}
rspamd_keypair_unref (kp);
}
}
int
main (int argc, char **argv)
{
ucl_object_t *obj, *cur, *ar, *ar1, *ref, *test_obj, *comments;
ucl_object_iter_t it;
const ucl_object_t *found, *it_obj, *test;
struct ucl_emitter_functions *fn;
FILE *out;
unsigned char *emitted;
const char *fname_out = NULL;
struct ucl_parser *parser;
int ret = 0;
switch (argc) {
case 2:
fname_out = argv[1];
break;
}
if (fname_out != NULL) {
out = fopen (fname_out, "w");
if (out == NULL) {
exit (-errno);
}
}
else {
out = stdout;
}
obj = ucl_object_typed_new (UCL_OBJECT);
/* Keys replacing */
cur = ucl_object_fromstring_common ("value1", 0, UCL_STRING_TRIM);
ucl_object_insert_key (obj, cur, "key0", 0, false);
cur = ucl_object_fromdouble (0.1);
assert (ucl_object_replace_key (obj, cur, "key0", 0, false));
/* Create some strings */
cur = ucl_object_fromstring_common (" test string ", 0, UCL_STRING_TRIM);
ucl_object_insert_key (obj, cur, "key1", 0, false);
cur = ucl_object_fromstring_common (" test \nstring\n\r\n\b\t\f\\\" ", 0,
UCL_STRING_TRIM | UCL_STRING_ESCAPE);
ucl_object_insert_key (obj, cur, "key2", 0, false);
cur = ucl_object_fromstring_common (" test string \n", 0, 0);
ucl_object_insert_key (obj, cur, "key3", 0, false);
/* Array of numbers */
ar = ucl_object_typed_new (UCL_ARRAY);
cur = ucl_object_fromint (10);
ucl_array_append (ar, cur);
assert (ucl_array_index_of (ar, cur) == 0);
cur = ucl_object_fromdouble (10.1);
ucl_array_append (ar, cur);
assert (ucl_array_index_of (ar, cur) == 1);
cur = ucl_object_fromdouble (9.999);
ucl_array_prepend (ar, cur);
assert (ucl_array_index_of (ar, cur) == 0);
ar1 = ucl_object_copy (ar);
cur = ucl_object_fromstring ("abc");
ucl_array_prepend (ar1, cur);
cur = ucl_object_fromstring ("cde");
ucl_array_prepend (ar1, cur);
cur = ucl_object_fromstring ("аАаБаВ"); /* UTF8 */
ucl_array_prepend (ar1, cur);
cur = ucl_object_fromstring ("ааБаВ"); /* UTF8 */
ucl_array_prepend (ar1, cur);
/*
* This is ususally broken or fragile as utf collate is far from perfect
cur = ucl_object_fromstring ("баБаВ");
ucl_array_prepend (ar1, cur);
cur = ucl_object_fromstring ("ааБаВ"); // hello to @bapt
ucl_array_prepend (ar1, cur);
*/
cur = ucl_object_fromstring ("№"); /* everybody likes emoji in the code */
ucl_array_prepend (ar1, cur);
ucl_object_array_sort (ar1, ucl_object_compare_qsort);
/* Removing from an array */
cur = ucl_object_fromdouble (1.0);
ucl_array_append (ar, cur);
cur = ucl_array_delete (ar, cur);
assert (ucl_object_todouble (cur) == 1.0);
ucl_object_unref (cur);
cur = ucl_object_fromdouble (2.0);
ucl_array_append (ar, cur);
cur = ucl_array_pop_last (ar);
assert (ucl_object_todouble (cur) == 2.0);
ucl_object_unref (cur);
cur = ucl_object_fromdouble (3.0);
ucl_array_prepend (ar, cur);
cur = ucl_array_pop_first (ar);
assert (ucl_object_todouble (cur) == 3.0);
ucl_object_unref (cur);
ucl_object_insert_key (obj, ar, "key4", 0, false);
cur = ucl_object_frombool (true);
/* Ref object to test refcounts */
ref = ucl_object_ref (cur);
ucl_object_insert_key (obj, cur, "key4", 0, false);
/* Empty strings */
cur = ucl_object_fromstring_common (" ", 0, UCL_STRING_TRIM);
ucl_object_insert_key (obj, cur, "key5", 0, false);
cur = ucl_object_fromstring_common ("", 0, UCL_STRING_ESCAPE);
ucl_object_insert_key (obj, cur, "key6", 0, false);
cur = ucl_object_fromstring_common (" \n", 0, UCL_STRING_ESCAPE);
ucl_object_insert_key (obj, cur, "key7", 0, false);
/* Numbers and booleans */
cur = ucl_object_fromstring_common ("1mb", 0, UCL_STRING_ESCAPE | UCL_STRING_PARSE);
ucl_object_insert_key (obj, cur, "key8", 0, false);
cur = ucl_object_fromstring_common ("3.14", 0, UCL_STRING_PARSE);
ucl_object_insert_key (obj, cur, "key9", 0, false);
cur = ucl_object_fromstring_common ("true", 0, UCL_STRING_PARSE);
ucl_object_insert_key (obj, cur, "key10", 0, false);
cur = ucl_object_fromstring_common (" off ", 0, UCL_STRING_PARSE | UCL_STRING_TRIM);
ucl_object_insert_key (obj, cur, "key11", 0, false);
cur = ucl_object_fromstring_common ("*****@*****.**", 0, UCL_STRING_PARSE_INT);
ucl_object_insert_key (obj, cur, "key12", 0, false);
cur = ucl_object_fromstring_common ("#test", 0, UCL_STRING_PARSE_INT);
ucl_object_insert_key (obj, cur, "key13", 0, false);
cur = ucl_object_frombool (true);
ucl_object_insert_key (obj, cur, "k=3", 0, false);
ucl_object_insert_key (obj, ar1, "key14", 0, false);
cur = ucl_object_new_userdata (ud_dtor, ud_emit, NULL);
ucl_object_insert_key (obj, cur, "key15", 0, false);
/* More tests for keys */
cur = ucl_object_fromlstring ("test", 3);
ucl_object_insert_key (obj, cur, "key16", 0, false);
test = ucl_object_lookup_any (obj, "key100", "key200", "key300", "key16", NULL);
assert (test == cur);
test = ucl_object_lookup_len (obj, "key160", 5);
assert (test == cur);
cur = ucl_object_pop_key (obj, "key16");
assert (test == cur);
test = ucl_object_pop_key (obj, "key16");
assert (test == NULL);
test = ucl_object_lookup_len (obj, "key160", 5);
assert (test == NULL);
/* Objects merging tests */
test_obj = ucl_object_new_full (UCL_OBJECT, 2);
ucl_object_insert_key (test_obj, cur, "key16", 0, true);
ucl_object_merge (obj, test_obj, true);
ucl_object_unref (test_obj);
/* Array merging test */
test_obj = ucl_object_new_full (UCL_ARRAY, 3);
ucl_array_append (test_obj, ucl_object_fromstring ("test"));
ucl_array_merge (test_obj, ar1, false);
ucl_object_insert_key (obj, test_obj, "key17", 0, true);
/* Object deletion */
cur = ucl_object_fromstring ("test");
ucl_object_insert_key (obj, cur, "key18", 0, true);
assert (ucl_object_delete_key (obj, "key18"));
assert (!ucl_object_delete_key (obj, "key18"));
cur = ucl_object_fromlstring ("test", 4);
ucl_object_insert_key (obj, cur, "key18\0\0", 7, true);
assert (ucl_object_lookup_len (obj, "key18\0\0", 7) == cur);
assert (ucl_object_lookup (obj, "key18") == NULL);
assert (ucl_object_lookup_len (obj, "key18\0\1", 7) == NULL);
assert (ucl_object_delete_keyl (obj, "key18\0\0", 7));
/* Comments */
comments = ucl_object_typed_new (UCL_OBJECT);
found = ucl_object_lookup (obj, "key17");
test = ucl_object_lookup (obj, "key16");
ucl_comments_add (comments, found, "# test comment");
assert (ucl_comments_find (comments, found) != NULL);
assert (ucl_comments_find (comments, test) == NULL);
ucl_comments_move (comments, found, test);
assert (ucl_comments_find (comments, found) == NULL);
assert (ucl_comments_find (comments, test) != NULL);
/* Array replace */
ar1 = ucl_object_typed_new (UCL_ARRAY);
cur = ucl_object_fromstring ("test");
cur = ucl_elt_append (cur, ucl_object_fromstring ("test1"));
ucl_array_append (ar1, cur);
test = ucl_array_replace_index (ar1, ucl_object_fromstring ("test2"), 0);
assert (test == cur);
/* Try to find using path */
/* Should exist */
found = ucl_object_lookup_path (obj, "key4.1");
assert (found != NULL && ucl_object_toint (found) == 10);
/* . should be ignored */
found = ucl_object_lookup_path (obj, ".key4.1");
assert (found != NULL && ucl_object_toint (found) == 10);
/* moar dots... */
found = ucl_object_lookup_path (obj, ".key4........1...");
assert (found != NULL && ucl_object_toint (found) == 10);
/* No such index */
found = ucl_object_lookup_path (obj, ".key4.3");
assert (found == NULL);
/* No such key */
found = ucl_object_lookup_path (obj, "key9..key1");
assert (found == NULL);
/* Test iteration */
it = ucl_object_iterate_new (obj);
it_obj = ucl_object_iterate_safe (it, true);
/* key0 = 0.1 */
assert (ucl_object_type (it_obj) == UCL_FLOAT);
it_obj = ucl_object_iterate_safe (it, true);
/* key1 = "" */
assert (ucl_object_type (it_obj) == UCL_STRING);
it_obj = ucl_object_iterate_safe (it, true);
/* key2 = "" */
assert (ucl_object_type (it_obj) == UCL_STRING);
it_obj = ucl_object_iterate_safe (it, true);
/* key3 = "" */
assert (ucl_object_type (it_obj) == UCL_STRING);
it_obj = ucl_object_iterate_safe (it, true);
/* key4 = ([float, int, float], boolean) */
ucl_object_iterate_reset (it, it_obj);
it_obj = ucl_object_iterate_safe (it, true);
assert (ucl_object_type (it_obj) == UCL_FLOAT);
it_obj = ucl_object_iterate_safe (it, true);
assert (ucl_object_type (it_obj) == UCL_INT);
it_obj = ucl_object_iterate_safe (it, true);
assert (ucl_object_type (it_obj) == UCL_FLOAT);
it_obj = ucl_object_iterate_safe (it, true);
assert (ucl_object_type (it_obj) == UCL_BOOLEAN);
ucl_object_iterate_free (it);
fn = ucl_object_emit_memory_funcs (&emitted);
assert (ucl_object_emit_full (obj, UCL_EMIT_CONFIG, fn, comments));
fprintf (out, "%s\n", emitted);
ucl_object_emit_funcs_free (fn);
ucl_object_unref (obj);
ucl_object_unref (comments);
parser = ucl_parser_new (UCL_PARSER_NO_IMPLICIT_ARRAYS);
if (ucl_parser_add_chunk_full (parser, emitted, strlen (emitted),
3, UCL_DUPLICATE_ERROR, UCL_PARSE_UCL)) {
/* Should fail due to duplicate */
assert (0);
}
else {
assert (ucl_parser_get_error (parser) != NULL);
ucl_parser_clear_error (parser);
ucl_parser_free (parser);
parser = ucl_parser_new (0);
ucl_parser_add_chunk_full (parser, emitted, strlen (emitted),
3, UCL_DUPLICATE_MERGE, UCL_PARSE_UCL);
}
assert (ucl_parser_get_column (parser) == 0);
assert (ucl_parser_get_linenum (parser) != 0);
ucl_parser_clear_error (parser);
assert (ucl_parser_get_error_code (parser) == 0);
obj = ucl_parser_get_object (parser);
ucl_parser_free (parser);
ucl_object_free (obj);
if (emitted != NULL) {
free (emitted);
}
fclose (out);
/* Ref should still be accessible */
ref->value.iv = 100500;
ucl_object_unref (ref);
return ret;
}
gboolean
rspamd_task_load_message (struct rspamd_task *task,
struct rspamd_http_message *msg, const gchar *start, gsize len)
{
guint control_len, r;
struct ucl_parser *parser;
ucl_object_t *control_obj;
gchar filepath[PATH_MAX], *fp;
gint fd, flen;
gpointer map;
struct stat st;
if (msg) {
rspamd_protocol_handle_headers (task, msg);
}
if (task->flags & RSPAMD_TASK_FLAG_FILE) {
g_assert (task->msg.len > 0);
r = rspamd_strlcpy (filepath, task->msg.start,
MIN (sizeof (filepath), task->msg.len + 1));
rspamd_decode_url (filepath, filepath, r + 1);
flen = strlen (filepath);
if (filepath[0] == '"' && flen > 2) {
/* We need to unquote filepath */
fp = &filepath[1];
fp[flen - 2] = '\0';
}
else {
fp = &filepath[0];
}
if (access (fp, R_OK) == -1 || stat (fp, &st) == -1) {
g_set_error (&task->err, rspamd_task_quark(), RSPAMD_PROTOCOL_ERROR,
"Invalid file (%s): %s", fp, strerror (errno));
return FALSE;
}
fd = open (fp, O_RDONLY);
if (fd == -1) {
g_set_error (&task->err, rspamd_task_quark(), RSPAMD_PROTOCOL_ERROR,
"Cannot open file (%s): %s", fp, strerror (errno));
return FALSE;
}
map = mmap (NULL, st.st_size, PROT_READ, MAP_SHARED, fd, 0);
if (map == MAP_FAILED) {
close (fd);
g_set_error (&task->err, rspamd_task_quark(), RSPAMD_PROTOCOL_ERROR,
"Cannot mmap file (%s): %s", fp, strerror (errno));
return FALSE;
}
close (fd);
task->msg.start = map;
task->msg.len = st.st_size;
rspamd_mempool_add_destructor (task->task_pool, rspamd_task_unmapper, task);
}
else {
debug_task ("got input of length %z", task->msg.len);
task->msg.start = start;
task->msg.len = len;
if (task->msg.len == 0) {
msg_warn_task ("message has invalid message length: %ud",
task->msg.len);
g_set_error (&task->err, rspamd_task_quark(), RSPAMD_PROTOCOL_ERROR,
"Invalid length");
return FALSE;
}
if (task->flags & RSPAMD_TASK_FLAG_HAS_CONTROL) {
/* We have control chunk, so we need to process it separately */
if (task->msg.len < task->message_len) {
msg_warn_task ("message has invalid message length: %ud and total len: %ud",
task->message_len, task->msg.len);
g_set_error (&task->err, rspamd_task_quark(), RSPAMD_PROTOCOL_ERROR,
"Invalid length");
return FALSE;
}
control_len = task->msg.len - task->message_len;
if (control_len > 0) {
parser = ucl_parser_new (UCL_PARSER_KEY_LOWERCASE);
if (!ucl_parser_add_chunk (parser, task->msg.start, control_len)) {
msg_warn_task ("processing of control chunk failed: %s",
ucl_parser_get_error (parser));
ucl_parser_free (parser);
}
else {
control_obj = ucl_parser_get_object (parser);
ucl_parser_free (parser);
rspamd_protocol_handle_control (task, control_obj);
ucl_object_unref (control_obj);
}
task->msg.start += control_len;
task->msg.len -= control_len;
}
}
}
return TRUE;
}
int
main(int argc, char **argv)
{
const char *fn = NULL;
char inbuf[8192];
struct ucl_parser *parser;
int k, ret = 0, r = 0;
ucl_object_t *obj = NULL;
const ucl_object_t *par;
FILE *in;
if (argc > 1) {
fn = argv[1];
}
if (fn != NULL) {
in = fopen (fn, "r");
if (in == NULL) {
exit (-errno);
}
}
else {
in = stdin;
}
parser = ucl_parser_new (0);
while (!feof (in) && r < (int)sizeof (inbuf)) {
r += fread (inbuf + r, 1, sizeof (inbuf) - r, in);
}
ucl_parser_add_chunk (parser, inbuf, r);
fclose (in);
if (ucl_parser_get_error(parser)) {
printf ("Error occured: %s\n", ucl_parser_get_error(parser));
ret = 1;
goto end;
}
obj = ucl_parser_get_object (parser);
if (ucl_parser_get_error (parser)) {
printf ("Error occured: %s\n", ucl_parser_get_error(parser));
ret = 1;
goto end;
}
if (argc > 2) {
for (k = 2; k < argc; k++) {
printf ("search for \"%s\"... ", argv[k]);
par = ucl_object_find_key (obj, argv[k]);
printf ("%sfound\n", (par == NULL )?"not ":"");
ucl_obj_dump (par, 0);
}
}
else {
ucl_obj_dump (obj, 0);
}
end:
if (parser != NULL) {
ucl_parser_free (parser);
}
if (obj != NULL) {
ucl_object_unref (obj);
}
return ret;
}
int
main (int argc, char **argv)
{
char inbuf[8192], *test_in = NULL;
struct ucl_parser *parser = NULL, *parser2 = NULL;
ucl_object_t *obj;
FILE *in, *out;
unsigned char *emitted = NULL;
const char *fname_in = NULL, *fname_out = NULL;
int ret = 0, inlen, opt, json = 0, compact = 0, yaml = 0;
while ((opt = getopt(argc, argv, "jcy")) != -1) {
switch (opt) {
case 'j':
json = 1;
break;
case 'c':
compact = 1;
break;
case 'y':
yaml = 1;
break;
default: /* '?' */
fprintf (stderr, "Usage: %s [-jcy] [in] [out]\n",
argv[0]);
exit (EXIT_FAILURE);
}
}
argc -= optind;
argv += optind;
switch (argc) {
case 1:
fname_in = argv[0];
break;
case 2:
fname_in = argv[0];
fname_out = argv[1];
break;
}
if (fname_in != NULL) {
in = fopen (fname_in, "r");
if (in == NULL) {
exit (-errno);
}
}
else {
in = stdin;
}
parser = ucl_parser_new (UCL_PARSER_KEY_LOWERCASE);
ucl_parser_register_variable (parser, "ABI", "unknown");
if (fname_in != NULL) {
ucl_parser_set_filevars (parser, fname_in, true);
}
while (!feof (in)) {
memset (inbuf, 0, sizeof (inbuf));
if (fread (inbuf, 1, sizeof (inbuf) - 1, in) == 0) {
break;
}
inlen = strlen (inbuf);
test_in = malloc (inlen);
memcpy (test_in, inbuf, inlen);
ucl_parser_add_chunk (parser, test_in, inlen);
}
fclose (in);
if (fname_out != NULL) {
out = fopen (fname_out, "w");
if (out == NULL) {
exit (-errno);
}
}
else {
out = stdout;
}
if (ucl_parser_get_error (parser) != NULL) {
fprintf (out, "Error occurred: %s\n", ucl_parser_get_error(parser));
ret = 1;
goto end;
}
obj = ucl_parser_get_object (parser);
if (json) {
if (compact) {
emitted = ucl_object_emit (obj, UCL_EMIT_JSON_COMPACT);
}
else {
emitted = ucl_object_emit (obj, UCL_EMIT_JSON);
}
}
else if (yaml) {
emitted = ucl_object_emit (obj, UCL_EMIT_YAML);
}
else {
emitted = ucl_object_emit (obj, UCL_EMIT_CONFIG);
}
ucl_parser_free (parser);
ucl_object_unref (obj);
parser2 = ucl_parser_new (UCL_PARSER_KEY_LOWERCASE);
ucl_parser_add_string (parser2, emitted, 0);
if (ucl_parser_get_error(parser2) != NULL) {
fprintf (out, "Error occurred: %s\n", ucl_parser_get_error(parser2));
fprintf (out, "%s\n", emitted);
ret = 1;
goto end;
}
if (emitted != NULL) {
free (emitted);
}
obj = ucl_parser_get_object (parser2);
if (json) {
if (compact) {
emitted = ucl_object_emit (obj, UCL_EMIT_JSON_COMPACT);
}
else {
emitted = ucl_object_emit (obj, UCL_EMIT_JSON);
}
}
else if (yaml) {
emitted = ucl_object_emit (obj, UCL_EMIT_YAML);
}
else {
emitted = ucl_object_emit (obj, UCL_EMIT_CONFIG);
}
fprintf (out, "%s\n", emitted);
ucl_object_unref (obj);
end:
if (emitted != NULL) {
free (emitted);
}
if (parser2 != NULL) {
ucl_parser_free (parser2);
}
if (test_in != NULL) {
free (test_in);
}
fclose (out);
return ret;
}