static gboolean iopen_add_result(struct iopen_result *r) {
bool should_add = true;
GtkTreeIter mah;
if (gtk_tree_model_get_iter_first(GTK_TREE_MODEL(results_list), &mah)) {
do {
GValue path_value = { 0 }, show_value = { 0 };
gtk_tree_model_get_value(GTK_TREE_MODEL(results_list), &mah, 1, &path_value);
gtk_tree_model_get_value(GTK_TREE_MODEL(results_list), &mah, 0, &show_value);
const char *path = g_value_get_string(&path_value);
const char *show = g_value_get_string(&show_value);
if ((null_strcmp(r->show, show) == 0) && (null_strcmp(r->path, path) == 0)) should_add = false;
g_value_unset(&path_value);
g_value_unset(&show_value);
} while (should_add && gtk_tree_model_iter_next(GTK_TREE_MODEL(results_list), &mah));
}
if (should_add) {
gtk_list_store_append(results_list, &mah);
gtk_list_store_set(results_list, &mah, 0, r->show, 1, r->path, 2, (r->search != NULL) ? r->search : "", 3, r->rank, -1);
}
iopen_result_free(r);
return FALSE;
}
static bool cmp_addr(const struct message_address *a1,
const struct message_address *a2)
{
return null_strcmp(a1->name, a2->name) == 0 &&
null_strcmp(a1->route, a2->route) == 0 &&
null_strcmp(a1->mailbox, a2->mailbox) == 0 &&
null_strcmp(a1->domain, a2->domain) == 0 &&
a1->invalid_syntax == a2->invalid_syntax;
}
static void
auth_str_append_extra_fields(struct auth_request *request, string_t *dest)
{
if (!auth_fields_is_empty(request->extra_fields)) {
str_append_c(dest, '\t');
auth_fields_append(request->extra_fields, dest,
AUTH_FIELD_FLAG_HIDDEN, 0);
}
if (request->original_username != NULL &&
null_strcmp(request->original_username, request->user) != 0) {
auth_str_add_keyvalue(dest, "original_user",
request->original_username);
}
if (request->master_user != NULL)
auth_str_add_keyvalue(dest, "auth_user", request->master_user);
if (!request->auth_only &&
auth_fields_exists(request->extra_fields, "proxy")) {
/* we're proxying */
if (!auth_fields_exists(request->extra_fields, "pass") &&
request->mech_password != NULL) {
/* send back the password that was sent by user
(not the password in passdb). */
auth_str_add_keyvalue(dest, "pass",
request->mech_password);
}
if (request->master_user != NULL &&
!auth_fields_exists(request->extra_fields, "master")) {
/* the master username needs to be forwarded */
auth_str_add_keyvalue(dest, "master",
request->master_user);
}
}
}
void test_str_sanitize(void)
{
static struct str_sanitize_test tests[] = {
{ NULL, 2, NULL },
{ "", 2, NULL },
{ "a", 2, NULL },
{ "ab", 2, NULL },
{ "abc", 2, "..." },
{ "abcd", 3, "..." },
{ "abcde", 4, "a..." },
{ "с", 10, NULL },
{ "с", 1, NULL },
{ "\001x\x1fy\x81", 10, "?x?y?" }
};
const char *str;
unsigned int i;
test_begin("str_sanitize");
for (i = 0; i < N_ELEMENTS(tests); i++) {
str = str_sanitize(tests[i].str, tests[i].max_len);
if (tests[i].sanitized != NULL)
test_assert_idx(null_strcmp(str, tests[i].sanitized) == 0, i);
else
test_assert_idx(str == tests[i].str, i);
}
test_end();
}
static bool
acl_mailbox_update_removed_id(struct acl_object *aclobj,
const struct acl_rights_update *update)
{
struct acl_object_list_iter *iter;
struct acl_rights rights;
int ret;
if (update->modify_mode != ACL_MODIFY_MODE_CLEAR &&
update->neg_modify_mode != ACL_MODIFY_MODE_CLEAR)
return FALSE;
if (update->modify_mode == ACL_MODIFY_MODE_CLEAR &&
update->neg_modify_mode == ACL_MODIFY_MODE_CLEAR)
return TRUE;
/* mixed clear/non-clear. see if the identifier exists anymore */
iter = acl_object_list_init(aclobj);
while ((ret = acl_object_list_next(iter, &rights)) > 0) {
if (rights.id_type == update->rights.id_type &&
null_strcmp(rights.identifier, update->rights.identifier) == 0)
break;
}
acl_object_list_deinit(&iter);
return ret == 0;
}
void test_json_tree(void)
{
struct json_tree *tree;
struct json_tree_node *root, *node, *node1, *node2;
unsigned int i;
test_begin("json tree");
tree = json_tree_init();
for (i = 0; i < N_ELEMENTS(test_input); i++) {
test_assert_idx(json_tree_append(tree, test_input[i].type,
test_input[i].value) == 0, i);
}
root = json_tree_root(tree);
i_assert(root != NULL);
test_assert(root->value_type == JSON_TYPE_OBJECT);
root = root->value.child;
i_assert(root != NULL);
for (i = 0; i < 10+2; i += 2) {
node = json_tree_find_key(root, test_input[i].value);
test_assert(node != NULL &&
node->value_type == test_input[i+1].type &&
null_strcmp(node->value.str, test_input[i+1].value) == 0);
}
node = json_tree_find_key(root, "key-obj");
test_assert(node != NULL);
node = json_tree_find_key(root, "key-arr-empty");
test_assert(node != NULL && node->value_type == JSON_TYPE_ARRAY &&
node->value.child == NULL);
node = json_tree_find_key(root, "key-arr");
test_assert(node != NULL && node->value_type == JSON_TYPE_ARRAY);
node = node->value.child;
test_assert(node != NULL && node->value_type == JSON_TYPE_STRING &&
strcmp(node->value.str, "foo") == 0);
node = node->next;
test_assert(node != NULL && node->value_type == JSON_TYPE_ARRAY &&
node->value.child != NULL &&
node->value.child->next == NULL &&
node->value.child->value_type == JSON_TYPE_TRUE);
node = node->next;
test_assert(node != NULL && node->value_type == JSON_TYPE_OBJECT &&
node->value.child != NULL &&
node->value.child->next == NULL &&
node->value.child->value_type == JSON_TYPE_ARRAY &&
node->value.child->value.child == NULL);
node1 = json_tree_find_child_with(node->parent, "aobj-key", "value1");
node2 = json_tree_find_child_with(node->parent, "aobj-key", "value2");
test_assert(node1 != NULL && node2 != NULL && node1 != node2);
test_assert(json_tree_find_child_with(node->parent, "aobj-key", "value3") == NULL);
json_tree_deinit(&tree);
test_end();
}
bool ldap_connection_have_settings(struct ldap_connection *conn,
const struct ldap_client_settings *set)
{
const struct ldap_client_settings *conn_set = &conn->set;
if (strcmp(conn_set->uri, set->uri) != 0)
return FALSE;
if (null_strcmp(conn_set->bind_dn, set->bind_dn) != 0)
return FALSE;
if (null_strcmp(conn_set->password, set->password) != 0)
return FALSE;
if (conn_set->timeout_secs != set->timeout_secs ||
conn_set->max_idle_time_secs != set->max_idle_time_secs ||
conn_set->debug != set->debug ||
conn_set->require_ssl != set->require_ssl ||
conn_set->start_tls != set->start_tls)
return FALSE;
if (set->ssl_set == NULL || !set->start_tls)
return TRUE;
/* check SSL settings */
if (null_strcmp(conn->ssl_set.protocols, set->ssl_set->protocols) != 0)
return FALSE;
if (null_strcmp(conn->ssl_set.cipher_list, set->ssl_set->cipher_list) != 0)
return FALSE;
if (null_strcmp(conn->ssl_set.ca_file, set->ssl_set->ca_file) != 0)
return FALSE;
if (null_strcmp(conn->ssl_set.cert, set->ssl_set->cert) != 0)
return FALSE;
if (null_strcmp(conn->ssl_set.key, set->ssl_set->key) != 0)
return FALSE;
return TRUE;
}
void test_str_sanitize(void)
{
static struct str_sanitize_test tests[] = {
{ NULL, 2, NULL },
{ "", 2, NULL },
{ "a", 2, NULL },
{ "ab", 2, NULL },
{ "abc", 2, "..." },
{ "abcd", 3, "..." },
{ "abcde", 4, "a..." },
{ "\xD1\x81", 1, "..." },
{ "\xD1\x81", 2, "\xD1\x81" },
{ "\xD1\x81", 3, NULL },
{ "\xC3\xA4\xC3\xA4zyxa", 1, "..." },
{ "\xC3\xA4\xC3\xA4zyxa", 2, "..." },
{ "\xC3\xA4\xC3\xA4zyxa", 3, "..." },
{ "\xC3\xA4\xC3\xA4zyxa", 4, "..." },
{ "\xC3\xA4\xC3\xA4zyxa", 5, "\xC3\xA4..." },
{ "\xC3\xA4\xC3\xA4zyxa", 6, "\xC3\xA4..." },
{ "\xC3\xA4\xC3\xA4zyxa", 7, "\xC3\xA4\xC3\xA4..." },
{ "\xC3\xA4\xC3\xA4zyxa", 8, "\xC3\xA4\xC3\xA4zyxa" },
{ "\001x\x1fy\x81", 10, "?x?y?" }
};
const char *str;
string_t *str2;
unsigned int i;
test_begin("str_sanitize");
for (i = 0; i < N_ELEMENTS(tests); i++) {
str = str_sanitize(tests[i].str, tests[i].max_len);
if (tests[i].sanitized != NULL)
test_assert_idx(null_strcmp(str, tests[i].sanitized) == 0, i);
else
test_assert_idx(str == tests[i].str, i);
}
test_end();
test_begin("str_sanitize_append");
str2 = t_str_new(128);
for (i = 0; i < N_ELEMENTS(tests); i++) {
if (tests[i].str == NULL)
continue;
str_truncate(str2, 0);
str_append(str2, "1234567890");
str_sanitize_append(str2, tests[i].str, tests[i].max_len);
test_assert_idx(strncmp(str_c(str2), "1234567890", 10) == 0, i);
if (tests[i].sanitized != NULL)
test_assert_idx(strcmp(str_c(str2)+10, tests[i].sanitized) == 0, i);
else
test_assert_idx(strcmp(str_c(str2)+10, tests[i].str) == 0, i);
}
test_end();
}
static void test_json_parser_success(bool full_size)
{
struct json_parser *parser;
struct istream *input, *jsoninput = NULL;
enum json_type type;
const char *value, *error;
unsigned int i, pos, json_input_len = strlen(json_input);
int ret = 0;
test_begin(full_size ? "json parser" : "json parser (nonblocking)");
input = test_istream_create_data(json_input, json_input_len);
test_istream_set_allow_eof(input, FALSE);
parser = json_parser_init(input);
i = full_size ? json_input_len : 0;
for (pos = 0; i <= json_input_len; i++) {
test_istream_set_size(input, i);
for (;;) {
value = NULL;
if (pos < N_ELEMENTS(json_output) &&
json_output[pos].type == (enum json_type)TYPE_SKIP) {
json_parse_skip_next(parser);
pos++;
continue;
} else if (pos == N_ELEMENTS(json_output) ||
json_output[pos].type != (enum json_type)TYPE_STREAM) {
ret = json_parse_next(parser, &type, &value);
} else {
ret = jsoninput != NULL ? 1 :
json_parse_next_stream(parser, &jsoninput);
if (jsoninput != NULL)
ret = stream_read_value(&jsoninput, &value);
type = TYPE_STREAM;
}
if (ret <= 0)
break;
i_assert(pos < N_ELEMENTS(json_output));
test_assert(json_output[pos].type == type);
test_assert(null_strcmp(json_output[pos].value, value) == 0);
pos++;
}
test_assert(ret == 0);
}
test_assert(pos == N_ELEMENTS(json_output));
test_istream_set_allow_eof(input, TRUE);
test_assert(json_parse_next(parser, &type, &value) == -1);
i_stream_unref(&input);
test_assert(json_parser_deinit(&parser, &error) == 0);
test_end();
}
bool config_filters_equal(const struct config_filter *f1,
const struct config_filter *f2)
{
if (null_strcmp(f1->service, f2->service) != 0)
return FALSE;
if (f1->remote_bits != f2->remote_bits)
return FALSE;
if (!net_ip_compare(&f1->remote_net, &f2->remote_net))
return FALSE;
if (f1->local_bits != f2->local_bits)
return FALSE;
if (!net_ip_compare(&f1->local_net, &f2->local_net))
return FALSE;
if (null_strcmp(f1->local_name, f2->local_name) != 0)
return FALSE;
return TRUE;
}
static void test_fts_filter_english_possessive(void)
{
struct fts_filter *norm = NULL;
const char *input[] = {
"foo'",
"foo's",
"foo\xC3\xA4's",
"foo'S",
"foos'S",
"foo's's",
"foo'ss",
"foo\xE2\x80\x99s",
"foo\xC3\xA4\xE2\x80\x99s",
"foo\xE2\x80\x99S",
"foos\xE2\x80\x99S",
"foo\xE2\x80\x99s\xE2\x80\x99s",
"foo\xE2\x80\x99ss"
};
const char *expected_output[] = {
"foo'",
"foo",
"foo\xC3\xA4",
"foo",
"foos",
"foo's",
"foo'ss",
"foo",
"foo\xC3\xA4",
"foo",
"foos",
"foo\xE2\x80\x99s",
"foo\xE2\x80\x99ss"
};
const char *error = NULL;
const char *token = NULL;
unsigned int i;
test_begin("fts filter english possessive");
test_assert(fts_filter_create(fts_filter_english_possessive, NULL, NULL, NULL, &norm, &error) == 0);
for (i = 0; i < N_ELEMENTS(input); i++) {
token = input[i];
test_assert_idx(fts_filter_filter(norm, &token, &error) == 1, i);
test_assert_idx(null_strcmp(token, expected_output[i]) == 0, i);
}
fts_filter_unref(&norm);
test_assert(norm == NULL);
test_end();
}
static void test_fts_filter_stopwords_normalizer_stemmer_sv(void)
{
int ret;
struct fts_filter *normalizer;
struct fts_filter *stemmer;
struct fts_filter *filter;
const char *error;
const char *token = NULL;
const char * const tokens[] = {
"Enär", "erkännandet", "av", "det", "inneboende", "värdet",
"hos", "alla", "medlemmar", "av", "människosläktet", "och",
"av", "deras", "lika", "och", "oförytterliga", "rättigheter",
"är", "grundvalen", "för", "frihet", "rättvisa", "och", "fred",
"i", "världen", NULL};
const char * const bases[] = {
"enar", "erkan", NULL, NULL, "inneboend", "vardet", "hos", NULL,
"medlemm", NULL, "manniskoslaktet", NULL, NULL, NULL, "lik",
NULL, "oforytter", "ratt", NULL, "grundval", NULL, "frihet",
"rattvis", NULL, "fred", NULL, "varld", NULL};
const char * const *tpp;
const char * const *bpp;
test_begin("fts filters with stopwords, default normalizer and stemming chained, Swedish");
test_assert(fts_filter_create(fts_filter_stopwords, NULL, &swedish_language, stopword_settings, &filter, &error) == 0);
test_assert(fts_filter_create(fts_filter_normalizer_icu, filter, NULL, NULL, &normalizer, &error) == 0);
test_assert(fts_filter_create(fts_filter_stemmer_snowball, normalizer, &swedish_language, NULL, &stemmer, &error) == 0);
bpp = bases;
for (tpp = tokens; *tpp != NULL; tpp++) {
token = *tpp;
ret = fts_filter_filter(stemmer, &token, &error);
if (ret <= 0) {
test_assert(ret == 0);
test_assert(*bpp == NULL);
} else {
test_assert(*bpp != NULL);
test_assert(null_strcmp(*bpp, token) == 0);
}
bpp++;
}
fts_filter_unref(&stemmer);
fts_filter_unref(&normalizer);
fts_filter_unref(&filter);
test_assert(stemmer == NULL);
test_assert(filter == NULL);
test_assert(normalizer == NULL);
test_end();
}
int acl_rights_cmp(const struct acl_rights *r1, const struct acl_rights *r2)
{
int ret;
if (r1->global != r2->global) {
/* globals have higher priority than locals */
return r1->global ? 1 : -1;
}
ret = r1->id_type - r2->id_type;
if (ret != 0)
return ret;
return null_strcmp(r1->identifier, r2->identifier);
}
static int acl_attribute_get_acl(struct mailbox *box, const char *key,
struct mail_attribute_value *value_r)
{
struct acl_object *aclobj = acl_mailbox_get_aclobj(box);
struct acl_object_list_iter *iter;
struct acl_rights rights, wanted_rights;
const char *id;
int ret;
memset(value_r, 0, sizeof(*value_r));
if (!box->storage->user->dsyncing) {
mail_storage_set_error(box->storage, MAIL_ERROR_PERM,
MAIL_ERRSTR_NO_PERMISSION);
return -1;
}
/* set last_change for all ACL objects, even if they don't exist
(because they could have been removed by the last change, and dsync
can use this information) */
(void)acl_object_last_changed(aclobj, &value_r->last_change);
memset(&wanted_rights, 0, sizeof(wanted_rights));
id = key + strlen(MAILBOX_ATTRIBUTE_PREFIX_ACL);
if (acl_identifier_parse(id, &wanted_rights) < 0) {
mail_storage_set_error(box->storage, MAIL_ERROR_PARAMS,
t_strdup_printf("Invalid ID: %s", id));
return -1;
}
iter = acl_object_list_init(aclobj);
while ((ret = acl_object_list_next(iter, &rights)) > 0) {
if (!rights.global &&
rights.id_type == wanted_rights.id_type &&
null_strcmp(rights.identifier, wanted_rights.identifier) == 0) {
value_r->value = acl_rights_export(&rights);
break;
}
}
if (ret < 0)
mail_storage_set_internal_error(box->storage);
acl_object_list_deinit(&iter);
return ret;
}
static struct mail_log_group_changes *
mail_log_action_get_group(struct mail_log_transaction_context *lt,
enum mail_log_event event, const char *data)
{
struct mail_log_group_changes *group;
unsigned int i, count;
if (!array_is_created(<->group_changes))
p_array_init(<->group_changes, lt->pool, 8);
group = array_get_modifiable(<->group_changes, &count);
for (i = 0; i < count; i++) {
if (group[i].event == event &&
null_strcmp(data, group[i].data) == 0)
return &group[i];
}
group = array_append_space(<->group_changes);
group->event = event;
group->data = p_strdup(lt->pool, data);
return group;
}
int mail_storage_service_read_settings(struct mail_storage_service_ctx *ctx,
const struct mail_storage_service_input *input,
pool_t pool,
const struct setting_parser_info **user_info_r,
const struct setting_parser_context **parser_r,
const char **error_r)
{
struct master_service_settings_input set_input;
const struct setting_parser_info *const *roots;
struct master_service_settings_output set_output;
const struct dynamic_settings_parser *dyn_parsers;
enum mail_storage_service_flags flags;
unsigned int i;
ctx->config_permission_denied = FALSE;
flags = input == NULL ? ctx->flags :
mail_storage_service_input_get_flags(ctx, input);
memset(&set_input, 0, sizeof(set_input));
set_input.roots = ctx->set_roots;
set_input.preserve_user = TRUE;
/* settings reader may exec doveconf, which is going to clear
environment, and if we're not doing a userdb lookup we want to
use $HOME */
set_input.preserve_home =
(flags & MAIL_STORAGE_SERVICE_FLAG_USERDB_LOOKUP) == 0;
set_input.use_sysexits =
(flags & MAIL_STORAGE_SERVICE_FLAG_USE_SYSEXITS) != 0;
if (input != NULL) {
set_input.module = input->module;
set_input.service = input->service;
set_input.username = input->username;
set_input.local_ip = input->local_ip;
set_input.remote_ip = input->remote_ip;
}
if (input == NULL) {
/* global settings read - don't create a cache for thi */
} else if (ctx->set_cache == NULL) {
ctx->set_cache_module = p_strdup(ctx->pool, set_input.module);
ctx->set_cache_service = p_strdup(ctx->pool, set_input.service);
ctx->set_cache = master_service_settings_cache_init(
ctx->service, set_input.module, set_input.service);
} else {
/* already looked up settings at least once.
we really shouldn't be execing anymore. */
set_input.never_exec = TRUE;
}
dyn_parsers = mail_storage_get_dynamic_parsers(pool);
if (null_strcmp(set_input.module, ctx->set_cache_module) == 0 &&
null_strcmp(set_input.service, ctx->set_cache_service) == 0 &&
ctx->set_cache != NULL) {
if (master_service_settings_cache_read(ctx->set_cache,
&set_input, dyn_parsers,
parser_r, error_r) < 0) {
*error_r = t_strdup_printf(
"Error reading configuration: %s", *error_r);
return -1;
}
} else {
settings_parser_dyn_update(pool, &set_input.roots, dyn_parsers);
if (master_service_settings_read(ctx->service, &set_input,
&set_output, error_r) < 0) {
*error_r = t_strdup_printf(
"Error reading configuration: %s", *error_r);
ctx->config_permission_denied =
set_output.permission_denied;
return -1;
}
*parser_r = ctx->service->set_parser;
}
roots = settings_parser_get_roots(*parser_r);
for (i = 0; roots[i] != NULL; i++) {
if (strcmp(roots[i]->module_name,
mail_user_setting_parser_info.module_name) == 0) {
*user_info_r = roots[i];
return 0;
}
}
i_unreached();
return -1;
}
static void test_fts_filter_stopwords_normalizer_stemmer_no(void)
{
int ret;
struct fts_filter *normalizer;
struct fts_filter *stemmer;
struct fts_filter *filter;
const char *error;
const char *token = NULL;
const char * const tokens[] = {
/* Nynorsk*/
"Alle", "har", "plikter", "andsynes", "samfunnet", "d\xC3\xA5",
"personlegdomen", "til", "den", "einskilde", "einast", "der",
"kan", "f\xC3\xA5", "frie", "og", "fullgode",
"voksterk\xC3\xA5r",
/* Bokmal */
"Alle", "mennesker", "er", "f\xC3\xB8""dt", "frie", "og", "med",
"samme", "menneskeverd", "og", "menneskerettigheter", "De",
"er", "utstyrt", "med", "fornuft", "og", "samvittighet",
"og", "b\xC3\xB8r", "handle", "mot", "hverandre", "i",
"brorskapets", "\xC3\xA5nd", NULL};
const char * const bases[] = {
/* Nynorsk*/
"all", NULL, "plikt", "andsyn", "samfunn", NULL,
"personlegdom", NULL, NULL, "einskild", "ein", NULL, NULL,
"fa", "frie", NULL, "fullgod", "voksterk",
/* Bokmal */
"all", "mennesk", NULL, "f\xC3\xB8""dt", "frie", NULL, NULL,
NULL, "menneskeverd", NULL, "menneskerett", "de", NULL,
"utstyrt", NULL, "fornuft", NULL, "samvitt", NULL, "b\xC3\xB8r",
"handl", NULL, "hverandr", NULL, "brorskap", "and", NULL};
const char * const *tpp;
const char * const *bpp;
test_begin("fts filters with stopwords, default normalizer and stemming chained, Norwegian");
test_assert(fts_filter_create(fts_filter_stopwords, NULL, &norwegian_language, stopword_settings, &filter, &error) == 0);
test_assert(fts_filter_create(fts_filter_normalizer_icu, filter, NULL, NULL, &normalizer, &error) == 0);
test_assert(fts_filter_create(fts_filter_stemmer_snowball, normalizer, &norwegian_language, NULL, &stemmer, &error) == 0);
bpp = bases;
for (tpp = tokens; *tpp != NULL; tpp++) {
token = *tpp;
ret = fts_filter_filter(stemmer, &token, &error);
if (ret <= 0) {
test_assert(ret == 0);
test_assert(*bpp == NULL);
} else {
test_assert(*bpp != NULL);
test_assert(null_strcmp(*bpp, token) == 0);
}
bpp++;
}
fts_filter_unref(&stemmer);
fts_filter_unref(&normalizer);
fts_filter_unref(&filter);
test_assert(stemmer == NULL);
test_assert(filter == NULL);
test_assert(normalizer == NULL);
test_end();
}
static void test_fts_filter_stopwords_fin(void)
{
const struct fts_language finnish = { .name = "fi" };
struct fts_filter *filter;
const char *error;
int ret;
const char *input[] = {"olla", "vaiko", "eik\xC3\xB6", "olla",
"kenest\xC3\xA4", "ja", "joista", "jonka",
"testi", NULL};
const char *output[] = {NULL, "vaiko", "eik\xC3\xB6", NULL, NULL,
NULL, NULL, NULL, "testi"};
const char *input2[] =
{"kuka", "kenet", "keneen", "testi", "eiv\xC3\xA4t", NULL};
const char *output2[] = {NULL, NULL, NULL, "testi", NULL};
const char **ip, **op;
const char *token;
test_begin("fts filter stopwords, Finnish");
test_assert(fts_filter_create(fts_filter_stopwords, NULL, &finnish, stopword_settings, &filter, &error) == 0);
ip = input;
op = output;
while (*ip != NULL) {
token = *ip;
ret = fts_filter_filter(filter, &token, &error);
if (ret <= 0) {
test_assert(ret == 0);
test_assert(*op == NULL);
} else {
test_assert(*op != NULL);
test_assert(strcmp(*ip, token) == 0);
}
op++;
ip++;
}
fts_filter_unref(&filter);
test_assert(filter == NULL);
test_assert(fts_filter_create(fts_filter_stopwords, NULL, &finnish, stopword_settings, &filter, &error) == 0);
ip = input2;
op = output2;
while (*ip != NULL) {
token = *ip;
ret = fts_filter_filter(filter, &token, &error);
if (ret <= 0) {
test_assert(ret == 0);
test_assert(*op == NULL);
} else {
test_assert(*op != NULL);
test_assert(strcmp(*ip, token) == 0);
}
op++;
ip++;
}
fts_filter_unref(&filter);
test_assert(filter == NULL);
test_end();
}
static void test_fts_filter_stopwords_fra(void)
{
struct fts_filter *filter;
const char *error;
int ret;
const char *input[] = {"e\xC3\xBBt", "soyez", "soi", "peut", "que",
"quelconque", "\xC3\xA9t\xC3\xA9",
"l\xE2\x80\x99""av\xC3\xA8nement",
NULL};
const char *output[] = {NULL, NULL, NULL, "peut", NULL,
"quelconque", NULL,
"l\xE2\x80\x99""av\xC3\xA8nement",};
const char **ip, **op;
const char *token;
test_begin("fts filter stopwords, French");
test_assert(fts_filter_create(fts_filter_stopwords, NULL, &french_language, stopword_settings, &filter, &error) == 0);
ip = input;
op = output;
while (*ip != NULL) {
token = *ip;
ret = fts_filter_filter(filter, &token, &error);
if (ret <= 0) {
test_assert(ret == 0);
test_assert(*op == NULL);
} else {
test_assert(*op != NULL);
test_assert(strcmp(*ip, token) == 0);
}
op++;
ip++;
}
fts_filter_unref(&filter);
test_assert(filter == NULL);
test_end();
}
static void test_fts_filter_stopwords_no(void)
{
struct fts_filter *filter;
const char *error;
int ret;
const char *input[] = {"og", "d\xC3\xA5", "medlemsstatane", "har",
"bunde", "seg", "til", "\xC3\xA5", "fremje",
"allmenn", "v\xC3\xB8rdnad", "for", "pakta",
"og", "halde", "seg", "etter", "menneskerettane",
"og", "den", "grunnleggjande", "fridomen", "i",
"samarbeid", "med", "Dei", "Sameinte",
"Nasjonane", NULL};
const char *output[] = {NULL, NULL, "medlemsstatane", NULL,
"bunde", NULL, NULL, NULL, "fremje",
"allmenn", "v\xC3\xB8rdnad", NULL, "pakta",
NULL, "halde", NULL, NULL, "menneskerettane",
NULL, NULL, "grunnleggjande", "fridomen", NULL,
"samarbeid", NULL, "Dei", "Sameinte",
"Nasjonane"};
const char **ip, **op;
const char *token;
test_begin("fts filter stopwords, Norwegian");
test_assert(fts_filter_create(fts_filter_stopwords, NULL, &norwegian_language, stopword_settings, &filter, &error) == 0);
ip = input;
op = output;
while (*ip != NULL) {
token = *ip;
ret = fts_filter_filter(filter, &token, &error);
if (ret <= 0) {
test_assert(ret == 0);
test_assert(*op == NULL);
} else {
test_assert(*op != NULL);
test_assert(strcmp(*ip, token) == 0);
}
op++;
ip++;
}
fts_filter_unref(&filter);
test_assert(filter == NULL);
test_end();
}
static void test_fts_filter_stopwords_fail_lazy_init(void)
{
const struct fts_language unknown = { .name = "bebobidoop" };
struct fts_filter *filter = NULL;
const char *error = NULL, *token = "foobar";
test_begin("fts filter stopwords, fail filter() (lazy init)");
test_assert(fts_filter_create(fts_filter_stopwords, NULL, &unknown, stopword_settings, &filter, &error) == 0);
test_assert(filter != NULL && error == NULL);
test_assert(fts_filter_filter(filter, &token, &error) < 0 && error != NULL);
fts_filter_unref(&filter);
test_end();
}
#ifdef HAVE_FTS_STEMMER
static void test_fts_filter_stemmer_snowball_stem_english(void)
{
struct fts_filter *stemmer;
const char *error;
const char *token = NULL;
const char * const tokens[] = {
"dries" ,"friendlies", "All", "human", "beings", "are",
"born", "free", "and", "equal", "in", "dignity", "and",
"rights", "They", "are", "endowed", "with", "reason", "and",
"conscience", "and", "should", "act", "towards", "one",
"another", "in", "a", "spirit", "of", "brotherhood", NULL};
const char * const bases[] = {
"dri" ,"friend", "All", "human", "be", "are", "born", "free",
"and", "equal", "in", "digniti", "and", "right", "They", "are",
"endow", "with", "reason", "and", "conscienc", "and", "should",
"act", "toward", "one", "anoth", "in", "a", "spirit", "of",
"brotherhood", NULL};
const char * const *tpp;
const char * const *bpp;
test_begin("fts filter stem English");
test_assert(fts_filter_create(fts_filter_stemmer_snowball, NULL, &english_language, NULL, &stemmer, &error) == 0);
bpp = bases;
for (tpp=tokens; *tpp != NULL; tpp++) {
token = *tpp;
test_assert(fts_filter_filter(stemmer, &token, &error) > 0);
test_assert(token != NULL);
test_assert(null_strcmp(token, *bpp) == 0);
bpp++;
}
fts_filter_unref(&stemmer);
test_assert(stemmer == NULL);
test_end();
}
static void test_fts_filter_stemmer_snowball_stem_french(void)
{
struct fts_filter *stemmer;
const char *error;
const char *token = NULL;
const char * const tokens[] = {
"Tous", "les", "\xC3\xAAtres", "humains", "naissent",
"libres", "et", "\xC3\xA9gaux", "en", "dignit\xC3\xA9",
"et", "en", "droits", NULL};
const char * const bases[] = {
"Tous" ,"le", "\xC3\xAAtre", "humain", "naissent", "libr", "et",
"\xC3\xA9gal", "en", "dignit", "et", "en", "droit", NULL};
const char * const *tpp;
const char * const *bpp;
test_begin("fts filter stem French");
test_assert(fts_filter_create(fts_filter_stemmer_snowball, NULL, &french_language, NULL, &stemmer, &error) == 0);
bpp = bases;
for (tpp=tokens; *tpp != NULL; tpp++) {
token = *tpp;
test_assert(fts_filter_filter(stemmer, &token, &error) > 0);
test_assert(token != NULL);
test_assert(null_strcmp(token, *bpp) == 0);
bpp++;
}
fts_filter_unref(&stemmer);
test_assert(stemmer == NULL);
test_end();
}
static void test_fts_filter_stopwords_stemmer_eng(void)
{
int ret;
struct fts_filter *stemmer;
struct fts_filter *filter;
const char *error;
const char *token = NULL;
const char * const tokens[] = {
"dries" ,"friendlies", "All", "human", "beings", "are",
"born", "free", "and", "equal", "in", "dignity", "and",
"rights", "They", "are", "endowed", "with", "reason", "and",
"conscience", "and", "should", "act", "towards", "one",
"another", "in", "a", "spirit", "of", "brotherhood", NULL};
const char * const bases[] = {
"dri" ,"friend", "All", "human", "be", NULL, "born", "free",
NULL, "equal", NULL, "digniti", NULL, "right", "They", NULL,
"endow", NULL, "reason", NULL, "conscienc", NULL, "should",
"act", "toward", "one", "anoth", NULL, NULL, "spirit", NULL,
"brotherhood", NULL};
const char * const *tpp;
const char * const *bpp;
test_begin("fts filters stopwords and stemming chained, English");
test_assert(fts_filter_create(fts_filter_stopwords, NULL, &english_language, stopword_settings, &filter, &error) == 0);
test_assert(fts_filter_create(fts_filter_stemmer_snowball, filter, &english_language, NULL, &stemmer, &error) == 0);
bpp = bases;
for (tpp=tokens; *tpp != NULL; tpp++) {
token = *tpp;
ret = fts_filter_filter(stemmer, &token, &error);
test_assert(ret >= 0);
if (ret == 0)
test_assert(*bpp == NULL);
else {
test_assert(*bpp != NULL);
test_assert(null_strcmp(*bpp, token) == 0);
}
bpp++;
}
fts_filter_unref(&stemmer);
fts_filter_unref(&filter);
test_assert(stemmer == NULL);
test_assert(filter == NULL);
test_end();
}
#endif
#ifdef HAVE_LIBICU
static void test_fts_filter_normalizer_swedish_short(void)
{
struct fts_filter *norm = NULL;
const char *input[] = {
"Vem",
"\xC3\x85",
"\xC3\x85\xC3\x84\xC3\x96",
"Vem kan segla f\xC3\xB6rutan vind?\n"
"\xC3\x85\xC3\x84\xC3\x96\xC3\xB6\xC3\xA4\xC3\xA5"
};
const char *expected_output[] = {
"vem",
"a",
"aao",
"vem kan segla forutan vind?\naaooaa"
};
const char * const settings[] =
{"id", "Any-Lower; NFKD; [: Nonspacing Mark :] Remove; NFC", NULL};
const char *error = NULL;
const char *token = NULL;
unsigned int i;
test_begin("fts filter normalizer Swedish short text");
test_assert(fts_filter_create(fts_filter_normalizer_icu, NULL, NULL, settings, &norm, &error) == 0);
for (i = 0; i < N_ELEMENTS(input); i++) {
token = input[i];
test_assert_idx(fts_filter_filter(norm, &token, &error) == 1, i);
test_assert_idx(null_strcmp(token, expected_output[i]) == 0, i);
}
fts_filter_unref(&norm);
test_assert(norm == NULL);
test_end();
}
static void test_fts_filter_normalizer_swedish_short_default_id(void)
{
struct fts_filter *norm = NULL;
const char *input[] = {
"Vem",
"\xC3\x85",
"\xC3\x85\xC3\x84\xC3\x96",
"Vem kan segla f\xC3\xB6rutan vind?\n"
"\xC3\x85\xC3\x84\xC3\x96\xC3\xB6\xC3\xA4\xC3\xA5"
};
const char *expected_output[] = {
"vem",
"a",
"aao",
"vemkanseglaforutanvind?\naaooaa"
};
const char *error = NULL;
const char *token = NULL;
unsigned int i;
test_begin("fts filter normalizer Swedish short text using default ID");
test_assert(fts_filter_create(fts_filter_normalizer_icu, NULL, NULL, NULL, &norm, &error) == 0);
for (i = 0; i < N_ELEMENTS(input); i++) {
token = input[i];
test_assert_idx(fts_filter_filter(norm, &token, &error) == 1, i);
test_assert_idx(null_strcmp(token, expected_output[i]) == 0, i);
}
fts_filter_unref(&norm);
test_assert(norm == NULL);
test_end();
}
/* UDHRDIR comes from Automake AM_CPPFLAGS */
#define UDHR_FRA_NAME "/udhr_fra.txt"
static void test_fts_filter_normalizer_french(void)
{
struct fts_filter *norm = NULL;
FILE *input;
const char * const settings[] =
{"id", "Any-Lower; NFKD; [: Nonspacing Mark :] Remove", NULL};
char buf[4096] = {0};
const char *error = NULL;
const char *tokens;
unsigned char sha512_digest[SHA512_RESULTLEN];
struct sha512_ctx ctx;
const unsigned char correct_digest[] = {
0x78, 0x1e, 0xb9, 0x04, 0xa4, 0x92, 0xca, 0x88,
0x1e, 0xef, 0x7b, 0xc8, 0x3e, 0x4a, 0xa8, 0xdb,
0x9c, 0xd4, 0x42, 0x5c, 0x64, 0x81, 0x06, 0xd5,
0x72, 0x93, 0x38, 0x0c, 0x09, 0xce, 0xbe, 0xdf,
0x65, 0xff, 0x36, 0x35, 0x05, 0x77, 0xcc, 0xc6,
0xff, 0x44, 0x2c, 0x31, 0x10, 0x00, 0xf6, 0x8d,
0x15, 0x25, 0x1e, 0x54, 0x67, 0x2a, 0x5b, 0xc1,
0xdb, 0x84, 0xc5, 0x0d, 0x43, 0x7e, 0x8c, 0x70};
const char *udhr_path;
test_begin("fts filter normalizer French UDHR");
udhr_path = t_strconcat(UDHRDIR, UDHR_FRA_NAME, NULL);
test_assert(fts_filter_create(fts_filter_normalizer_icu, NULL, NULL, settings, &norm, &error) == 0);
input = fopen(udhr_path, "r");
test_assert(input != NULL);
sha512_init(&ctx);
while (NULL != fgets(buf, sizeof(buf), input)) {
tokens = buf;
if (fts_filter_filter(norm, &tokens, &error) != 1){
break;
}
sha512_loop(&ctx, tokens, strlen(tokens));
}
fclose(input);
sha512_result(&ctx, sha512_digest);
test_assert(memcmp(sha512_digest, correct_digest,
sizeof(sha512_digest)) == 0);
fts_filter_unref(&norm);
test_assert(norm == NULL);
test_end();
}
static void test_fts_filter_normalizer_empty(void)
{
/* test just a couple of these */
static const char *empty_tokens[] = {
"\xC2\xAF", /* U+00AF */
"\xCC\x80", /* U+0300 */
"\xF3\xA0\x87\xAF", /* U+E01EF */
"\xCC\x80\xF3\xA0\x87\xAF" /* U+0300 U+E01EF */
};
const char * const settings[] =
{"id", "Any-Lower; NFKD; [: Nonspacing Mark :] Remove; [\\x20] Remove", NULL};
struct fts_filter *norm;
const char *error;
unsigned int i;
test_begin("fts filter normalizer empty tokens");
test_assert(fts_filter_create(fts_filter_normalizer_icu, NULL, NULL, settings, &norm, &error) == 0);
for (i = 0; i < N_ELEMENTS(empty_tokens); i++) {
const char *token = empty_tokens[i];
test_assert_idx(fts_filter_filter(norm, &token, &error) == 0, i);
}
fts_filter_unref(&norm);
test_end();
}
static void test_fts_filter_normalizer_baddata(void)
{
const char * const settings[] =
{"id", "Any-Lower; NFKD; [: Nonspacing Mark :] Remove", NULL};
struct fts_filter *norm;
const char *token, *error;
string_t *str;
unsigned int i;
test_begin("fts filter normalizer bad data");
test_assert(fts_filter_create(fts_filter_normalizer_icu, NULL, NULL, settings, &norm, &error) == 0);
str = t_str_new(128);
for (i = 1; i < 0x1ffff; i++) {
str_truncate(str, 0);
uni_ucs4_to_utf8_c(i, str);
token = str_c(str);
T_BEGIN {
test_assert_idx(fts_filter_filter(norm, &token, &error) >= 0, i);
} T_END;
}
str_truncate(str, 0);
uni_ucs4_to_utf8_c(0x7fffffff, str);
token = str_c(str);
test_assert(fts_filter_filter(norm, &token, &error) >= 0);
fts_filter_unref(&norm);
test_end();
}
static void test_fts_filter_normalizer_invalid_id(void)
{
struct fts_filter *norm = NULL;
const char *settings[] =
{"id", "Any-One-Out-There; DKFN; [: Nonspacing Mark :] Remove",
NULL};
const char *error = NULL, *token = "foo";
test_begin("fts filter normalizer invalid id");
test_assert(fts_filter_create(fts_filter_normalizer_icu, NULL, NULL, settings, &norm, &error) == 0);
test_assert(error == NULL);
test_assert(fts_filter_filter(norm, &token, &error) < 0 && error != NULL);
fts_filter_unref(&norm);
test_end();
}
#ifdef HAVE_FTS_STEMMER
static void test_fts_filter_normalizer_stopwords_stemmer_eng(void)
{
int ret;
struct fts_filter *normalizer;
struct fts_filter *stemmer;
struct fts_filter *filter;
const char *error;
const char * const id_settings[] =
//{"id", "Any-Lower; NFKD; [: Nonspacing Mark :] Remove; NFC", NULL};
{"id", "Lower", NULL};
const char *token = NULL;
const char * const tokens[] = {
"dries" ,"friendlies", "All", "human", "beings", "are",
"born", "free", "and", "equal", "in", "dignity", "and",
"rights", "They", "are", "endowed", "with", "reason", "and",
"conscience", "and", "should", "act", "towards", "one",
"another", "in", "a", "spirit", "of", "brotherhood", "ABCFoo",
NULL};
const char * const bases[] = {
"dri" ,"friend", "all", "human", "be", NULL, "born", "free",
NULL, "equal", NULL, "digniti", NULL, "right", NULL, NULL,
"endow", NULL, "reason", NULL, "conscienc", NULL, "should",
"act", "toward", "one", "anoth", NULL, NULL, "spirit", NULL,
"brotherhood", "abcfoo", NULL};
const char * const *tpp;
const char * const *bpp;
test_begin("fts filters normalizer, stopwords and stemming chained, English");
test_assert(fts_filter_create(fts_filter_normalizer_icu, NULL, NULL, id_settings, &normalizer, &error) == 0);
test_assert(fts_filter_create(fts_filter_stopwords, normalizer, &english_language, stopword_settings, &filter, &error) == 0);
test_assert(fts_filter_create(fts_filter_stemmer_snowball, filter, &english_language, NULL, &stemmer, &error) == 0);
bpp = bases;
for (tpp = tokens; *tpp != NULL; tpp++) {
token = *tpp;
ret = fts_filter_filter(stemmer, &token, &error);
if (ret <= 0) {
test_assert(ret == 0);
test_assert(*bpp == NULL);
} else {
test_assert(*bpp != NULL);
test_assert(strcmp(*bpp, token) == 0);
}
bpp++;
}
fts_filter_unref(&stemmer);
fts_filter_unref(&filter);
fts_filter_unref(&normalizer);
test_assert(stemmer == NULL);
test_assert(filter == NULL);
test_assert(normalizer == NULL);
test_end();
}
bool config_export_type(string_t *str, const void *value,
const void *default_value,
enum setting_type type, bool dump_default,
bool *dump_r)
{
switch (type) {
case SET_BOOL: {
const bool *val = value, *dval = default_value;
if (dump_default || dval == NULL || *val != *dval)
str_append(str, *val ? "yes" : "no");
break;
}
case SET_SIZE: {
const uoff_t *val = value, *dval = default_value;
if (dump_default || dval == NULL || *val != *dval)
config_export_size(str, *val);
break;
}
case SET_UINT:
case SET_UINT_OCT:
case SET_TIME: {
const unsigned int *val = value, *dval = default_value;
if (dump_default || dval == NULL || *val != *dval) {
switch (type) {
case SET_UINT_OCT:
str_printfa(str, "0%o", *val);
break;
case SET_TIME:
config_export_time(str, *val);
break;
default:
str_printfa(str, "%u", *val);
break;
}
}
break;
}
case SET_STR_VARS: {
const char *const *val = value, *sval;
const char *const *_dval = default_value;
const char *dval = _dval == NULL ? NULL : *_dval;
i_assert(*val == NULL ||
**val == SETTING_STRVAR_UNEXPANDED[0]);
sval = *val == NULL ? NULL : (*val + 1);
if ((dump_default || null_strcmp(sval, dval) != 0) &&
sval != NULL) {
str_append(str, sval);
*dump_r = TRUE;
}
break;
}
case SET_STR: {
const char *const *val = value;
const char *const *_dval = default_value;
const char *dval = _dval == NULL ? NULL : *_dval;
if ((dump_default || null_strcmp(*val, dval) != 0) &&
*val != NULL) {
str_append(str, *val);
*dump_r = TRUE;
}
break;
}
case SET_ENUM: {
const char *const *val = value;
unsigned int len = strlen(*val);
if (dump_default)
str_append(str, *val);
else {
const char *const *_dval = default_value;
const char *dval = _dval == NULL ? NULL : *_dval;
i_assert(dval != NULL);
if (strncmp(*val, dval, len) != 0 ||
((*val)[len] != ':' && (*val)[len] != '\0'))
str_append(str, *val);
}
break;
}
default:
return FALSE;
}
return TRUE;
}
