static int
fts_mail_precache_range(struct mailbox_transaction_context *trans,
struct fts_backend_update_context *update_ctx,
uint32_t seq1, uint32_t seq2)
{
struct mail_search_args *search_args;
struct mail_search_context *ctx;
struct mail *mail;
int ret = 0;
search_args = mail_search_build_init();
mail_search_build_add_seqset(search_args, seq1, seq2);
ctx = mailbox_search_init(trans, search_args, NULL,
MAIL_FETCH_STREAM_HEADER |
MAIL_FETCH_STREAM_BODY, NULL);
mail_search_args_unref(&search_args);
while (mailbox_search_next(ctx, &mail)) {
if (fts_build_mail(update_ctx, mail) < 0) {
mail_storage_set_internal_error(trans->box->storage);
ret = -1;
break;
}
mail_precache(mail);
}
if (mailbox_search_deinit(&ctx) < 0)
ret = -1;
return ret;
}
static int imap_map_read_hdr_hashes(struct mailbox *box)
{
struct pop3_migration_mailbox *mbox = POP3_MIGRATION_CONTEXT(box);
struct mailbox_transaction_context *t;
struct mail_search_args *search_args;
struct mail_search_context *ctx;
struct mail *mail;
struct imap_msg_map *map;
int ret = 0;
t = mailbox_transaction_begin(box, 0);
search_args = mail_search_build_init();
mail_search_build_add_seqset(search_args, mbox->first_unfound_idx+1,
array_count(&mbox->imap_msg_map)+1);
ctx = mailbox_search_init(t, search_args, NULL,
MAIL_FETCH_STREAM_HEADER, NULL);
mail_search_args_unref(&search_args);
while (mailbox_search_next(ctx, &mail)) {
map = array_idx_modifiable(&mbox->imap_msg_map, mail->seq-1);
if (get_hdr_sha1(mail, map->hdr_sha1) < 0)
ret = -1;
else
map->hdr_sha1_set = TRUE;
}
if (mailbox_search_deinit(&ctx) < 0)
ret = -1;
(void)mailbox_transaction_commit(&t);
return ret;
}
static int cmd_deduplicate_uidlist(struct mailbox *box, struct uidlist *uidlist)
{
struct mailbox_transaction_context *trans;
struct mail_search_context *search_ctx;
struct mail_search_args *search_args;
struct mail_search_arg *arg;
struct mail *mail;
ARRAY_TYPE(seq_range) uids;
int ret = 0;
/* the uidlist is reversed with oldest mails at the end.
we'll delete everything but the oldest mail. */
if (uidlist->next == NULL)
return 0;
t_array_init(&uids, 8);
for (; uidlist->next != NULL; uidlist = uidlist->next)
seq_range_array_add(&uids, uidlist->uid);
search_args = mail_search_build_init();
arg = mail_search_build_add(search_args, SEARCH_UIDSET);
arg->value.seqset = uids;
trans = mailbox_transaction_begin(box, 0);
search_ctx = mailbox_search_init(trans, search_args, NULL, 0, NULL);
mail_search_args_unref(&search_args);
while (mailbox_search_next(search_ctx, &mail))
mail_expunge(mail);
if (mailbox_search_deinit(&search_ctx) < 0)
ret = -1;
if (mailbox_transaction_commit(&trans) < 0)
ret = -1;
return ret;
}
static int
index_mailbox_precache(struct master_connection *conn, struct mailbox *box)
{
struct mail_storage *storage = mailbox_get_storage(box);
const char *username = mail_storage_get_user(storage)->username;
const char *box_vname = mailbox_get_vname(box);
struct mailbox_status status;
struct mailbox_transaction_context *trans;
struct mail_search_args *search_args;
struct mail_search_context *ctx;
struct mail *mail;
struct mailbox_metadata metadata;
uint32_t seq;
char percentage_str[2+1+1];
unsigned int counter = 0, max, percentage, percentage_sent = 0;
int ret = 0;
if (mailbox_get_metadata(box, MAILBOX_METADATA_PRECACHE_FIELDS,
&metadata) < 0 ||
mailbox_get_status(box, STATUS_MESSAGES | STATUS_LAST_CACHED_SEQ,
&status) < 0)
return -1;
seq = status.last_cached_seq + 1;
trans = mailbox_transaction_begin(box, MAILBOX_TRANSACTION_FLAG_NO_CACHE_DEC);
search_args = mail_search_build_init();
mail_search_build_add_seqset(search_args, seq, status.messages);
ctx = mailbox_search_init(trans, search_args, NULL,
metadata.precache_fields, NULL);
mail_search_args_unref(&search_args);
max = status.messages - seq + 1;
while (mailbox_search_next(ctx, &mail)) {
mail_precache(mail);
if (++counter % 100 == 0) {
percentage = counter*100 / max;
if (percentage != percentage_sent && percentage < 100) {
percentage_sent = percentage;
if (i_snprintf(percentage_str,
sizeof(percentage_str), "%u\n",
percentage) < 0)
i_unreached();
(void)write_full(conn->fd, percentage_str,
strlen(percentage_str));
}
indexer_worker_refresh_proctitle(username, box_vname,
counter, max);
}
}
if (mailbox_search_deinit(&ctx) < 0)
ret = -1;
if (mailbox_transaction_commit(&trans) < 0)
ret = -1;
if (ret == 0) {
i_info("Indexed %u messages in %s",
counter, mailbox_get_vname(box));
}
return ret;
}
static int
quota_count_mailbox(struct quota_root *root, struct mail_namespace *ns,
const char *vname, uint64_t *bytes_r, uint64_t *count_r)
{
struct quota_rule *rule;
struct mailbox *box;
struct mailbox_transaction_context *trans;
struct mail_search_context *ctx;
struct mail *mail;
struct mail_search_args *search_args;
enum mail_error error;
uoff_t size;
int ret = 0;
rule = quota_root_rule_find(root->set, vname);
if (rule != NULL && rule->ignore) {
/* mailbox not included in quota */
return 0;
}
box = mailbox_alloc(ns->list, vname, MAILBOX_FLAG_READONLY);
if (mailbox_sync(box, MAILBOX_SYNC_FLAG_FULL_READ) < 0) {
error = mailbox_get_last_mail_error(box);
mailbox_free(&box);
if (error == MAIL_ERROR_TEMP)
return -1;
/* non-temporary error, e.g. ACLs denied access. */
return 0;
}
trans = mailbox_transaction_begin(box, 0);
search_args = mail_search_build_init();
mail_search_build_add_all(search_args);
ctx = mailbox_search_init(trans, search_args, NULL,
MAIL_FETCH_PHYSICAL_SIZE, NULL);
mail_search_args_unref(&search_args);
while (mailbox_search_next(ctx, &mail)) {
if (mail_get_physical_size(mail, &size) == 0)
*bytes_r += size;
*count_r += 1;
}
if (mailbox_search_deinit(&ctx) < 0)
ret = -1;
if (ret < 0)
mailbox_transaction_rollback(&trans);
else
(void)mailbox_transaction_commit(&trans);
mailbox_free(&box);
return ret;
}
static void test_mail_search_args_simplify_empty_lists(void)
{
struct mail_search_args *args;
test_begin("mail search args simplify empty args");
args = mail_search_build_init();
mail_search_args_simplify(args);
mail_search_args_unref(&args);
test_end();
}
int mail_search_build(struct mail_search_register *reg,
struct mail_search_parser *parser, const char **charset,
struct mail_search_args **args_r, const char **error_r)
{
struct mail_search_build_context ctx;
struct mail_search_args *args;
struct mail_search_arg *root;
const char *str;
int ret;
*args_r = NULL;
*error_r = NULL;
args = mail_search_build_init();
memset(&ctx, 0, sizeof(ctx));
ctx.pool = args->pool;
ctx.reg = reg;
ctx.parser = parser;
ctx.charset = p_strdup(ctx.pool, *charset);
ret = mail_search_build_list(&ctx, &root);
if (!ctx.charset_checked && ret == 0) {
/* make sure we give an error message if charset is invalid */
ret = mail_search_build_get_utf8(&ctx, "", &str);
}
if (ret < 0) {
*error_r = ctx._error != NULL ? t_strdup(ctx._error) :
t_strdup(mail_search_parser_get_error(parser));
if (ctx.unknown_charset)
*charset = NULL;
pool_unref(&args->pool);
return -1;
}
if (root->type == SEARCH_SUB && !root->match_not) {
/* simple SUB root */
args->args = root->value.subargs;
} else {
args->args = root;
}
*args_r = args;
return 0;
}
static int pop3_map_read_hdr_hashes(struct mail_storage *storage,
unsigned first_seq)
{
struct pop3_migration_mail_storage *mstorage =
POP3_MIGRATION_CONTEXT(storage);
struct mailbox_transaction_context *t;
struct mail_search_args *search_args;
struct mail_search_context *ctx;
struct mail *mail;
struct pop3_uidl_map *map;
int ret = 0;
if (mstorage->pop3_all_hdr_sha1_set)
return 0;
if (mstorage->all_mailboxes) {
/* we may be matching against multiple mailboxes.
read all the hashes only once. */
first_seq = 1;
}
t = mailbox_transaction_begin(mstorage->pop3_box, 0);
search_args = mail_search_build_init();
mail_search_build_add_seqset(search_args, first_seq,
array_count(&mstorage->pop3_uidl_map)+1);
ctx = mailbox_search_init(t, search_args, NULL,
MAIL_FETCH_STREAM_HEADER, NULL);
mail_search_args_unref(&search_args);
while (mailbox_search_next(ctx, &mail)) {
map = array_idx_modifiable(&mstorage->pop3_uidl_map,
mail->seq-1);
if (get_hdr_sha1(mail, map->hdr_sha1) < 0)
ret = -1;
else
map->hdr_sha1_set = TRUE;
}
if (mailbox_search_deinit(&ctx) < 0)
ret = -1;
(void)mailbox_transaction_commit(&t);
if (ret == 0 && first_seq == 1)
mstorage->pop3_all_hdr_sha1_set = TRUE;
return ret;
}
static int imap_map_read(struct mailbox *box)
{
struct pop3_migration_mailbox *mbox = POP3_MIGRATION_CONTEXT(box);
struct mailbox_status status;
struct mailbox_transaction_context *t;
struct mail_search_args *search_args;
struct mail_search_context *ctx;
struct mail *mail;
struct imap_msg_map *map;
uoff_t psize;
int ret = 0;
mailbox_get_open_status(box, STATUS_MESSAGES, &status);
i_assert(!array_is_created(&mbox->imap_msg_map));
p_array_init(&mbox->imap_msg_map, box->pool, status.messages);
t = mailbox_transaction_begin(box, 0);
search_args = mail_search_build_init();
mail_search_build_add_all(search_args);
ctx = mailbox_search_init(t, search_args, NULL,
MAIL_FETCH_PHYSICAL_SIZE, NULL);
mail_search_args_unref(&search_args);
while (mailbox_search_next(ctx, &mail)) {
if (mail_get_physical_size(mail, &psize) < 0) {
i_error("pop3_migration: Failed to get psize for imap uid %u: %s",
mail->uid,
mailbox_get_last_error(box, NULL));
ret = -1;
break;
}
map = array_append_space(&mbox->imap_msg_map);
map->uid = mail->uid;
map->psize = psize;
}
if (mailbox_search_deinit(&ctx) < 0)
ret = -1;
(void)mailbox_transaction_commit(&t);
return ret;
}
int imap_expunge(struct mailbox *box, struct mail_search_arg *next_search_arg,
unsigned int *expunged_count)
{
struct mail_search_context *ctx;
struct mailbox_transaction_context *t;
struct mail *mail;
struct mail_search_args *search_args;
bool expunges = FALSE;
if (mailbox_is_readonly(box)) {
/* silently ignore */
return 0;
}
search_args = mail_search_build_init();
search_args->args = p_new(search_args->pool, struct mail_search_arg, 1);
search_args->args->type = SEARCH_FLAGS;
search_args->args->value.flags = MAIL_DELETED;
search_args->args->next = next_search_arg;
/* Refresh the flags so we'll expunge all messages marked as \Deleted
by any session. */
t = mailbox_transaction_begin(box, MAILBOX_TRANSACTION_FLAG_REFRESH);
ctx = mailbox_search_init(t, search_args, NULL, 0, NULL);
mail_search_args_unref(&search_args);
while (mailbox_search_next(ctx, &mail)) {
*expunged_count += 1;
mail_expunge(mail);
expunges = TRUE;
}
if (mailbox_search_deinit(&ctx) < 0) {
mailbox_transaction_rollback(&t);
return -1;
} else {
if (mailbox_transaction_commit(&t) < 0)
return -1;
}
return expunges ? 1 : 0;
}
static struct mail_search_args *
virtual_search_args_parse(const string_t *rule, const char **error_r)
{
struct istream *input;
struct imap_parser *imap_parser;
const struct imap_arg *args;
struct mail_search_parser *parser;
struct mail_search_args *sargs;
const char *charset = "UTF-8";
bool fatal;
int ret;
if (str_len(rule) == 0) {
sargs = mail_search_build_init();
mail_search_build_add_all(sargs);
return sargs;
}
input = i_stream_create_from_data(str_data(rule), str_len(rule));
(void)i_stream_read(input);
imap_parser = imap_parser_create(input, NULL, (size_t)-1);
ret = imap_parser_finish_line(imap_parser, 0, 0, &args);
if (ret < 0) {
sargs = NULL;
*error_r = t_strdup(imap_parser_get_error(imap_parser, &fatal));
} else {
parser = mail_search_parser_init_imap(args);
if (mail_search_build(mail_search_register_get_imap(),
parser, &charset, &sargs, error_r) < 0)
sargs = NULL;
mail_search_parser_deinit(&parser);
}
imap_parser_unref(&imap_parser);
i_stream_destroy(&input);
return sargs;
}
int mail_search_build(struct mail_search_register *reg,
struct mail_search_parser *parser, const char *charset,
struct mail_search_args **args_r, const char **error_r)
{
struct mail_search_build_context ctx;
struct mail_search_args *args;
struct mail_search_arg *root;
*args_r = NULL;
*error_r = NULL;
args = mail_search_build_init();
args->charset = p_strdup(args->pool, charset);
memset(&ctx, 0, sizeof(ctx));
ctx.pool = args->pool;
ctx.reg = reg;
ctx.parser = parser;
if (mail_search_build_list(&ctx, &root) < 0) {
*error_r = ctx._error != NULL ? t_strdup(ctx._error) :
t_strdup(mail_search_parser_get_error(parser));
pool_unref(&args->pool);
return -1;
}
if (root->type == SEARCH_SUB && !root->not) {
/* simple SUB root */
args->args = root->value.subargs;
} else {
args->args = root;
}
*args_r = args;
return 0;
}
static int
virtual_size_add_new(struct mailbox *box,
struct index_vsize_header *vsize_hdr)
{
struct index_mailbox_context *ibox = INDEX_STORAGE_CONTEXT(box);
const struct mail_index_header *hdr;
struct mailbox_transaction_context *trans;
struct mail_search_context *search_ctx;
struct mail_search_args *search_args;
struct mail *mail;
uint32_t seq1, seq2;
uoff_t vsize;
int ret = 0;
hdr = mail_index_get_header(box->view);
if (vsize_hdr->highest_uid == 0)
seq2 = 0;
else if (!mail_index_lookup_seq_range(box->view, 1,
vsize_hdr->highest_uid,
&seq1, &seq2))
seq2 = 0;
if (vsize_hdr->message_count != seq2) {
if (vsize_hdr->message_count < seq2) {
mail_storage_set_critical(box->storage,
"vsize-hdr has invalid message-count (%u < %u)",
vsize_hdr->message_count, seq2);
} else {
/* some messages have been expunged, rescan */
}
memset(vsize_hdr, 0, sizeof(*vsize_hdr));
seq2 = 0;
}
search_args = mail_search_build_init();
mail_search_build_add_seqset(search_args, seq2 + 1,
hdr->messages_count);
trans = mailbox_transaction_begin(box, 0);
search_ctx = mailbox_search_init(trans, search_args, NULL,
MAIL_FETCH_VIRTUAL_SIZE, NULL);
while (mailbox_search_next(search_ctx, &mail)) {
if (mail_get_virtual_size(mail, &vsize) < 0) {
if (mail->expunged)
continue;
ret = -1;
break;
}
vsize_hdr->vsize += vsize;
vsize_hdr->highest_uid = mail->uid;
vsize_hdr->message_count++;
}
if (mailbox_search_deinit(&search_ctx) < 0)
ret = -1;
mail_search_args_unref(&search_args);
if (ret == 0) {
/* success, cache all */
vsize_hdr->highest_uid = hdr->next_uid - 1;
} else {
/* search failed, cache only up to highest seen uid */
}
mail_index_update_header_ext(trans->itrans, ibox->vsize_hdr_ext_id,
0, vsize_hdr, sizeof(*vsize_hdr));
(void)mailbox_transaction_commit(&trans);
return ret;
}
static int
index_mailbox_precache(struct master_connection *conn, struct mailbox *box)
{
struct mail_storage *storage = mailbox_get_storage(box);
const char *username = mail_storage_get_user(storage)->username;
const char *box_vname = mailbox_get_vname(box);
struct mailbox_status status;
struct mailbox_transaction_context *trans;
struct mail_search_args *search_args;
struct mail_search_context *ctx;
struct mail *mail;
struct mailbox_metadata metadata;
uint32_t seq, first_uid = 0, last_uid = 0;
char percentage_str[2+1+1];
unsigned int counter = 0, max, percentage, percentage_sent = 0;
int ret = 0;
if (mailbox_get_metadata(box, MAILBOX_METADATA_PRECACHE_FIELDS,
&metadata) < 0) {
i_error("Mailbox %s: Precache-fields lookup failed: %s",
mailbox_get_vname(box),
mailbox_get_last_internal_error(box, NULL));
return -1;
}
if (mailbox_get_status(box, STATUS_MESSAGES | STATUS_LAST_CACHED_SEQ,
&status) < 0) {
i_error("Mailbox %s: Status lookup failed: %s",
mailbox_get_vname(box),
mailbox_get_last_internal_error(box, NULL));
return -1;
}
seq = status.last_cached_seq + 1;
trans = mailbox_transaction_begin(box, MAILBOX_TRANSACTION_FLAG_NO_CACHE_DEC,
"indexing");
search_args = mail_search_build_init();
mail_search_build_add_seqset(search_args, seq, status.messages);
ctx = mailbox_search_init(trans, search_args, NULL,
metadata.precache_fields, NULL);
mail_search_args_unref(&search_args);
max = status.messages + 1 - seq;
while (mailbox_search_next(ctx, &mail)) {
if (first_uid == 0)
first_uid = mail->uid;
last_uid = mail->uid;
mail_precache(mail);
if (++counter % 100 == 0) {
percentage = counter*100 / max;
if (percentage != percentage_sent && percentage < 100) {
percentage_sent = percentage;
if (i_snprintf(percentage_str,
sizeof(percentage_str), "%u\n",
percentage) < 0)
i_unreached();
(void)write_full(conn->fd, percentage_str,
strlen(percentage_str));
}
indexer_worker_refresh_proctitle(username, box_vname,
counter, max);
}
}
if (mailbox_search_deinit(&ctx) < 0) {
i_error("Mailbox %s: Mail search failed: %s",
mailbox_get_vname(box),
mailbox_get_last_internal_error(box, NULL));
ret = -1;
}
const char *uids = first_uid == 0 ? "" :
t_strdup_printf(" (UIDs %u..%u)", first_uid, last_uid);
if (mailbox_transaction_commit(&trans) < 0) {
i_error("Mailbox %s: Transaction commit failed: %s"
" (attempted to index %u messages%s)",
mailbox_get_vname(box),
mailbox_get_last_internal_error(box, NULL),
counter, uids);
ret = -1;
} else {
i_info("Indexed %u messages in %s%s",
counter, mailbox_get_vname(box), uids);
}
return ret;
}
static int snarf(struct mailbox *srcbox, struct mailbox *destbox)
{
struct mail_search_args *search_args;
struct mail_search_context *search_ctx;
struct mailbox_transaction_context *src_trans, *dest_trans;
struct mail_save_context *save_ctx;
struct mail *mail;
enum mail_error error;
int ret;
/* make sure the destination mailbox has been opened.
note that this locks the mailbox. */
if (mailbox_open(destbox) < 0)
return -1;
if (mailbox_sync(srcbox, MAILBOX_SYNC_FLAG_FULL_READ) < 0)
return -1;
src_trans = mailbox_transaction_begin(srcbox, 0);
dest_trans = mailbox_transaction_begin(destbox,
MAILBOX_TRANSACTION_FLAG_EXTERNAL);
search_args = mail_search_build_init();
mail_search_build_add_all(search_args);
search_ctx = mailbox_search_init(src_trans, search_args, NULL,
MAIL_FETCH_STREAM_HEADER |
MAIL_FETCH_STREAM_BODY, NULL);
mail_search_args_unref(&search_args);
ret = 0;
while (mailbox_search_next(search_ctx, &mail)) {
if (mail->expunged)
continue;
save_ctx = mailbox_save_alloc(dest_trans);
if (mailbox_copy(&save_ctx, mail) < 0 && !mail->expunged) {
error = mailbox_get_last_mail_error(destbox);
/* if we failed because of out of disk space, just
move those messages we managed to move so far. */
if (error != MAIL_ERROR_NOQUOTA)
ret = -1;
break;
}
mail_expunge(mail);
}
if (mailbox_search_deinit(&search_ctx) < 0)
ret = -1;
/* commit the copied messages to the destination mailbox. if we crash
between that and between expunging the messages from the source
mailbox, we're left with duplicates. */
if (ret < 0)
mailbox_transaction_rollback(&dest_trans);
else if (mailbox_transaction_commit(&dest_trans) < 0)
ret = -1;
if (ret < 0)
mailbox_transaction_rollback(&src_trans);
else {
if (mailbox_transaction_commit(&src_trans) < 0)
ret = -1;
}
if (ret == 0) {
if (mailbox_sync(srcbox, 0) < 0)
ret = -1;
}
return ret;
}
static int pop3_map_read(struct mail_storage *storage)
{
struct pop3_migration_mail_storage *mstorage =
POP3_MIGRATION_CONTEXT(storage);
struct mailbox *pop3_box = mstorage->pop3_box;
struct mailbox_transaction_context *t;
struct mail_search_args *search_args;
struct mail_search_context *ctx;
struct mail *mail;
struct pop3_uidl_map *map;
const char *uidl;
uoff_t size;
int ret = 0;
if (array_is_created(&mstorage->pop3_uidl_map)) {
/* already read these, just reset the imap_uids */
array_foreach_modifiable(&mstorage->pop3_uidl_map, map)
map->imap_uid = 0;
return 0;
}
i_array_init(&mstorage->pop3_uidl_map, 128);
if (mailbox_sync(pop3_box, 0) < 0) {
i_error("pop3_migration: Couldn't sync mailbox %s: %s",
pop3_box->vname, mailbox_get_last_error(pop3_box, NULL));
return -1;
}
t = mailbox_transaction_begin(pop3_box, 0);
search_args = mail_search_build_init();
mail_search_build_add_all(search_args);
ctx = mailbox_search_init(t, search_args, NULL,
MAIL_FETCH_VIRTUAL_SIZE, NULL);
mail_search_args_unref(&search_args);
while (mailbox_search_next(ctx, &mail)) {
if (mail_get_virtual_size(mail, &size) < 0) {
i_error("pop3_migration: Failed to get size for msg %u: %s",
mail->seq,
mailbox_get_last_error(pop3_box, NULL));
ret = -1;
break;
}
if (mail_get_special(mail, MAIL_FETCH_UIDL_BACKEND, &uidl) < 0) {
i_error("pop3_migration: Failed to get UIDL for msg %u: %s",
mail->seq,
mailbox_get_last_error(pop3_box, NULL));
ret = -1;
break;
}
if (*uidl == '\0') {
i_warning("pop3_migration: UIDL for msg %u is empty",
mail->seq);
continue;
}
map = array_append_space(&mstorage->pop3_uidl_map);
map->pop3_seq = mail->seq;
map->pop3_uidl = p_strdup(storage->pool, uidl);
map->size = size;
}
if (mailbox_search_deinit(&ctx) < 0)
ret = -1;
(void)mailbox_transaction_commit(&t);
return ret;
}
