static int cmd_deduplicate_uidlist(struct mailbox *box, struct uidlist *uidlist)
{
struct mailbox_transaction_context *trans;
struct mail_search_context *search_ctx;
struct mail_search_args *search_args;
struct mail_search_arg *arg;
struct mail *mail;
ARRAY_TYPE(seq_range) uids;
int ret = 0;
/* the uidlist is reversed with oldest mails at the end.
we'll delete everything but the oldest mail. */
if (uidlist->next == NULL)
return 0;
t_array_init(&uids, 8);
for (; uidlist->next != NULL; uidlist = uidlist->next)
seq_range_array_add(&uids, uidlist->uid);
search_args = mail_search_build_init();
arg = mail_search_build_add(search_args, SEARCH_UIDSET);
arg->value.seqset = uids;
trans = mailbox_transaction_begin(box, 0);
search_ctx = mailbox_search_init(trans, search_args, NULL, 0, NULL);
mail_search_args_unref(&search_args);
while (mailbox_search_next(search_ctx, &mail))
mail_expunge(mail);
if (mailbox_search_deinit(&search_ctx) < 0)
ret = -1;
if (mailbox_transaction_commit(&trans) < 0)
ret = -1;
return ret;
}
static int
cmd_import_box_contents(struct doveadm_mail_iter *iter, struct mail *src_mail,
struct mailbox *dest_box)
{
struct mail_save_context *save_ctx;
struct mailbox_transaction_context *dest_trans;
const char *mailbox = mailbox_get_vname(dest_box);
int ret = 0;
dest_trans = mailbox_transaction_begin(dest_box,
MAILBOX_TRANSACTION_FLAG_EXTERNAL);
do {
if (doveadm_debug) {
i_debug("import: box=%s uid=%u",
mailbox, src_mail->uid);
}
save_ctx = mailbox_save_alloc(dest_trans);
mailbox_save_copy_flags(save_ctx, src_mail);
if (mailbox_copy(&save_ctx, src_mail) < 0) {
i_error("Copying box=%s uid=%u failed: %s",
mailbox, src_mail->uid,
mailbox_get_last_error(dest_box, NULL));
ret = -1;
}
} while (doveadm_mail_iter_next(iter, &src_mail));
if (mailbox_transaction_commit(&dest_trans) < 0) {
i_error("Committing copied mails to %s failed: %s", mailbox,
mailbox_get_last_error(dest_box, NULL));
ret = -1;
}
return ret;
}
static int imap_map_read_hdr_hashes(struct mailbox *box)
{
struct pop3_migration_mailbox *mbox = POP3_MIGRATION_CONTEXT(box);
struct mailbox_transaction_context *t;
struct mail_search_args *search_args;
struct mail_search_context *ctx;
struct mail *mail;
struct imap_msg_map *map;
int ret = 0;
t = mailbox_transaction_begin(box, 0);
search_args = mail_search_build_init();
mail_search_build_add_seqset(search_args, mbox->first_unfound_idx+1,
array_count(&mbox->imap_msg_map)+1);
ctx = mailbox_search_init(t, search_args, NULL,
MAIL_FETCH_STREAM_HEADER, NULL);
mail_search_args_unref(&search_args);
while (mailbox_search_next(ctx, &mail)) {
map = array_idx_modifiable(&mbox->imap_msg_map, mail->seq-1);
if (get_hdr_sha1(mail, map->hdr_sha1) < 0)
ret = -1;
else
map->hdr_sha1_set = TRUE;
}
if (mailbox_search_deinit(&ctx) < 0)
ret = -1;
(void)mailbox_transaction_commit(&t);
return ret;
}
static int
index_mailbox_precache(struct master_connection *conn, struct mailbox *box)
{
struct mail_storage *storage = mailbox_get_storage(box);
const char *username = mail_storage_get_user(storage)->username;
const char *box_vname = mailbox_get_vname(box);
struct mailbox_status status;
struct mailbox_transaction_context *trans;
struct mail_search_args *search_args;
struct mail_search_context *ctx;
struct mail *mail;
struct mailbox_metadata metadata;
uint32_t seq;
char percentage_str[2+1+1];
unsigned int counter = 0, max, percentage, percentage_sent = 0;
int ret = 0;
if (mailbox_get_metadata(box, MAILBOX_METADATA_PRECACHE_FIELDS,
&metadata) < 0 ||
mailbox_get_status(box, STATUS_MESSAGES | STATUS_LAST_CACHED_SEQ,
&status) < 0)
return -1;
seq = status.last_cached_seq + 1;
trans = mailbox_transaction_begin(box, MAILBOX_TRANSACTION_FLAG_NO_CACHE_DEC);
search_args = mail_search_build_init();
mail_search_build_add_seqset(search_args, seq, status.messages);
ctx = mailbox_search_init(trans, search_args, NULL,
metadata.precache_fields, NULL);
mail_search_args_unref(&search_args);
max = status.messages - seq + 1;
while (mailbox_search_next(ctx, &mail)) {
mail_precache(mail);
if (++counter % 100 == 0) {
percentage = counter*100 / max;
if (percentage != percentage_sent && percentage < 100) {
percentage_sent = percentage;
if (i_snprintf(percentage_str,
sizeof(percentage_str), "%u\n",
percentage) < 0)
i_unreached();
(void)write_full(conn->fd, percentage_str,
strlen(percentage_str));
}
indexer_worker_refresh_proctitle(username, box_vname,
counter, max);
}
}
if (mailbox_search_deinit(&ctx) < 0)
ret = -1;
if (mailbox_transaction_commit(&trans) < 0)
ret = -1;
if (ret == 0) {
i_info("Indexed %u messages in %s",
counter, mailbox_get_vname(box));
}
return ret;
}
static int
cmd_save_to_mailbox(struct save_cmd_context *ctx, struct mailbox *box,
struct istream *input)
{
struct mail_storage *storage = mailbox_get_storage(box);
struct mailbox_transaction_context *trans;
struct mail_save_context *save_ctx;
ssize_t ret;
bool save_failed = FALSE;
if (mailbox_open(box) < 0) {
i_error("Failed to open mailbox %s: %s",
mailbox_get_vname(box), mailbox_get_last_error(box, NULL));
doveadm_mail_failed_storage(&ctx->ctx, storage);
return -1;
}
trans = mailbox_transaction_begin(box, MAILBOX_TRANSACTION_FLAG_EXTERNAL);
save_ctx = mailbox_save_alloc(trans);
if (mailbox_save_begin(&save_ctx, input) < 0) {
i_error("Saving failed: %s", mailbox_get_last_error(box, NULL));
doveadm_mail_failed_storage(&ctx->ctx, storage);
mailbox_transaction_rollback(&trans);
return -1;
}
while ((ret = i_stream_read(input)) > 0 || ret == -2) {
if (mailbox_save_continue(save_ctx) < 0) {
save_failed = TRUE;
ret = -1;
break;
}
}
i_assert(ret == -1);
if (input->stream_errno != 0) {
i_error("read(msg input) failed: %s", i_stream_get_error(input));
doveadm_mail_failed_error(&ctx->ctx, MAIL_ERROR_TEMP);
} else if (save_failed) {
i_error("Saving failed: %s", mailbox_get_last_error(box, NULL));
doveadm_mail_failed_storage(&ctx->ctx, storage);
} else if (mailbox_save_finish(&save_ctx) < 0) {
i_error("Saving failed: %s",
mailbox_get_last_error(box, NULL));
doveadm_mail_failed_storage(&ctx->ctx, storage);
} else if (mailbox_transaction_commit(&trans) < 0) {
i_error("Save transaction commit failed: %s",
mailbox_get_last_error(box, NULL));
doveadm_mail_failed_storage(&ctx->ctx, storage);
} else {
ret = 0;
}
if (save_ctx != NULL)
mailbox_save_cancel(&save_ctx);
if (trans != NULL)
mailbox_transaction_rollback(&trans);
i_assert(input->eof);
return ret < 0 ? -1 : 0;
}
static int fetch_and_copy(struct client *client, bool move,
struct mailbox_transaction_context *t,
struct mailbox_transaction_context **src_trans_r,
struct mail_search_args *search_args,
const char **src_uidset_r,
unsigned int *copy_count_r)
{
struct mail_search_context *search_ctx;
struct mailbox_transaction_context *src_trans;
struct mail_save_context *save_ctx;
struct mail *mail;
unsigned int copy_count = 0;
struct msgset_generator_context srcset_ctx;
string_t *src_uidset;
int ret;
src_uidset = t_str_new(256);
msgset_generator_init(&srcset_ctx, src_uidset);
src_trans = mailbox_transaction_begin(client->mailbox, 0);
search_ctx = mailbox_search_init(src_trans, search_args, NULL, 0, NULL);
ret = 1;
while (mailbox_search_next(search_ctx, &mail) && ret > 0) {
if (mail->expunged) {
ret = 0;
break;
}
if ((++copy_count % COPY_CHECK_INTERVAL) == 0)
client_send_sendalive_if_needed(client);
save_ctx = mailbox_save_alloc(t);
mailbox_save_copy_flags(save_ctx, mail);
if (move) {
if (mailbox_move(&save_ctx, mail) < 0)
ret = -1;
} else {
if (mailbox_copy(&save_ctx, mail) < 0)
ret = -1;
}
if (ret < 0 && mail->expunged)
ret = 0;
msgset_generator_next(&srcset_ctx, mail->uid);
}
msgset_generator_finish(&srcset_ctx);
if (mailbox_search_deinit(&search_ctx) < 0)
ret = -1;
*src_trans_r = src_trans;
*src_uidset_r = str_c(src_uidset);
*copy_count_r = copy_count;
return ret;
}
static void client_commit_timeout(struct client *client)
{
if (client->cmd != NULL) {
/* Can't commit while commands are running */
return;
}
(void)mailbox_transaction_commit(&client->trans);
client->trans = mailbox_transaction_begin(client->mailbox, 0);
}
static int
quota_count_mailbox(struct quota_root *root, struct mail_namespace *ns,
const char *vname, uint64_t *bytes_r, uint64_t *count_r)
{
struct quota_rule *rule;
struct mailbox *box;
struct mailbox_transaction_context *trans;
struct mail_search_context *ctx;
struct mail *mail;
struct mail_search_args *search_args;
enum mail_error error;
uoff_t size;
int ret = 0;
rule = quota_root_rule_find(root->set, vname);
if (rule != NULL && rule->ignore) {
/* mailbox not included in quota */
return 0;
}
box = mailbox_alloc(ns->list, vname, MAILBOX_FLAG_READONLY);
if (mailbox_sync(box, MAILBOX_SYNC_FLAG_FULL_READ) < 0) {
error = mailbox_get_last_mail_error(box);
mailbox_free(&box);
if (error == MAIL_ERROR_TEMP)
return -1;
/* non-temporary error, e.g. ACLs denied access. */
return 0;
}
trans = mailbox_transaction_begin(box, 0);
search_args = mail_search_build_init();
mail_search_build_add_all(search_args);
ctx = mailbox_search_init(trans, search_args, NULL,
MAIL_FETCH_PHYSICAL_SIZE, NULL);
mail_search_args_unref(&search_args);
while (mailbox_search_next(ctx, &mail)) {
if (mail_get_physical_size(mail, &size) == 0)
*bytes_r += size;
*count_r += 1;
}
if (mailbox_search_deinit(&ctx) < 0)
ret = -1;
if (ret < 0)
mailbox_transaction_rollback(&trans);
else
(void)mailbox_transaction_commit(&trans);
mailbox_free(&box);
return ret;
}
static int
imap_metadata_get_mailbox_transaction(struct imap_metadata_transaction *imtrans)
{
if (imtrans->trans != NULL)
return 0;
if (imtrans->box == NULL || mailbox_open(imtrans->box) < 0)
return -1;
imtrans->trans = mailbox_transaction_begin(imtrans->box, MAILBOX_TRANSACTION_FLAG_EXTERNAL);
return 0;
}
static int
imap_metadata_get_mailbox_transaction(struct imap_metadata_transaction *imtrans)
{
if (imtrans->trans != NULL)
return 0;
if (imtrans->box == NULL || mailbox_open(imtrans->box) < 0)
return -1;
imtrans->trans = mailbox_transaction_begin(imtrans->box, 0);
return 0;
}
static int
cmd_copy_box(struct copy_cmd_context *ctx, struct mailbox *destbox,
const struct mailbox_info *info)
{
struct doveadm_mail_iter *iter;
struct mailbox_transaction_context *desttrans;
struct mail_save_context *save_ctx;
struct mail *mail;
int ret = 0;
if (doveadm_mail_iter_init(&ctx->ctx, info, ctx->ctx.search_args, 0,
NULL, &iter) < 0)
return -1;
/* use a separately committed transaction for each mailbox.
this guarantees that mails aren't expunged without actually having
been copied. */
desttrans = mailbox_transaction_begin(destbox,
MAILBOX_TRANSACTION_FLAG_EXTERNAL);
while (doveadm_mail_iter_next(iter, &mail)) {
save_ctx = mailbox_save_alloc(desttrans);
mailbox_save_copy_flags(save_ctx, mail);
if (mailbox_copy(&save_ctx, mail) == 0) {
if (ctx->move)
mail_expunge(mail);
} else {
i_error("Copying message UID %u from '%s' failed: %s",
mail->uid, info->vname,
mailbox_get_last_error(destbox, NULL));
doveadm_mail_failed_mailbox(&ctx->ctx, destbox);
ret = -1;
}
}
if (mailbox_transaction_commit(&desttrans) < 0) {
i_error("Committing %s mails failed: %s",
ctx->move ? "moved" : "copied",
mailbox_get_last_error(destbox, NULL));
doveadm_mail_failed_mailbox(&ctx->ctx, destbox);
/* rollback expunges */
doveadm_mail_iter_deinit_rollback(&iter);
ret = -1;
} else {
if (doveadm_mail_iter_deinit_sync(&iter) < 0)
ret = -1;
}
return ret;
}
bool cmd_append(struct client_command_context *cmd)
{
struct client *client = cmd->client;
struct cmd_append_context *ctx;
const char *mailbox;
if (client->syncing) {
/* if transaction is created while its view is synced,
appends aren't allowed for it. */
cmd->state = CLIENT_COMMAND_STATE_WAIT_UNAMBIGUITY;
return FALSE;
}
/* <mailbox> */
if (!client_read_string_args(cmd, 1, &mailbox))
return FALSE;
/* we keep the input locked all the time */
client->input_lock = cmd;
ctx = p_new(cmd->pool, struct cmd_append_context, 1);
ctx->cmd = cmd;
ctx->client = client;
ctx->started = ioloop_time;
if (client_open_save_dest_box(cmd, mailbox, &ctx->box) < 0)
ctx->failed = TRUE;
else {
ctx->t = mailbox_transaction_begin(ctx->box,
MAILBOX_TRANSACTION_FLAG_EXTERNAL |
MAILBOX_TRANSACTION_FLAG_ASSIGN_UIDS);
}
io_remove(&client->io);
client->io = io_add(i_stream_get_fd(client->input), IO_READ,
client_input_append, cmd);
/* append is special because we're only waiting on client input, not
client output, so disable the standard output handler until we're
finished */
o_stream_unset_flush_callback(client->output);
ctx->save_parser = imap_parser_create(client->input, client->output,
client->set->imap_max_line_length);
cmd->func = cmd_append_parse_new_msg;
cmd->context = ctx;
return cmd_append_parse_new_msg(cmd);
}
static int pop3_map_read_hdr_hashes(struct mail_storage *storage,
unsigned first_seq)
{
struct pop3_migration_mail_storage *mstorage =
POP3_MIGRATION_CONTEXT(storage);
struct mailbox_transaction_context *t;
struct mail_search_args *search_args;
struct mail_search_context *ctx;
struct mail *mail;
struct pop3_uidl_map *map;
int ret = 0;
if (mstorage->pop3_all_hdr_sha1_set)
return 0;
if (mstorage->all_mailboxes) {
/* we may be matching against multiple mailboxes.
read all the hashes only once. */
first_seq = 1;
}
t = mailbox_transaction_begin(mstorage->pop3_box, 0);
search_args = mail_search_build_init();
mail_search_build_add_seqset(search_args, first_seq,
array_count(&mstorage->pop3_uidl_map)+1);
ctx = mailbox_search_init(t, search_args, NULL,
MAIL_FETCH_STREAM_HEADER, NULL);
mail_search_args_unref(&search_args);
while (mailbox_search_next(ctx, &mail)) {
map = array_idx_modifiable(&mstorage->pop3_uidl_map,
mail->seq-1);
if (get_hdr_sha1(mail, map->hdr_sha1) < 0)
ret = -1;
else
map->hdr_sha1_set = TRUE;
}
if (mailbox_search_deinit(&ctx) < 0)
ret = -1;
(void)mailbox_transaction_commit(&t);
if (ret == 0 && first_seq == 1)
mstorage->pop3_all_hdr_sha1_set = TRUE;
return ret;
}
struct mailbox_transaction_context *
virtual_transaction_get(struct mailbox_transaction_context *trans,
struct mailbox *backend_box)
{
struct virtual_transaction_context *vt =
(struct virtual_transaction_context *)trans;
struct mailbox_transaction_context *const *bt, *new_bt;
unsigned int i, count;
bt = array_get(&vt->backend_transactions, &count);
for (i = 0; i < count; i++) {
if (bt[i]->box == backend_box)
return bt[i];
}
new_bt = mailbox_transaction_begin(backend_box, trans->flags);
array_append(&vt->backend_transactions, &new_bt, 1);
return new_bt;
}
static int imap_map_read(struct mailbox *box)
{
struct pop3_migration_mailbox *mbox = POP3_MIGRATION_CONTEXT(box);
struct mailbox_status status;
struct mailbox_transaction_context *t;
struct mail_search_args *search_args;
struct mail_search_context *ctx;
struct mail *mail;
struct imap_msg_map *map;
uoff_t psize;
int ret = 0;
mailbox_get_open_status(box, STATUS_MESSAGES, &status);
i_assert(!array_is_created(&mbox->imap_msg_map));
p_array_init(&mbox->imap_msg_map, box->pool, status.messages);
t = mailbox_transaction_begin(box, 0);
search_args = mail_search_build_init();
mail_search_build_add_all(search_args);
ctx = mailbox_search_init(t, search_args, NULL,
MAIL_FETCH_PHYSICAL_SIZE, NULL);
mail_search_args_unref(&search_args);
while (mailbox_search_next(ctx, &mail)) {
if (mail_get_physical_size(mail, &psize) < 0) {
i_error("pop3_migration: Failed to get psize for imap uid %u: %s",
mail->uid,
mailbox_get_last_error(box, NULL));
ret = -1;
break;
}
map = array_append_space(&mbox->imap_msg_map);
map->uid = mail->uid;
map->psize = psize;
}
if (mailbox_search_deinit(&ctx) < 0)
ret = -1;
(void)mailbox_transaction_commit(&t);
return ret;
}
int imap_expunge(struct mailbox *box, struct mail_search_arg *next_search_arg,
unsigned int *expunged_count)
{
struct mail_search_context *ctx;
struct mailbox_transaction_context *t;
struct mail *mail;
struct mail_search_args *search_args;
bool expunges = FALSE;
if (mailbox_is_readonly(box)) {
/* silently ignore */
return 0;
}
search_args = mail_search_build_init();
search_args->args = p_new(search_args->pool, struct mail_search_arg, 1);
search_args->args->type = SEARCH_FLAGS;
search_args->args->value.flags = MAIL_DELETED;
search_args->args->next = next_search_arg;
/* Refresh the flags so we'll expunge all messages marked as \Deleted
by any session. */
t = mailbox_transaction_begin(box, MAILBOX_TRANSACTION_FLAG_REFRESH);
ctx = mailbox_search_init(t, search_args, NULL, 0, NULL);
mail_search_args_unref(&search_args);
while (mailbox_search_next(ctx, &mail)) {
*expunged_count += 1;
mail_expunge(mail);
expunges = TRUE;
}
if (mailbox_search_deinit(&ctx) < 0) {
mailbox_transaction_rollback(&t);
return -1;
} else {
if (mailbox_transaction_commit(&t) < 0)
return -1;
}
return expunges ? 1 : 0;
}
static struct mail_raw *mail_raw_create
(struct mail_user *ruser, struct istream *input,
const char *mailfile, const char *sender, time_t mtime)
{
struct mail_raw *mailr;
struct mailbox_header_lookup_ctx *headers_ctx;
const char *envelope_sender;
int ret;
if ( mailfile != NULL && *mailfile != '/' )
mailfile = t_abspath(mailfile);
mailr = i_new(struct mail_raw, 1);
envelope_sender = sender != NULL ? sender : DEFAULT_ENVELOPE_SENDER;
if ( mailfile == NULL ) {
ret = raw_mailbox_alloc_stream(ruser, input, mtime,
envelope_sender, &mailr->box);
} else {
ret = raw_mailbox_alloc_path(ruser, mailfile, (time_t)-1,
envelope_sender, &mailr->box);
}
if ( ret < 0 ) {
if ( mailfile == NULL ) {
i_fatal("Can't open delivery mail as raw: %s",
mailbox_get_last_error(mailr->box, NULL));
} else {
i_fatal("Can't open delivery mail as raw (file=%s): %s",
mailfile, mailbox_get_last_error(mailr->box, NULL));
}
}
mailr->trans = mailbox_transaction_begin(mailr->box, 0);
headers_ctx = mailbox_header_lookup_init(mailr->box, wanted_headers);
mailr->mail = mail_alloc(mailr->trans, 0, headers_ctx);
mailbox_header_lookup_unref(&headers_ctx);
mail_set_seq(mailr->mail, 1);
return mailr;
}
static int
index_mailbox_get_first_save_date(struct mailbox *box,
struct mailbox_metadata *metadata_r)
{
const struct mail_index_header *hdr;
struct mailbox_transaction_context *t;
struct mail *mail;
uint32_t seq;
int ret = -1;
hdr = mail_index_get_header(box->view);
if (hdr->messages_count == 0) {
metadata_r->first_save_date = (time_t)-1;
return 0;
}
t = mailbox_transaction_begin(box, 0, __func__);
mail = mail_alloc(t, 0, NULL);
for (seq = 1; seq <= hdr->messages_count; seq++) {
mail_set_seq(mail, seq);
if (mail_get_save_date(mail, &metadata_r->first_save_date) == 0) {
ret = 0;
break;
}
if (mailbox_get_last_mail_error(box) != MAIL_ERROR_EXPUNGED) {
/* failed */
break;
}
}
mail_free(&mail);
(void)mailbox_transaction_commit(&t);
if (seq > hdr->messages_count) {
/* all messages were expunged after all */
metadata_r->first_save_date = (time_t)-1;
return 0;
}
return ret;
}
int imap_msgpart_url_open_mail(struct imap_msgpart_url *mpurl,
struct mail **mail_r, const char **error_r)
{
struct mailbox_transaction_context *t;
struct mailbox *box;
enum mail_error error_code;
struct mail *mail;
int ret;
if (mpurl->mail != NULL) {
*mail_r = mpurl->mail;
return 1;
}
/* open mailbox if it is not yet open */
if ((ret = imap_msgpart_url_open_mailbox(mpurl, &box, &error_code,
error_r)) <= 0)
return ret;
/* start transaction */
t = mailbox_transaction_begin(box, 0);
mail = mail_alloc(t, MAIL_FETCH_MESSAGE_PARTS |
MAIL_FETCH_IMAP_BODYSTRUCTURE, NULL);
/* find the message */
if (!mail_set_uid(mail, mpurl->uid)) {
*error_r = "Message not found";
mail_free(&mail);
mailbox_transaction_rollback(&t);
return 0;
}
mpurl->trans = t;
mpurl->mail = mail;
*mail_r = mail;
return 1;
}
static int
cmd_mailbox_metadata_set_run(struct doveadm_mail_cmd_context *_ctx,
struct mail_user *user)
{
struct metadata_cmd_context *ctx = (struct metadata_cmd_context *)_ctx;
struct mail_namespace *ns;
struct mailbox *box;
struct mailbox_transaction_context *trans;
int ret;
ret = cmd_mailbox_metadata_open_mailbox(ctx, user, "set attribute",
&ns, &box);
if (ret != 0)
return ret;
trans = mailbox_transaction_begin(box, ctx->empty_mailbox_name ?
MAILBOX_TRANSACTION_FLAG_EXTERNAL : 0);
ret = ctx->value.value == NULL ?
mailbox_attribute_unset(trans, ctx->key_type, ctx->key) :
mailbox_attribute_set(trans, ctx->key_type, ctx->key, &ctx->value);
if (ret < 0) {
i_error("Failed to set attribute: %s",
mailbox_get_last_error(box, NULL));
doveadm_mail_failed_mailbox(_ctx, box);
mailbox_transaction_rollback(&trans);
} else if (mailbox_transaction_commit(&trans) < 0) {
i_error("Failed to commit transaction: %s",
mailbox_get_last_error(box, NULL));
doveadm_mail_failed_mailbox(_ctx, box);
ret = -1;
}
mailbox_free(&box);
return ret;
}
static int snarf(struct mailbox *srcbox, struct mailbox *destbox)
{
struct mail_search_args *search_args;
struct mail_search_context *search_ctx;
struct mailbox_transaction_context *src_trans, *dest_trans;
struct mail_save_context *save_ctx;
struct mail *mail;
enum mail_error error;
int ret;
/* make sure the destination mailbox has been opened.
note that this locks the mailbox. */
if (mailbox_open(destbox) < 0)
return -1;
if (mailbox_sync(srcbox, MAILBOX_SYNC_FLAG_FULL_READ) < 0)
return -1;
src_trans = mailbox_transaction_begin(srcbox, 0);
dest_trans = mailbox_transaction_begin(destbox,
MAILBOX_TRANSACTION_FLAG_EXTERNAL);
search_args = mail_search_build_init();
mail_search_build_add_all(search_args);
search_ctx = mailbox_search_init(src_trans, search_args, NULL,
MAIL_FETCH_STREAM_HEADER |
MAIL_FETCH_STREAM_BODY, NULL);
mail_search_args_unref(&search_args);
ret = 0;
while (mailbox_search_next(search_ctx, &mail)) {
if (mail->expunged)
continue;
save_ctx = mailbox_save_alloc(dest_trans);
if (mailbox_copy(&save_ctx, mail) < 0 && !mail->expunged) {
error = mailbox_get_last_mail_error(destbox);
/* if we failed because of out of disk space, just
move those messages we managed to move so far. */
if (error != MAIL_ERROR_NOQUOTA)
ret = -1;
break;
}
mail_expunge(mail);
}
if (mailbox_search_deinit(&search_ctx) < 0)
ret = -1;
/* commit the copied messages to the destination mailbox. if we crash
between that and between expunging the messages from the source
mailbox, we're left with duplicates. */
if (ret < 0)
mailbox_transaction_rollback(&dest_trans);
else if (mailbox_transaction_commit(&dest_trans) < 0)
ret = -1;
if (ret < 0)
mailbox_transaction_rollback(&src_trans);
else {
if (mailbox_transaction_commit(&src_trans) < 0)
ret = -1;
}
if (ret == 0) {
if (mailbox_sync(srcbox, 0) < 0)
ret = -1;
}
return ret;
}
static int pop3_map_read(struct mail_storage *storage)
{
struct pop3_migration_mail_storage *mstorage =
POP3_MIGRATION_CONTEXT(storage);
struct mailbox *pop3_box = mstorage->pop3_box;
struct mailbox_transaction_context *t;
struct mail_search_args *search_args;
struct mail_search_context *ctx;
struct mail *mail;
struct pop3_uidl_map *map;
const char *uidl;
uoff_t size;
int ret = 0;
if (array_is_created(&mstorage->pop3_uidl_map)) {
/* already read these, just reset the imap_uids */
array_foreach_modifiable(&mstorage->pop3_uidl_map, map)
map->imap_uid = 0;
return 0;
}
i_array_init(&mstorage->pop3_uidl_map, 128);
if (mailbox_sync(pop3_box, 0) < 0) {
i_error("pop3_migration: Couldn't sync mailbox %s: %s",
pop3_box->vname, mailbox_get_last_error(pop3_box, NULL));
return -1;
}
t = mailbox_transaction_begin(pop3_box, 0);
search_args = mail_search_build_init();
mail_search_build_add_all(search_args);
ctx = mailbox_search_init(t, search_args, NULL,
MAIL_FETCH_VIRTUAL_SIZE, NULL);
mail_search_args_unref(&search_args);
while (mailbox_search_next(ctx, &mail)) {
if (mail_get_virtual_size(mail, &size) < 0) {
i_error("pop3_migration: Failed to get size for msg %u: %s",
mail->seq,
mailbox_get_last_error(pop3_box, NULL));
ret = -1;
break;
}
if (mail_get_special(mail, MAIL_FETCH_UIDL_BACKEND, &uidl) < 0) {
i_error("pop3_migration: Failed to get UIDL for msg %u: %s",
mail->seq,
mailbox_get_last_error(pop3_box, NULL));
ret = -1;
break;
}
if (*uidl == '\0') {
i_warning("pop3_migration: UIDL for msg %u is empty",
mail->seq);
continue;
}
map = array_append_space(&mstorage->pop3_uidl_map);
map->pop3_seq = mail->seq;
map->pop3_uidl = p_strdup(storage->pool, uidl);
map->size = size;
}
if (mailbox_search_deinit(&ctx) < 0)
ret = -1;
(void)mailbox_transaction_commit(&t);
return ret;
}
static int
virtual_size_add_new(struct mailbox *box,
struct index_vsize_header *vsize_hdr)
{
struct index_mailbox_context *ibox = INDEX_STORAGE_CONTEXT(box);
const struct mail_index_header *hdr;
struct mailbox_transaction_context *trans;
struct mail_search_context *search_ctx;
struct mail_search_args *search_args;
struct mail *mail;
uint32_t seq1, seq2;
uoff_t vsize;
int ret = 0;
hdr = mail_index_get_header(box->view);
if (vsize_hdr->highest_uid == 0)
seq2 = 0;
else if (!mail_index_lookup_seq_range(box->view, 1,
vsize_hdr->highest_uid,
&seq1, &seq2))
seq2 = 0;
if (vsize_hdr->message_count != seq2) {
if (vsize_hdr->message_count < seq2) {
mail_storage_set_critical(box->storage,
"vsize-hdr has invalid message-count (%u < %u)",
vsize_hdr->message_count, seq2);
} else {
/* some messages have been expunged, rescan */
}
memset(vsize_hdr, 0, sizeof(*vsize_hdr));
seq2 = 0;
}
search_args = mail_search_build_init();
mail_search_build_add_seqset(search_args, seq2 + 1,
hdr->messages_count);
trans = mailbox_transaction_begin(box, 0);
search_ctx = mailbox_search_init(trans, search_args, NULL,
MAIL_FETCH_VIRTUAL_SIZE, NULL);
while (mailbox_search_next(search_ctx, &mail)) {
if (mail_get_virtual_size(mail, &vsize) < 0) {
if (mail->expunged)
continue;
ret = -1;
break;
}
vsize_hdr->vsize += vsize;
vsize_hdr->highest_uid = mail->uid;
vsize_hdr->message_count++;
}
if (mailbox_search_deinit(&search_ctx) < 0)
ret = -1;
mail_search_args_unref(&search_args);
if (ret == 0) {
/* success, cache all */
vsize_hdr->highest_uid = hdr->next_uid - 1;
} else {
/* search failed, cache only up to highest seen uid */
}
mail_index_update_header_ext(trans->itrans, ibox->vsize_hdr_ext_id,
0, vsize_hdr, sizeof(*vsize_hdr));
(void)mailbox_transaction_commit(&trans);
return ret;
}
bool cmd_copy(struct client_command_context *cmd)
{
struct client *client = cmd->client;
struct mail_namespace *dest_ns;
struct mail_storage *dest_storage;
struct mailbox *destbox;
struct mailbox_transaction_context *t;
struct mail_search_args *search_args;
const char *messageset, *mailbox, *storage_name, *src_uidset;
enum mailbox_name_status status;
enum mailbox_sync_flags sync_flags = 0;
enum imap_sync_flags imap_flags = 0;
struct mail_transaction_commit_changes changes;
unsigned int copy_count;
string_t *msg;
int ret;
/* <message set> <mailbox> */
if (!client_read_string_args(cmd, 2, &messageset, &mailbox))
return FALSE;
if (!client_verify_open_mailbox(cmd))
return TRUE;
ret = imap_search_get_seqset(cmd, messageset, cmd->uid, &search_args);
if (ret <= 0)
return ret < 0;
/* open the destination mailbox */
dest_ns = client_find_namespace(cmd, mailbox, &storage_name, &status);
if (dest_ns == NULL) {
mail_search_args_unref(&search_args);
return TRUE;
}
switch (status) {
case MAILBOX_NAME_EXISTS_MAILBOX:
break;
case MAILBOX_NAME_EXISTS_DIR:
status = MAILBOX_NAME_VALID;
/* fall through */
case MAILBOX_NAME_VALID:
case MAILBOX_NAME_INVALID:
case MAILBOX_NAME_NOINFERIORS:
client_fail_mailbox_name_status(cmd, mailbox,
"TRYCREATE", status);
mail_search_args_unref(&search_args);
return TRUE;
}
if (mailbox_equals(client->mailbox, dest_ns, storage_name))
destbox = client->mailbox;
else {
destbox = mailbox_alloc(dest_ns->list, storage_name,
MAILBOX_FLAG_SAVEONLY |
MAILBOX_FLAG_KEEP_RECENT);
if (mailbox_open(destbox) < 0) {
client_send_storage_error(cmd,
mailbox_get_storage(destbox));
mailbox_free(&destbox);
mail_search_args_unref(&search_args);
return TRUE;
}
if (client->enabled_features != 0)
mailbox_enable(destbox, client->enabled_features);
}
t = mailbox_transaction_begin(destbox,
MAILBOX_TRANSACTION_FLAG_EXTERNAL |
MAILBOX_TRANSACTION_FLAG_ASSIGN_UIDS);
ret = fetch_and_copy(client, t, search_args, &src_uidset, ©_count);
mail_search_args_unref(&search_args);
msg = t_str_new(256);
if (ret <= 0)
mailbox_transaction_rollback(&t);
else if (mailbox_transaction_commit_get_changes(&t, &changes) < 0)
ret = -1;
else if (copy_count == 0) {
str_append(msg, "OK No messages copied.");
pool_unref(&changes.pool);
} else if (seq_range_count(&changes.saved_uids) == 0) {
/* not supported by backend (virtual) */
str_append(msg, "OK Copy completed.");
pool_unref(&changes.pool);
} else {
i_assert(copy_count == seq_range_count(&changes.saved_uids));
str_printfa(msg, "OK [COPYUID %u %s ", changes.uid_validity,
src_uidset);
imap_write_seq_range(msg, &changes.saved_uids);
str_append(msg, "] Copy completed.");
pool_unref(&changes.pool);
}
dest_storage = mailbox_get_storage(destbox);
if (destbox != client->mailbox) {
sync_flags |= MAILBOX_SYNC_FLAG_FAST;
imap_flags |= IMAP_SYNC_FLAG_SAFE;
mailbox_free(&destbox);
}
if (ret > 0)
return cmd_sync(cmd, sync_flags, imap_flags, str_c(msg));
else if (ret == 0) {
/* some messages were expunged, sync them */
return cmd_sync(cmd, 0, 0,
"NO ["IMAP_RESP_CODE_EXPUNGEISSUED"] "
"Some of the requested messages no longer exist.");
} else {
client_send_storage_error(cmd, dest_storage);
return TRUE;
}
}
bool cmd_store(struct client_command_context *cmd)
{
struct client *client = cmd->client;
const struct imap_arg *args;
struct mail_search_args *search_args;
struct mail_search_context *search_ctx;
struct mailbox_transaction_context *t;
struct mail *mail;
struct imap_store_context ctx;
ARRAY_TYPE(seq_range) modified_set, uids;
enum mailbox_transaction_flags flags = 0;
enum imap_sync_flags imap_sync_flags = 0;
const char *set, *reply, *tagged_reply;
string_t *str;
int ret;
if (!client_read_args(cmd, 0, 0, &args))
return FALSE;
if (!client_verify_open_mailbox(cmd))
return TRUE;
if (!imap_arg_get_atom(args, &set)) {
client_send_command_error(cmd, "Invalid arguments.");
return TRUE;
}
ret = imap_search_get_seqset(cmd, set, cmd->uid, &search_args);
if (ret <= 0)
return ret < 0;
memset(&ctx, 0, sizeof(ctx));
ctx.cmd = cmd;
if (!store_parse_args(&ctx, ++args)) {
mail_search_args_unref(&search_args);
return TRUE;
}
if (client->mailbox_examined) {
mail_search_args_unref(&search_args);
if (ctx.max_modseq < (uint64_t)-1)
reply = "NO CONDSTORE failed: Mailbox is read-only.";
else
reply = "OK Store ignored with read-only mailbox.";
return cmd_sync(cmd, MAILBOX_SYNC_FLAG_FAST |
(cmd->uid ? 0 : MAILBOX_SYNC_FLAG_NO_EXPUNGES),
0, reply);
}
if (ctx.silent)
flags |= MAILBOX_TRANSACTION_FLAG_HIDE;
if (ctx.max_modseq < (uint64_t)-1) {
/* update modseqs so we can check them early */
flags |= MAILBOX_TRANSACTION_FLAG_REFRESH;
}
t = mailbox_transaction_begin(client->mailbox, flags);
search_ctx = mailbox_search_init(t, search_args, NULL,
MAIL_FETCH_FLAGS, NULL);
mail_search_args_unref(&search_args);
i_array_init(&modified_set, 64);
if (ctx.max_modseq < (uint32_t)-1) {
/* STORE UNCHANGEDSINCE is being used */
mailbox_transaction_set_max_modseq(t, ctx.max_modseq,
&modified_set);
}
while (mailbox_search_next(search_ctx, &mail)) {
if (ctx.max_modseq < (uint64_t)-1) {
/* check early so there's less work for transaction
commit if something has to be cancelled */
if (mail_get_modseq(mail) > ctx.max_modseq) {
seq_range_array_add(&modified_set, mail->seq);
continue;
}
}
if (ctx.modify_type == MODIFY_REPLACE || ctx.flags != 0)
mail_update_flags(mail, ctx.modify_type, ctx.flags);
if (ctx.modify_type == MODIFY_REPLACE || ctx.keywords != NULL) {
mail_update_keywords(mail, ctx.modify_type,
ctx.keywords);
}
}
if (ctx.keywords != NULL)
mailbox_keywords_unref(&ctx.keywords);
ret = mailbox_search_deinit(&search_ctx);
if (ret < 0)
mailbox_transaction_rollback(&t);
else
ret = mailbox_transaction_commit(&t);
if (ret < 0) {
array_free(&modified_set);
client_send_box_error(cmd, client->mailbox);
return TRUE;
}
if (array_count(&modified_set) == 0)
tagged_reply = "OK Store completed.";
else {
if (cmd->uid) {
i_array_init(&uids, array_count(&modified_set)*2);
mailbox_get_uid_range(client->mailbox, &modified_set,
&uids);
array_free(&modified_set);
modified_set = uids;
}
str = str_new(cmd->pool, 256);
str_append(str, "OK [MODIFIED ");
imap_write_seq_range(str, &modified_set);
str_append(str, "] Conditional store failed.");
tagged_reply = str_c(str);
}
array_free(&modified_set);
/* With UID STORE we have to return UID for the flags as well.
Unfortunately we don't have the ability to separate those
flag changes that were caused by UID STORE and those that
came externally, so we'll just send the UID for all flag
changes that we see. */
if (cmd->uid && (!ctx.silent || (client->enabled_features &
MAILBOX_FEATURE_CONDSTORE) != 0))
imap_sync_flags |= IMAP_SYNC_FLAG_SEND_UID;
return cmd_sync(cmd, (cmd->uid ? 0 : MAILBOX_SYNC_FLAG_NO_EXPUNGES),
imap_sync_flags, tagged_reply);
}
static bool cmd_copy_full(struct client_command_context *cmd, bool move)
{
struct client *client = cmd->client;
struct mail_storage *dest_storage;
struct mailbox *destbox;
struct mailbox_transaction_context *t, *src_trans;
struct mail_search_args *search_args;
const char *messageset, *mailbox, *src_uidset;
enum mailbox_sync_flags sync_flags = 0;
enum imap_sync_flags imap_flags = 0;
struct mail_transaction_commit_changes changes;
unsigned int copy_count;
string_t *msg;
int ret;
/* <message set> <mailbox> */
if (!client_read_string_args(cmd, 2, &messageset, &mailbox))
return FALSE;
if (!client_verify_open_mailbox(cmd))
return TRUE;
ret = imap_search_get_seqset(cmd, messageset, cmd->uid, &search_args);
if (ret <= 0)
return ret < 0;
if (client_open_save_dest_box(cmd, mailbox, &destbox) < 0) {
mail_search_args_unref(&search_args);
return TRUE;
}
t = mailbox_transaction_begin(destbox,
MAILBOX_TRANSACTION_FLAG_EXTERNAL |
MAILBOX_TRANSACTION_FLAG_ASSIGN_UIDS);
ret = fetch_and_copy(client, move, t, &src_trans, search_args,
&src_uidset, ©_count);
mail_search_args_unref(&search_args);
msg = t_str_new(256);
if (ret <= 0)
mailbox_transaction_rollback(&t);
else if (mailbox_transaction_commit_get_changes(&t, &changes) < 0)
ret = -1;
else if (copy_count == 0) {
str_append(msg, "OK No messages found.");
pool_unref(&changes.pool);
} else if (seq_range_count(&changes.saved_uids) == 0 ||
changes.no_read_perm) {
/* not supported by backend (virtual) or no read permissions
for mailbox */
str_append(msg, move ? "OK Move completed." :
"OK Copy completed.");
pool_unref(&changes.pool);
} else if (move) {
i_assert(copy_count == seq_range_count(&changes.saved_uids));
str_printfa(msg, "* OK [COPYUID %u %s ",
changes.uid_validity, src_uidset);
imap_write_seq_range(msg, &changes.saved_uids);
str_append(msg, "] Moved UIDs.");
client_send_line(client, str_c(msg));
str_truncate(msg, 0);
str_append(msg, "OK Move completed.");
pool_unref(&changes.pool);
} else {
i_assert(copy_count == seq_range_count(&changes.saved_uids));
str_printfa(msg, "OK [COPYUID %u %s ", changes.uid_validity,
src_uidset);
imap_write_seq_range(msg, &changes.saved_uids);
str_append(msg, "] Copy completed.");
pool_unref(&changes.pool);
}
if (ret <= 0 && move) {
/* move failed, don't expunge anything */
mailbox_transaction_rollback(&src_trans);
} else {
if (mailbox_transaction_commit(&src_trans) < 0)
ret = -1;
}
dest_storage = mailbox_get_storage(destbox);
if (destbox != client->mailbox) {
if (move)
sync_flags |= MAILBOX_SYNC_FLAG_EXPUNGE;
else
sync_flags |= MAILBOX_SYNC_FLAG_FAST;
imap_flags |= IMAP_SYNC_FLAG_SAFE;
mailbox_free(&destbox);
} else if (move) {
sync_flags |= MAILBOX_SYNC_FLAG_EXPUNGE;
imap_flags |= IMAP_SYNC_FLAG_SAFE;
}
if (ret > 0)
return cmd_sync(cmd, sync_flags, imap_flags, str_c(msg));
else if (ret == 0) {
/* some messages were expunged, sync them */
return cmd_sync(cmd, 0, 0,
"NO ["IMAP_RESP_CODE_EXPUNGEISSUED"] "
"Some of the requested messages no longer exist.");
} else {
client_send_storage_error(cmd, dest_storage);
return TRUE;
}
}
bool cmd_setmetadata(struct client_command_context *cmd)
{
struct imap_setmetadata_context *ctx;
const struct imap_arg *args;
const char *mailbox;
struct mail_namespace *ns;
int ret;
ret = imap_parser_read_args(cmd->parser, 2,
IMAP_PARSE_FLAG_STOP_AT_LIST, &args);
if (ret == -1) {
client_send_command_error(cmd, NULL);
return TRUE;
}
if (ret == -2)
return FALSE;
if (!imap_arg_get_astring(&args[0], &mailbox) ||
args[1].type != IMAP_ARG_LIST) {
client_send_command_error(cmd, "Invalid arguments.");
return TRUE;
}
if (!cmd->client->imap_metadata_enabled) {
client_send_command_error(cmd, "METADATA disabled.");
return TRUE;
}
ctx = p_new(cmd->pool, struct imap_setmetadata_context, 1);
ctx->cmd = cmd;
ctx->cmd->context = ctx;
if (mailbox[0] == '\0') {
/* server attribute */
ctx->key_prefix = MAILBOX_ATTRIBUTE_PREFIX_DOVECOT_PVT_SERVER;
ns = mail_namespace_find_inbox(cmd->client->user->namespaces);
mailbox = "INBOX";
} else {
ns = client_find_namespace(cmd, &mailbox);
if (ns == NULL)
return TRUE;
}
if (cmd->client->mailbox != NULL && !cmd->client->mailbox_examined &&
mailbox_equals(cmd->client->mailbox, ns, mailbox))
ctx->box = cmd->client->mailbox;
else {
ctx->box = mailbox_alloc(ns->list, mailbox, 0);
if (mailbox_open(ctx->box) < 0) {
client_send_box_error(cmd, ctx->box);
mailbox_free(&ctx->box);
return TRUE;
}
}
ctx->trans = mailbox_transaction_begin(ctx->box, 0);
/* we support large literals, so read the values from client
asynchronously the same way as APPEND does. */
cmd->client->input_lock = cmd;
ctx->parser = imap_parser_create(cmd->client->input, cmd->client->output,
cmd->client->set->imap_max_line_length);
o_stream_unset_flush_callback(cmd->client->output);
cmd->func = cmd_setmetadata_continue;
cmd->context = ctx;
return cmd_setmetadata_continue(cmd);
}
static int
index_mailbox_precache(struct master_connection *conn, struct mailbox *box)
{
struct mail_storage *storage = mailbox_get_storage(box);
const char *username = mail_storage_get_user(storage)->username;
const char *box_vname = mailbox_get_vname(box);
struct mailbox_status status;
struct mailbox_transaction_context *trans;
struct mail_search_args *search_args;
struct mail_search_context *ctx;
struct mail *mail;
struct mailbox_metadata metadata;
uint32_t seq, first_uid = 0, last_uid = 0;
char percentage_str[2+1+1];
unsigned int counter = 0, max, percentage, percentage_sent = 0;
int ret = 0;
if (mailbox_get_metadata(box, MAILBOX_METADATA_PRECACHE_FIELDS,
&metadata) < 0) {
i_error("Mailbox %s: Precache-fields lookup failed: %s",
mailbox_get_vname(box),
mailbox_get_last_internal_error(box, NULL));
return -1;
}
if (mailbox_get_status(box, STATUS_MESSAGES | STATUS_LAST_CACHED_SEQ,
&status) < 0) {
i_error("Mailbox %s: Status lookup failed: %s",
mailbox_get_vname(box),
mailbox_get_last_internal_error(box, NULL));
return -1;
}
seq = status.last_cached_seq + 1;
trans = mailbox_transaction_begin(box, MAILBOX_TRANSACTION_FLAG_NO_CACHE_DEC,
"indexing");
search_args = mail_search_build_init();
mail_search_build_add_seqset(search_args, seq, status.messages);
ctx = mailbox_search_init(trans, search_args, NULL,
metadata.precache_fields, NULL);
mail_search_args_unref(&search_args);
max = status.messages + 1 - seq;
while (mailbox_search_next(ctx, &mail)) {
if (first_uid == 0)
first_uid = mail->uid;
last_uid = mail->uid;
mail_precache(mail);
if (++counter % 100 == 0) {
percentage = counter*100 / max;
if (percentage != percentage_sent && percentage < 100) {
percentage_sent = percentage;
if (i_snprintf(percentage_str,
sizeof(percentage_str), "%u\n",
percentage) < 0)
i_unreached();
(void)write_full(conn->fd, percentage_str,
strlen(percentage_str));
}
indexer_worker_refresh_proctitle(username, box_vname,
counter, max);
}
}
if (mailbox_search_deinit(&ctx) < 0) {
i_error("Mailbox %s: Mail search failed: %s",
mailbox_get_vname(box),
mailbox_get_last_internal_error(box, NULL));
ret = -1;
}
const char *uids = first_uid == 0 ? "" :
t_strdup_printf(" (UIDs %u..%u)", first_uid, last_uid);
if (mailbox_transaction_commit(&trans) < 0) {
i_error("Mailbox %s: Transaction commit failed: %s"
" (attempted to index %u messages%s)",
mailbox_get_vname(box),
mailbox_get_last_internal_error(box, NULL),
counter, uids);
ret = -1;
} else {
i_info("Indexed %u messages in %s%s",
counter, mailbox_get_vname(box), uids);
}
return ret;
}