static int
index_mailbox_precache(struct master_connection *conn, struct mailbox *box)
{
struct mail_storage *storage = mailbox_get_storage(box);
const char *username = mail_storage_get_user(storage)->username;
const char *box_vname = mailbox_get_vname(box);
struct mailbox_status status;
struct mailbox_transaction_context *trans;
struct mail_search_args *search_args;
struct mail_search_context *ctx;
struct mail *mail;
struct mailbox_metadata metadata;
uint32_t seq;
char percentage_str[2+1+1];
unsigned int counter = 0, max, percentage, percentage_sent = 0;
int ret = 0;
if (mailbox_get_metadata(box, MAILBOX_METADATA_PRECACHE_FIELDS,
&metadata) < 0 ||
mailbox_get_status(box, STATUS_MESSAGES | STATUS_LAST_CACHED_SEQ,
&status) < 0)
return -1;
seq = status.last_cached_seq + 1;
trans = mailbox_transaction_begin(box, MAILBOX_TRANSACTION_FLAG_NO_CACHE_DEC);
search_args = mail_search_build_init();
mail_search_build_add_seqset(search_args, seq, status.messages);
ctx = mailbox_search_init(trans, search_args, NULL,
metadata.precache_fields, NULL);
mail_search_args_unref(&search_args);
max = status.messages - seq + 1;
while (mailbox_search_next(ctx, &mail)) {
mail_precache(mail);
if (++counter % 100 == 0) {
percentage = counter*100 / max;
if (percentage != percentage_sent && percentage < 100) {
percentage_sent = percentage;
if (i_snprintf(percentage_str,
sizeof(percentage_str), "%u\n",
percentage) < 0)
i_unreached();
(void)write_full(conn->fd, percentage_str,
strlen(percentage_str));
}
indexer_worker_refresh_proctitle(username, box_vname,
counter, max);
}
}
if (mailbox_search_deinit(&ctx) < 0)
ret = -1;
if (mailbox_transaction_commit(&trans) < 0)
ret = -1;
if (ret == 0) {
i_info("Indexed %u messages in %s",
counter, mailbox_get_vname(box));
}
return ret;
}
bool imap_client_hibernate(struct client **_client)
{
struct client *client = *_client;
buffer_t *state;
const char *error;
int ret, fd_notify = -1;
if (client->fd_in != client->fd_out) {
/* we won't try to hibernate stdio clients */
return FALSE;
}
if (o_stream_get_buffer_used_size(client->output) > 0) {
/* wait until we've sent the pending output to client */
return FALSE;
}
state = buffer_create_dynamic(default_pool, 1024);
ret = imap_state_export_internal(client, state, &error);
if (ret < 0) {
i_error("Couldn't hibernate imap client: "
"Couldn't export state: %s (mailbox=%s)", error,
client->mailbox == NULL ? "" :
mailbox_get_vname(client->mailbox));
} else if (ret == 0 && client->user->mail_debug) {
i_debug("Couldn't hibernate imap client: "
"Couldn't export state: %s (mailbox=%s)", error,
client->mailbox == NULL ? "" :
mailbox_get_vname(client->mailbox));
}
if (ret > 0 && client->mailbox != NULL) {
fd_notify = mailbox_watch_extract_notify_fd(client->mailbox,
&error);
if (fd_notify == -1) {
if (client->user->mail_debug) {
i_debug("Couldn't hibernate imap client: "
"Couldn't extract notifications fd: %s",
error);
}
ret = -1;
}
}
if (ret > 0) {
if (imap_hibernate_process_send(client, state, fd_notify) < 0)
ret = -1;
}
if (fd_notify != -1)
i_close_fd(&fd_notify);
if (ret > 0) {
/* hide the disconnect log message, because the client didn't
actually log out */
client->disconnected = TRUE;
client_destroy(client, NULL);
*_client = NULL;
}
buffer_free(&state);
return ret > 0;
}
static int
cmd_expunge_box(struct doveadm_mail_cmd_context *_ctx,
const struct mailbox_info *info,
struct mail_search_args *search_args)
{
struct expunge_cmd_context *ctx = (struct expunge_cmd_context *)_ctx;
struct doveadm_mail_iter *iter;
struct mailbox *box;
struct mail *mail;
enum mail_error error;
int ret = 0;
if (doveadm_mail_iter_init(_ctx, info, search_args, 0, NULL, FALSE,
&iter) < 0)
return -1;
while (doveadm_mail_iter_next(iter, &mail)) {
if (doveadm_debug) {
i_debug("expunge: box=%s uid=%u",
info->vname, mail->uid);
}
mail_expunge(mail);
}
if (doveadm_mail_iter_deinit_keep_box(&iter, &box) < 0)
ret = -1;
else if (mailbox_sync(box, 0) < 0) {
i_error("Syncing mailbox '%s' failed: %s",
mailbox_get_vname(box),
mailbox_get_last_internal_error(box, NULL));
doveadm_mail_failed_mailbox(_ctx, box);
ret = -1;
}
if (ctx->delete_empty_mailbox && ret == 0) {
if (mailbox_delete_empty(box) < 0) {
error = mailbox_get_last_mail_error(box);
if (error != MAIL_ERROR_EXISTS) {
i_error("Deleting mailbox '%s' failed: %s",
mailbox_get_vname(box),
mailbox_get_last_internal_error(box, NULL));
doveadm_mail_failed_mailbox(_ctx, box);
ret = -1;
}
} else {
if (mailbox_set_subscribed(box, FALSE) < 0) {
i_error("Unsubscribing mailbox '%s' failed: %s",
mailbox_get_vname(box),
mailbox_get_last_internal_error(box, NULL));
doveadm_mail_failed_mailbox(_ctx, box);
ret = -1;
}
}
}
mailbox_free(&box);
return ret;
}
static bool fts_autoindex_exclude_match(struct mailbox *box)
{
const char *const *exclude_list;
unsigned int i;
const struct mailbox_settings *set;
const char *const *special_use;
struct mail_user *user = box->storage->user;
exclude_list = fts_exclude_get_patterns(user);
if (exclude_list == NULL)
return FALSE;
set = mailbox_settings_find(mailbox_get_namespace(box),
mailbox_get_vname(box));
special_use = set == NULL ? NULL :
t_strsplit_spaces(set->special_use, " ");
for (i = 0; exclude_list[i] != NULL; i++) {
if (exclude_list[i][0] == '\\') {
/* \Special-use flag */
if (str_array_icase_find(special_use, exclude_list[i]))
return TRUE;
} else {
/* mailbox name with wildcards */
if (wildcard_match(box->name, exclude_list[i]))
return TRUE;
}
}
return FALSE;
}
static int
cmd_import_box_contents(struct doveadm_mail_iter *iter, struct mail *src_mail,
struct mailbox *dest_box)
{
struct mail_save_context *save_ctx;
struct mailbox_transaction_context *dest_trans;
const char *mailbox = mailbox_get_vname(dest_box);
int ret = 0;
dest_trans = mailbox_transaction_begin(dest_box,
MAILBOX_TRANSACTION_FLAG_EXTERNAL);
do {
if (doveadm_debug) {
i_debug("import: box=%s uid=%u",
mailbox, src_mail->uid);
}
save_ctx = mailbox_save_alloc(dest_trans);
mailbox_save_copy_flags(save_ctx, src_mail);
if (mailbox_copy(&save_ctx, src_mail) < 0) {
i_error("Copying box=%s uid=%u failed: %s",
mailbox, src_mail->uid,
mailbox_get_last_error(dest_box, NULL));
ret = -1;
}
} while (doveadm_mail_iter_next(iter, &src_mail));
if (mailbox_transaction_commit(&dest_trans) < 0) {
i_error("Committing copied mails to %s failed: %s", mailbox,
mailbox_get_last_error(dest_box, NULL));
ret = -1;
}
return ret;
}
static void
i_stream_mail_set_size_corrupted(struct mail_istream *mstream, size_t size)
{
uoff_t cur_size = mstream->istream.istream.v_offset + size;
const char *str, *mail_id;
char chr;
if (mstream->expected_size < cur_size) {
str = "smaller";
chr = '<';
} else {
str = "larger";
chr = '>';
}
mail_id = i_stream_mail_get_cached_mail_id(mstream);
if (mail_id[0] != '\0')
mail_id = t_strconcat(", cached ", mail_id, NULL);
io_stream_set_error(&mstream->istream.iostream,
"Cached message size %s than expected "
"(%"PRIuUOFF_T" %c %"PRIuUOFF_T", box=%s, UID=%u%s)", str,
mstream->expected_size, chr, cur_size,
mailbox_get_vname(mstream->mail->box),
mstream->mail->uid, mail_id);
mail_set_cache_corrupted_reason(mstream->mail, MAIL_FETCH_PHYSICAL_SIZE,
t_strdup_printf("read(%s) failed: %s",
i_stream_get_name(&mstream->istream.istream),
mstream->istream.iostream.error));
mstream->istream.istream.stream_errno = EINVAL;
}
static void mail_log_append_mailbox_name(string_t *str, struct mail *mail)
{
const char *mailbox_str;
mailbox_str = mailbox_get_vname(mail->box);
str_printfa(str, "box=%s",
str_sanitize(mailbox_str, MAILBOX_NAME_LOG_LEN));
}
static void fetch_read_error(struct imap_fetch_context *ctx)
{
errno = ctx->cur_input->stream_errno;
mail_storage_set_critical(ctx->box->storage,
"read(%s) failed: %m (FETCH for mailbox %s UID %u)",
i_stream_get_name(ctx->cur_input),
mailbox_get_vname(ctx->mail->box), ctx->mail->uid);
}
static int cmd_deduplicate_uidlist(struct doveadm_mail_cmd_context *_ctx,
struct mailbox *box, struct uidlist *uidlist)
{
struct mailbox_transaction_context *trans;
struct mail_search_context *search_ctx;
struct mail_search_args *search_args;
struct mail_search_arg *arg;
struct mail *mail;
ARRAY_TYPE(seq_range) uids;
int ret = 0;
/* the uidlist is reversed with oldest mails at the end.
we'll delete everything but the oldest mail. */
if (uidlist->next == NULL)
return 0;
t_array_init(&uids, 8);
for (; uidlist->next != NULL; uidlist = uidlist->next)
seq_range_array_add(&uids, uidlist->uid);
search_args = mail_search_build_init();
arg = mail_search_build_add(search_args, SEARCH_UIDSET);
arg->value.seqset = uids;
trans = mailbox_transaction_begin(box, 0);
search_ctx = mailbox_search_init(trans, search_args, NULL, 0, NULL);
mail_search_args_unref(&search_args);
while (mailbox_search_next(search_ctx, &mail))
mail_expunge(mail);
if (mailbox_search_deinit(&search_ctx) < 0) {
i_error("Searching mailbox '%s' failed: %s",
mailbox_get_vname(box),
mailbox_get_last_error(box, NULL));
doveadm_mail_failed_mailbox(_ctx, box);
ret = -1;
}
if (mailbox_transaction_commit(&trans) < 0) {
i_error("Committing mailbox '%s' transaction failed: %s",
mailbox_get_vname(box),
mailbox_get_last_error(box, NULL));
doveadm_mail_failed_mailbox(_ctx, box);
ret = -1;
}
return ret;
}
static int
cmd_save_to_mailbox(struct save_cmd_context *ctx, struct mailbox *box,
struct istream *input)
{
struct mail_storage *storage = mailbox_get_storage(box);
struct mailbox_transaction_context *trans;
struct mail_save_context *save_ctx;
ssize_t ret;
bool save_failed = FALSE;
if (mailbox_open(box) < 0) {
i_error("Failed to open mailbox %s: %s",
mailbox_get_vname(box), mailbox_get_last_error(box, NULL));
doveadm_mail_failed_storage(&ctx->ctx, storage);
return -1;
}
trans = mailbox_transaction_begin(box, MAILBOX_TRANSACTION_FLAG_EXTERNAL);
save_ctx = mailbox_save_alloc(trans);
if (mailbox_save_begin(&save_ctx, input) < 0) {
i_error("Saving failed: %s", mailbox_get_last_error(box, NULL));
doveadm_mail_failed_storage(&ctx->ctx, storage);
mailbox_transaction_rollback(&trans);
return -1;
}
while ((ret = i_stream_read(input)) > 0 || ret == -2) {
if (mailbox_save_continue(save_ctx) < 0) {
save_failed = TRUE;
ret = -1;
break;
}
}
i_assert(ret == -1);
if (input->stream_errno != 0) {
i_error("read(msg input) failed: %s", i_stream_get_error(input));
doveadm_mail_failed_error(&ctx->ctx, MAIL_ERROR_TEMP);
} else if (save_failed) {
i_error("Saving failed: %s", mailbox_get_last_error(box, NULL));
doveadm_mail_failed_storage(&ctx->ctx, storage);
} else if (mailbox_save_finish(&save_ctx) < 0) {
i_error("Saving failed: %s",
mailbox_get_last_error(box, NULL));
doveadm_mail_failed_storage(&ctx->ctx, storage);
} else if (mailbox_transaction_commit(&trans) < 0) {
i_error("Save transaction commit failed: %s",
mailbox_get_last_error(box, NULL));
doveadm_mail_failed_storage(&ctx->ctx, storage);
} else {
ret = 0;
}
if (save_ctx != NULL)
mailbox_save_cancel(&save_ctx);
if (trans != NULL)
mailbox_transaction_rollback(&trans);
i_assert(input->eof);
return ret < 0 ? -1 : 0;
}
int mail_set_attachment_keywords(struct mail *mail)
{
int ret;
const struct mail_storage_settings *mail_set =
mail_storage_get_settings(mailbox_get_storage(mail->box));
const char *const keyword_has_attachment[] = {
MAIL_KEYWORD_HAS_ATTACHMENT,
NULL,
};
const char *const keyword_has_no_attachment[] = {
MAIL_KEYWORD_HAS_NO_ATTACHMENT,
NULL
};
struct message_part_attachment_settings set = {
.content_type_filter =
mail_set->parsed_mail_attachment_content_type_filter,
.exclude_inlined =
mail_set->parsed_mail_attachment_exclude_inlined,
};
struct mail_keywords *kw_has = NULL, *kw_has_not = NULL;
/* walk all parts and see if there is an attachment */
struct message_part *parts;
if (mail_get_parts(mail, &parts) < 0) {
mail_set_critical(mail, "Failed to add attachment keywords: "
"mail_get_parts() failed: %s",
mail_storage_get_last_internal_error(mail->box->storage, NULL));
ret = -1;
} else if (parts->data == NULL &&
mail_parse_parts(mail, &parts) < 0) {
ret = -1;
} else if (mailbox_keywords_create(mail->box, keyword_has_attachment, &kw_has) < 0 ||
mailbox_keywords_create(mail->box, keyword_has_no_attachment, &kw_has_not) < 0) {
mail_set_critical(mail, "Failed to add attachment keywords: "
"mailbox_keywords_create(%s) failed: %s",
mailbox_get_vname(mail->box),
mail_storage_get_last_internal_error(mail->box->storage, NULL));
ret = -1;
} else {
bool has_attachment = mail_message_has_attachment(parts, &set);
/* make sure only one of the keywords gets set */
mail_update_keywords(mail, MODIFY_REMOVE, has_attachment ? kw_has_not : kw_has);
mail_update_keywords(mail, MODIFY_ADD, has_attachment ? kw_has : kw_has_not);
ret = has_attachment ? 1 : 0;
}
if (kw_has != NULL)
mailbox_keywords_unref(&kw_has);
if (kw_has_not != NULL)
mailbox_keywords_unref(&kw_has_not);
return ret;
}
static void copy_update_trashed(struct client *client, struct mailbox *box,
unsigned int count)
{
const struct mailbox_settings *set;
set = mailbox_settings_find(mailbox_get_namespace(box),
mailbox_get_vname(box));
if (set != NULL && set->special_use[0] != '\0' &&
str_array_icase_find(t_strsplit_spaces(set->special_use, " "),
"\\Trash"))
client->trashed_count += count;
}
static void fetch_read_error(struct imap_fetch_context *ctx)
{
struct imap_fetch_state *state = &ctx->state;
errno = state->cur_input->stream_errno;
mail_storage_set_critical(state->cur_mail->box->storage,
"read(%s) failed: %s (FETCH %s for mailbox %s UID %u)",
i_stream_get_name(state->cur_input),
i_stream_get_error(state->cur_input),
state->cur_human_name,
mailbox_get_vname(state->cur_mail->box), state->cur_mail->uid);
}
static void dsync_brain_sync_half_finished(struct dsync_brain *brain)
{
struct dsync_mailbox_state state;
const char *errstr;
enum mail_error error;
if (brain->box_recv_state < DSYNC_BOX_STATE_RECV_LAST_COMMON ||
brain->box_send_state < DSYNC_BOX_STATE_RECV_LAST_COMMON)
return;
/* finished with this mailbox */
if (brain->box_exporter != NULL) {
if (dsync_mailbox_export_deinit(&brain->box_exporter,
&errstr, &error) < 0) {
i_error("Exporting mailbox %s failed: %s",
mailbox_get_vname(brain->box), errstr);
brain->mail_error = error;
brain->failed = TRUE;
return;
}
}
memset(&state, 0, sizeof(state));
memcpy(state.mailbox_guid, brain->local_dsync_box.mailbox_guid,
sizeof(state.mailbox_guid));
state.last_uidvalidity = brain->local_dsync_box.uid_validity;
if (brain->box_importer == NULL) {
/* this mailbox didn't exist on remote */
state.last_common_uid = brain->local_dsync_box.uid_next-1;
state.last_common_modseq =
brain->local_dsync_box.highest_modseq;
state.last_common_pvt_modseq =
brain->local_dsync_box.highest_pvt_modseq;
state.last_messages_count =
brain->local_dsync_box.messages_count;
} else {
if (dsync_mailbox_import_deinit(&brain->box_importer, TRUE,
&state.last_common_uid,
&state.last_common_modseq,
&state.last_common_pvt_modseq,
&state.last_messages_count,
&state.changes_during_sync,
&brain->mail_error) < 0) {
brain->failed = TRUE;
return;
}
if (state.changes_during_sync)
brain->changes_during_sync = TRUE;
}
brain->mailbox_state = state;
dsync_ibc_send_mailbox_state(brain->ibc, &state);
}
static void push_notification_event_mailboxrename_event(
struct push_notification_txn *ptxn,
struct push_notification_event_config *ec,
struct push_notification_txn_mbox *mbox,
struct mailbox *old)
{
struct push_notification_event_mailboxrename_data *data;
data = p_new(ptxn->pool,
struct push_notification_event_mailboxrename_data, 1);
data->old_mbox = mailbox_get_vname(old);
push_notification_txn_mbox_set_eventdata(ptxn, mbox, ec, data);
}
static int dsync_brain_export_deinit(struct dsync_brain *brain)
{
const char *errstr;
enum mail_error error;
if (dsync_mailbox_export_deinit(&brain->box_exporter,
&errstr, &error) < 0) {
i_error("Exporting mailbox %s failed: %s",
mailbox_get_vname(brain->box), errstr);
brain->mail_error = error;
brain->failed = TRUE;
return -1;
}
return 0;
}
static int fetch_stream_continue(struct imap_fetch_context *ctx)
{
struct imap_fetch_state *state = &ctx->state;
const char *disconnect_reason;
off_t ret;
o_stream_set_max_buffer_size(ctx->client->output, 0);
ret = o_stream_send_istream(ctx->client->output, state->cur_input);
o_stream_set_max_buffer_size(ctx->client->output, (size_t)-1);
if (ret > 0) {
state->cur_offset += ret;
if (ctx->state.cur_stats_sizep != NULL)
*ctx->state.cur_stats_sizep += ret;
}
if (state->cur_offset != state->cur_size) {
/* unfinished */
if (state->cur_input->stream_errno != 0) {
fetch_read_error(ctx, &disconnect_reason);
client_disconnect(ctx->client, disconnect_reason);
return -1;
}
if (!i_stream_have_bytes_left(state->cur_input)) {
/* Input stream gave less data than expected */
i_error("read(%s): FETCH %s for mailbox %s UID %u "
"got too little data: "
"%"PRIuUOFF_T" vs %"PRIuUOFF_T,
i_stream_get_name(state->cur_input),
state->cur_human_name,
mailbox_get_vname(state->cur_mail->box),
state->cur_mail->uid,
state->cur_offset, state->cur_size);
mail_set_cache_corrupted(state->cur_mail,
state->cur_size_field);
client_disconnect(ctx->client, "FETCH failed");
return -1;
}
if (ret < 0) {
/* client probably disconnected */
return -1;
}
o_stream_set_flush_pending(ctx->client->output, TRUE);
return 0;
}
return 1;
}
static void dbox_mail_set_expunged(struct dbox_mail *mail, uint32_t map_uid)
{
struct mail *_mail = &mail->imail.mail.mail;
struct mdbox_mailbox *mbox = (struct mdbox_mailbox *)_mail->box;
(void)mail_index_refresh(_mail->box->index);
if (mail_index_is_expunged(_mail->transaction->view, _mail->seq)) {
mail_set_expunged(_mail);
return;
}
mdbox_map_set_corrupted(mbox->storage->map,
"Unexpectedly lost %s uid=%u map_uid=%u",
mailbox_get_vname(_mail->box),
_mail->uid, map_uid);
}
static int dsync_box_get(struct mailbox *box, struct dsync_mailbox *dsync_box_r)
{
const enum mailbox_status_items status_items =
STATUS_UIDVALIDITY | STATUS_UIDNEXT | STATUS_MESSAGES |
STATUS_FIRST_RECENT_UID | STATUS_HIGHESTMODSEQ |
STATUS_HIGHESTPVTMODSEQ;
const enum mailbox_metadata_items metadata_items =
MAILBOX_METADATA_CACHE_FIELDS | MAILBOX_METADATA_GUID;
struct mailbox_status status;
struct mailbox_metadata metadata;
const char *errstr;
enum mail_error error;
/* get metadata first, since it may autocreate the mailbox */
if (mailbox_get_metadata(box, metadata_items, &metadata) < 0 ||
mailbox_get_status(box, status_items, &status) < 0) {
errstr = mailbox_get_last_error(box, &error);
if (error == MAIL_ERROR_NOTFOUND ||
error == MAIL_ERROR_NOTPOSSIBLE) {
/* Mailbox isn't selectable, try the next one. We
should have already caught \Noselect mailboxes, but
check them anyway here. The NOTPOSSIBLE check is
mainly for invalid mbox files. */
return 0;
}
i_error("Failed to access mailbox %s: %s",
mailbox_get_vname(box), errstr);
return -1;
}
i_assert(status.uidvalidity != 0 || status.messages == 0);
memset(dsync_box_r, 0, sizeof(*dsync_box_r));
memcpy(dsync_box_r->mailbox_guid, metadata.guid,
sizeof(dsync_box_r->mailbox_guid));
dsync_box_r->uid_validity = status.uidvalidity;
dsync_box_r->uid_next = status.uidnext;
dsync_box_r->messages_count = status.messages;
dsync_box_r->first_recent_uid = status.first_recent_uid;
dsync_box_r->highest_modseq = status.highest_modseq;
dsync_box_r->highest_pvt_modseq = status.highest_pvt_modseq;
dsync_box_r->cache_fields = *metadata.cache_fields;
dsync_box_r->have_guids = status.have_guids;
dsync_box_r->have_save_guids = status.have_save_guids;
dsync_box_r->have_only_guid128 = status.have_only_guid128;
return 1;
}
static void fetch_read_error(struct imap_fetch_context *ctx,
const char **disconnect_reason_r)
{
struct imap_fetch_state *state = &ctx->state;
if (state->cur_input->stream_errno == ENOENT) {
if (state->cur_mail->expunged) {
*disconnect_reason_r = "Mail expunged while it was being FETCHed";
return;
}
}
mail_storage_set_critical(state->cur_mail->box->storage,
"read(%s) failed: %s (FETCH %s for mailbox %s UID %u)",
i_stream_get_name(state->cur_input),
i_stream_get_error(state->cur_input),
state->cur_human_name,
mailbox_get_vname(state->cur_mail->box), state->cur_mail->uid);
*disconnect_reason_r = "FETCH read() failed";
}
static int
cmd_mailbox_metadata_list_run_iter(struct metadata_cmd_context *ctx,
struct mailbox *box,
enum mail_attribute_type type)
{
struct mailbox_attribute_iter *iter;
const char *key;
iter = mailbox_attribute_iter_init(box, type, ctx->key);
while ((key = mailbox_attribute_iter_next(iter)) != NULL)
doveadm_print(key);
if (mailbox_attribute_iter_deinit(&iter) < 0) {
i_error("Mailbox %s: Failed to iterate mailbox attributes: %s",
mailbox_get_vname(box),
mailbox_get_last_error(box, NULL));
return -1;
}
return 0;
}
void dsync_brain_mailbox_update_pre(struct dsync_brain *brain,
struct mailbox *box,
const struct dsync_mailbox *local_box,
const struct dsync_mailbox *remote_box)
{
struct mailbox_update update;
const struct dsync_mailbox_state *state;
memset(&update, 0, sizeof(update));
if (local_box->uid_validity != remote_box->uid_validity) {
/* Keep the UIDVALIDITY for the mailbox that has more
messages. If they equal, use the higher UIDVALIDITY. */
if (remote_box->messages_count > local_box->messages_count ||
(remote_box->messages_count == local_box->messages_count &&
remote_box->uid_validity > local_box->uid_validity))
update.uid_validity = remote_box->uid_validity;
state = dsync_mailbox_state_find(brain, local_box->mailbox_guid);
if (state != NULL && state->last_common_uid > 0) {
/* we can't continue syncing this mailbox in this
session, because the other side already started
sending mailbox changes, but not for all mails. */
dsync_mailbox_state_remove(brain, local_box->mailbox_guid);
// FIXME: handle this properly
}
}
dsync_cache_fields_update(local_box, remote_box, &update);
if (update.uid_validity == 0 &&
update.cache_updates == NULL) {
/* no changes */
return;
}
if (mailbox_update(box, &update) < 0) {
i_error("Couldn't update mailbox %s metadata: %s",
mailbox_get_vname(box),
mailbox_get_last_error(box, NULL));
brain->failed = TRUE;
}
}
int dsync_brain_sync_mailbox_open(struct dsync_brain *brain,
const struct dsync_mailbox *remote_dsync_box)
{
enum dsync_mailbox_exporter_flags exporter_flags = 0;
uint32_t last_common_uid, highest_wanted_uid;
uint64_t last_common_modseq, last_common_pvt_modseq;
i_assert(brain->log_scan == NULL);
i_assert(brain->box_exporter == NULL);
last_common_uid = brain->mailbox_state.last_common_uid;
last_common_modseq = brain->mailbox_state.last_common_modseq;
last_common_pvt_modseq = brain->mailbox_state.last_common_pvt_modseq;
highest_wanted_uid = last_common_uid == 0 ?
(uint32_t)-1 : last_common_uid;
if (dsync_transaction_log_scan_init(brain->box->view,
brain->box->view_pvt,
highest_wanted_uid,
last_common_modseq,
last_common_pvt_modseq,
&brain->log_scan) < 0) {
i_error("Failed to read transaction log for mailbox %s",
mailbox_get_vname(brain->box));
brain->failed = TRUE;
return -1;
}
if (!brain->mail_requests)
exporter_flags |= DSYNC_MAILBOX_EXPORTER_FLAG_AUTO_EXPORT_MAILS;
if (remote_dsync_box->have_save_guids &&
(brain->local_dsync_box.have_save_guids ||
(brain->backup_send && brain->local_dsync_box.have_guids)))
exporter_flags |= DSYNC_MAILBOX_EXPORTER_FLAG_MAILS_HAVE_GUIDS;
brain->box_exporter = brain->backup_recv ? NULL :
dsync_mailbox_export_init(brain->box, brain->log_scan,
last_common_uid,
exporter_flags);
dsync_brain_sync_mailbox_init_remote(brain, remote_dsync_box);
return 0;
}
static int fetch_stream_send_direct(struct imap_fetch_context *ctx)
{
off_t ret;
o_stream_set_max_buffer_size(ctx->client->output, 0);
ret = o_stream_send_istream(ctx->client->output, ctx->cur_input);
o_stream_set_max_buffer_size(ctx->client->output, (size_t)-1);
if (ret < 0)
return -1;
ctx->cur_offset += ret;
if (ctx->cur_append_eoh && ctx->cur_offset + 2 == ctx->cur_size) {
/* Netscape missing EOH workaround. */
if (o_stream_send(ctx->client->output, "\r\n", 2) < 0)
return -1;
ctx->cur_offset += 2;
ctx->cur_append_eoh = FALSE;
}
if (ctx->cur_offset != ctx->cur_size) {
/* unfinished */
if (!i_stream_have_bytes_left(ctx->cur_input)) {
/* Input stream gave less data than expected */
i_error("FETCH %s for mailbox %s UID %u "
"got too little data (copying): "
"%"PRIuUOFF_T" vs %"PRIuUOFF_T,
ctx->cur_name, mailbox_get_vname(ctx->mail->box),
ctx->mail->uid, ctx->cur_offset, ctx->cur_size);
mail_set_cache_corrupted(ctx->mail,
ctx->cur_size_field);
client_disconnect(ctx->client, "FETCH failed");
return -1;
}
o_stream_set_flush_pending(ctx->client->output, TRUE);
return 0;
}
return 1;
}
int imap_msgpart_url_parse(struct mail_user *user, struct mailbox *selected_box,
const char *urlstr, struct imap_msgpart_url **mpurl_r,
const char **error_r)
{
struct mailbox_status box_status;
struct imap_url base_url, *url;
const char *error;
/* build base url */
memset(&base_url, 0, sizeof(base_url));
if (selected_box != NULL) {
mailbox_get_open_status(selected_box, STATUS_UIDVALIDITY,
&box_status);
base_url.mailbox = mailbox_get_vname(selected_box);
base_url.uidvalidity = box_status.uidvalidity;
}
/* parse url */
if (imap_url_parse(urlstr, &base_url,
IMAP_URL_PARSE_REQUIRE_RELATIVE, &url, &error) < 0) {
*error_r = t_strconcat("Invalid IMAP URL: ", error, NULL);
return 0;
}
if (url->mailbox == NULL) {
*error_r = "Mailbox-relative IMAP URL, but no mailbox selected";
return 0;
}
if (url->uid == 0 || url->search_program != NULL) {
*error_r = "Invalid messagepart IMAP URL";
return 0;
}
if (imap_msgpart_url_create(user, url, mpurl_r, error_r) < 0)
return -1;
(*mpurl_r)->selected_box = selected_box;
return 1;
}
status_output(struct status_cmd_context *ctx, struct mailbox *box,
const struct mailbox_status *status,
const struct mailbox_metadata *metadata)
{
if (box != NULL)
doveadm_print(mailbox_get_vname(box));
if ((ctx->status_items & STATUS_MESSAGES) != 0)
doveadm_print_num(status->messages);
if ((ctx->status_items & STATUS_RECENT) != 0)
doveadm_print_num(status->recent);
if ((ctx->status_items & STATUS_UIDNEXT) != 0)
doveadm_print_num(status->uidnext);
if ((ctx->status_items & STATUS_UIDVALIDITY) != 0)
doveadm_print_num(status->uidvalidity);
if ((ctx->status_items & STATUS_UNSEEN) != 0)
doveadm_print_num(status->unseen);
if ((ctx->status_items & STATUS_HIGHESTMODSEQ) != 0)
doveadm_print_num(status->highest_modseq);
if ((ctx->metadata_items & MAILBOX_METADATA_VIRTUAL_SIZE) != 0)
doveadm_print_num(metadata->virtual_size);
if ((ctx->metadata_items & MAILBOX_METADATA_GUID) != 0)
doveadm_print(guid_128_to_string(metadata->guid));
}
bool dsync_brain_slave_recv_mailbox(struct dsync_brain *brain)
{
const struct dsync_mailbox *dsync_box;
struct dsync_mailbox local_dsync_box;
struct mailbox *box;
int ret;
i_assert(!brain->master_brain);
i_assert(brain->box == NULL);
if ((ret = dsync_ibc_recv_mailbox(brain->ibc, &dsync_box)) == 0)
return FALSE;
if (ret < 0) {
brain->state = DSYNC_STATE_DONE;
return TRUE;
}
if (dsync_brain_mailbox_alloc(brain, dsync_box->mailbox_guid, &box) < 0) {
i_assert(brain->failed);
return TRUE;
}
if (box == NULL) {
/* mailbox was probably deleted/renamed during sync */
//FIXME: verify this from log, and if not log an error.
brain->changes_during_sync = TRUE;
dsync_brain_slave_send_mailbox_lost(brain, dsync_box);
return TRUE;
}
if (mailbox_sync(box, MAILBOX_SYNC_FLAG_FULL_READ) < 0) {
i_error("Can't sync mailbox %s: %s",
mailbox_get_vname(box),
mailbox_get_last_error(box, NULL));
mailbox_free(&box);
brain->failed = TRUE;
return TRUE;
}
if ((ret = dsync_box_get(box, &local_dsync_box)) <= 0) {
mailbox_free(&box);
if (ret < 0) {
brain->failed = TRUE;
return TRUE;
}
/* another process just deleted this mailbox? */
dsync_brain_slave_send_mailbox_lost(brain, dsync_box);
return TRUE;
}
i_assert(local_dsync_box.uid_validity != 0);
i_assert(memcmp(dsync_box->mailbox_guid, local_dsync_box.mailbox_guid,
sizeof(dsync_box->mailbox_guid)) == 0);
dsync_ibc_send_mailbox(brain->ibc, &local_dsync_box);
dsync_brain_mailbox_update_pre(brain, box, &local_dsync_box, dsync_box);
if (brain->sync_type == DSYNC_BRAIN_SYNC_TYPE_CHANGED &&
!dsync_boxes_need_sync(&local_dsync_box, dsync_box)) {
/* no fields appear to have changed, skip this mailbox */
mailbox_free(&box);
return TRUE;
}
/* start export/import */
dsync_brain_sync_mailbox_init(brain, box, &local_dsync_box, FALSE);
if (dsync_brain_sync_mailbox_open(brain, dsync_box) < 0)
return TRUE;
brain->state = DSYNC_STATE_SYNC_MAILS;
return TRUE;
}
static int
dsync_brain_try_next_mailbox(struct dsync_brain *brain, struct mailbox **box_r,
struct dsync_mailbox *dsync_box_r)
{
enum mailbox_flags flags = 0;
struct dsync_mailbox dsync_box;
struct mailbox *box;
struct dsync_mailbox_node *node;
const char *vname = NULL;
bool synced = FALSE;
int ret;
*box_r = NULL;
while (dsync_mailbox_tree_iter_next(brain->local_tree_iter, &vname, &node)) {
if (node->existence == DSYNC_MAILBOX_NODE_EXISTS &&
!guid_128_is_empty(node->mailbox_guid))
break;
vname = NULL;
}
if (vname == NULL) {
/* no more mailboxes */
dsync_mailbox_tree_iter_deinit(&brain->local_tree_iter);
return -1;
}
if (brain->backup_send) {
/* make sure mailbox isn't modified */
flags |= MAILBOX_FLAG_READONLY;
}
box = mailbox_alloc(node->ns->list, vname, flags);
for (;;) {
if ((ret = dsync_box_get(box, &dsync_box)) <= 0) {
if (ret < 0)
brain->failed = TRUE;
mailbox_free(&box);
return ret;
}
/* if mailbox's last_common_* state equals the current state,
we can skip the mailbox */
if (!dsync_brain_has_mailbox_state_changed(brain, &dsync_box)) {
mailbox_free(&box);
return 0;
}
if (synced) {
/* ok, the mailbox really changed */
break;
}
/* mailbox appears to have changed. do a full sync here and get the
state again */
if (mailbox_sync(box, MAILBOX_SYNC_FLAG_FULL_READ) < 0) {
i_error("Can't sync mailbox %s: %s",
mailbox_get_vname(box),
mailbox_get_last_error(box, NULL));
brain->failed = TRUE;
mailbox_free(&box);
return -1;
}
synced = TRUE;
}
*box_r = box;
*dsync_box_r = dsync_box;
return 1;
}
bool dsync_brain_slave_recv_mailbox(struct dsync_brain *brain)
{
const struct dsync_mailbox *dsync_box;
struct dsync_mailbox local_dsync_box;
struct mailbox *box;
const char *error;
int ret;
i_assert(!brain->master_brain);
i_assert(brain->box == NULL);
if ((ret = dsync_ibc_recv_mailbox(brain->ibc, &dsync_box)) == 0)
return FALSE;
if (ret < 0) {
brain->state = DSYNC_STATE_DONE;
return TRUE;
}
if (dsync_brain_mailbox_alloc(brain, dsync_box->mailbox_guid,
&box, &error) < 0) {
i_error("Couldn't allocate mailbox GUID %s: %s",
guid_128_to_string(dsync_box->mailbox_guid), error);
i_assert(brain->failed);
return TRUE;
}
if (box == NULL) {
/* mailbox was probably deleted/renamed during sync */
if (brain->backup_send && brain->no_backup_overwrite) {
if (brain->debug) {
i_debug("brain %c: Ignore nonexistent "
"mailbox GUID %s with -1 sync",
brain->master_brain ? 'M' : 'S',
guid_128_to_string(dsync_box->mailbox_guid));
}
dsync_brain_slave_send_mailbox_lost(brain, dsync_box);
return TRUE;
}
//FIXME: verify this from log, and if not log an error.
if (brain->debug) {
i_debug("brain %c: Change during sync: "
"Mailbox GUID %s was lost",
brain->master_brain ? 'M' : 'S',
guid_128_to_string(dsync_box->mailbox_guid));
}
brain->changes_during_sync = TRUE;
dsync_brain_slave_send_mailbox_lost(brain, dsync_box);
return TRUE;
}
if (mailbox_sync(box, MAILBOX_SYNC_FLAG_FULL_READ) < 0) {
i_error("Can't sync mailbox %s: %s",
mailbox_get_vname(box),
mailbox_get_last_error(box, NULL));
mailbox_free(&box);
brain->failed = TRUE;
return TRUE;
}
if ((ret = dsync_box_get(box, &local_dsync_box)) <= 0) {
mailbox_free(&box);
if (ret < 0) {
brain->failed = TRUE;
return TRUE;
}
/* another process just deleted this mailbox? */
dsync_brain_slave_send_mailbox_lost(brain, dsync_box);
return TRUE;
}
i_assert(local_dsync_box.uid_validity != 0);
i_assert(memcmp(dsync_box->mailbox_guid, local_dsync_box.mailbox_guid,
sizeof(dsync_box->mailbox_guid)) == 0);
dsync_ibc_send_mailbox(brain->ibc, &local_dsync_box);
dsync_brain_mailbox_update_pre(brain, box, &local_dsync_box, dsync_box);
if (!dsync_boxes_need_sync(brain, &local_dsync_box, dsync_box)) {
/* no fields appear to have changed, skip this mailbox */
mailbox_free(&box);
return TRUE;
}
/* start export/import */
dsync_brain_sync_mailbox_init(brain, box, &local_dsync_box, FALSE);
if (dsync_brain_sync_mailbox_open(brain, dsync_box) < 0)
return TRUE;
brain->state = DSYNC_STATE_SYNC_MAILS;
return TRUE;
}
static int
mail_crypt_mail_save_begin(struct mail_save_context *ctx,
struct istream *input)
{
const char *pubid;
struct mailbox *box = ctx->transaction->box;
struct mail_crypt_mailbox *mbox = MAIL_CRYPT_CONTEXT(box);
struct mail_crypt_user *muser =
MAIL_CRYPT_USER_CONTEXT(box->storage->user);
i_assert(muser != NULL);
enum io_stream_encrypt_flags enc_flags;
if (muser->save_version == 1) {
enc_flags = IO_STREAM_ENC_VERSION_1;
} else if (muser->save_version == 2) {
enc_flags = IO_STREAM_ENC_INTEGRITY_AEAD;
} else {
i_assert(muser->save_version == 0);
i_panic("mail_crypt_mail_save_begin not supposed to be called"
"when mail_crypt_save_version is 0");
}
if (mbox->module_ctx.super.save_begin(ctx, input) < 0)
return -1;
struct dcrypt_public_key *pub_key;
if (muser->global_keys.public_key != NULL)
pub_key = muser->global_keys.public_key;
else if (mbox->pub_key != NULL)
pub_key = mbox->pub_key;
else {
const char *error;
int ret;
if ((ret = mail_crypt_box_get_public_key(box, &pub_key,
&error)) <= 0)
{
struct dcrypt_keypair pair;
if (ret < 0) {
mail_storage_set_error(box->storage,
MAIL_ERROR_PARAMS,
t_strdup_printf("get_public_key(%s) failed: %s",
mailbox_get_vname(box),
error));
return ret;
}
if (muser->save_version < 2) {
mail_storage_set_error(box->storage,
MAIL_ERROR_PARAMS,
t_strdup_printf("generate_keypair(%s) failed: "
"unsupported save_version=%d",
mailbox_get_vname(box),
muser->save_version));
return -1;
}
if (mail_crypt_box_generate_keypair(box, &pair, NULL,
&pubid, &error) < 0) {
mail_storage_set_error(box->storage,
MAIL_ERROR_PARAMS,
t_strdup_printf("generate_keypair(%s) failed: %s",
mailbox_get_vname(box),
error));
return -1;
}
pub_key = pair.pub;
dcrypt_key_unref_private(&pair.priv);
}
mbox->pub_key = pub_key;
}
/* encryption is the outermost layer (zlib etc. are inside) */
struct ostream *output = o_stream_create_encrypt(ctx->data.output,
MAIL_CRYPT_ENC_ALGORITHM, pub_key, enc_flags);
o_stream_unref(&ctx->data.output);
ctx->data.output = output;
o_stream_cork(ctx->data.output);
return 0;
}