static int
mdbox_map_get_seq(struct mdbox_map *map, uint32_t map_uid, uint32_t *seq_r)
{
if (!mail_index_lookup_seq(map->view, map_uid, seq_r)) {
/* not found - try again after a refresh */
if (mdbox_map_refresh(map) < 0)
return -1;
if (!mail_index_lookup_seq(map->view, map_uid, seq_r))
return 0;
}
return 1;
}
static int
maildir_mail_get_fname(struct maildir_mailbox *mbox, struct mail *mail,
const char **fname_r)
{
enum maildir_uidlist_rec_flag flags;
struct mail_index_view *view;
uint32_t seq;
bool exists;
int ret;
ret = maildir_sync_lookup(mbox, mail->uid, &flags, fname_r);
if (ret != 0)
return ret;
/* file exists in index file, but not in dovecot-uidlist anymore. */
mail_set_expunged(mail);
/* one reason this could happen is if we delayed opening
dovecot-uidlist and we're trying to open a mail that got recently
expunged. Let's test this theory first: */
mail_index_refresh(mbox->box.index);
view = mail_index_view_open(mbox->box.index);
exists = mail_index_lookup_seq(view, mail->uid, &seq);
mail_index_view_close(&view);
if (exists) {
/* the message still exists in index. this means there's some
kind of a desync, which doesn't get fixed if cur/ mtime is
the same as in index. fix this by forcing a resync. */
(void)maildir_storage_sync_force(mbox, mail->uid);
}
return 0;
}
static void
index_mailbox_expunge_unseen_recent(struct index_mailbox_sync_context *ctx)
{
struct index_mailbox_context *ibox =
INDEX_STORAGE_CONTEXT(ctx->ctx.box);
struct mail_index_view *view = ctx->ctx.box->view;
const struct mail_index_header *hdr;
uint32_t seq, start_uid, uid;
if (!array_is_created(&ibox->recent_flags))
return;
/* expunges array contained expunges for the messages that were already
visible in this view, but append+expunge would be invisible.
recent_flags may however contain the append UID, so we'll have to
remove it separately */
hdr = mail_index_get_header(view);
if (ctx->messages_count == 0)
uid = 0;
else if (ctx->messages_count <= hdr->messages_count)
mail_index_lookup_uid(view, ctx->messages_count, &uid);
else {
i_assert(mail_index_view_is_inconsistent(view));
return;
}
for (seq = ctx->messages_count + 1; seq <= hdr->messages_count; seq++) {
start_uid = uid;
mail_index_lookup_uid(view, seq, &uid);
if (start_uid + 1 > uid - 1)
continue;
ibox->recent_flags_count -=
seq_range_array_remove_range(&ibox->recent_flags,
start_uid + 1, uid - 1);
}
if (uid + 1 < hdr->next_uid) {
ibox->recent_flags_count -=
seq_range_array_remove_range(&ibox->recent_flags,
uid + 1,
hdr->next_uid - 1);
}
#ifdef DEBUG
if (!mail_index_view_is_inconsistent(view)) {
const struct seq_range *range;
unsigned int i, count;
range = array_get(&ibox->recent_flags, &count);
for (i = 0; i < count; i++) {
for (uid = range[i].seq1; uid <= range[i].seq2; uid++) {
if (uid >= hdr->next_uid)
break;
(void)mail_index_lookup_seq(view, uid, &seq);
i_assert(seq != 0);
}
}
}
#endif
}
static uint32_t index_storage_find_first_pvt_unseen_seq(struct mailbox *box)
{
const struct mail_index_header *pvt_hdr;
const struct mail_index_record *pvt_rec;
uint32_t pvt_seq, pvt_count, shared_seq, seq2;
pvt_count = mail_index_view_get_messages_count(box->view_pvt);
mail_index_lookup_first(box->view_pvt, 0, MAIL_SEEN, &pvt_seq);
if (pvt_seq == 0)
pvt_seq = pvt_count+1;
for (; pvt_seq <= pvt_count; pvt_seq++) {
pvt_rec = mail_index_lookup(box->view_pvt, pvt_seq);
if ((pvt_rec->flags & MAIL_SEEN) == 0 &&
mail_index_lookup_seq(box->view, pvt_rec->uid, &shared_seq))
return shared_seq;
}
/* if shared index has any messages that don't exist in private index,
the first of them is the first unseen message */
pvt_hdr = mail_index_get_header(box->view_pvt);
if (mail_index_lookup_seq_range(box->view,
pvt_hdr->next_uid, (uint32_t)-1,
&shared_seq, &seq2))
return shared_seq;
return 0;
}
void dbox_sync_rebuild_index_metadata(struct dbox_sync_rebuild_context *ctx,
uint32_t new_seq, uint32_t uid)
{
uint32_t old_seq;
if (mail_index_lookup_seq(ctx->view, uid, &old_seq)) {
/* the message exists in the old index.
copy the metadata from it. */
dbox_sync_index_copy_from_old(ctx, ctx->view, old_seq, new_seq);
} else if (ctx->backup_view != NULL &&
mail_index_lookup_seq(ctx->backup_view, uid, &old_seq)) {
/* copy the metadata from backup index. */
dbox_sync_index_copy_from_old(ctx, ctx->backup_view,
old_seq, new_seq);
}
}
static bool imapc_mail_is_expunged(struct mail *_mail)
{
struct imapc_mailbox *mbox = (struct imapc_mailbox *)_mail->box;
struct imapc_msgmap *msgmap;
uint32_t lseq, rseq;
if (!mbox->initial_sync_done) {
/* unknown at this point */
return FALSE;
}
if (mbox->sync_view != NULL) {
/* check if another session has already expunged it */
if (!mail_index_lookup_seq(mbox->sync_view, _mail->uid, &lseq))
return TRUE;
}
/* check if we've received EXPUNGE for it */
msgmap = imapc_client_mailbox_get_msgmap(mbox->client_box);
if (!imapc_msgmap_uid_to_rseq(msgmap, _mail->uid, &rseq))
return TRUE;
/* we may be running against a server that hasn't bothered sending
us an EXPUNGE. see if NOOP sends it. */
imapc_mailbox_noop(mbox);
if (!mbox->initial_sync_done) {
/* NOOP caused a reconnection and desync */
return FALSE;
}
return !imapc_msgmap_uid_to_rseq(msgmap, _mail->uid, &rseq);
}
static bool virtual_mail_set_uid(struct mail *mail, uint32_t uid)
{
uint32_t seq;
if (!mail_index_lookup_seq(mail->box->view, uid, &seq))
return FALSE;
virtual_mail_set_seq(mail, seq);
return TRUE;
}
bool index_mail_set_uid(struct mail *_mail, uint32_t uid)
{
struct index_mail *mail = (struct index_mail *)_mail;
uint32_t seq;
if (mail_index_lookup_seq(_mail->box->view, uid, &seq)) {
index_mail_set_seq(_mail, seq);
return TRUE;
} else {
index_mail_reset(mail);
mail->mail.mail.uid = uid;
mail_set_expunged(&mail->mail.mail);
return FALSE;
}
}
static const char *
maildir_filename_guess(struct maildir_mailbox *mbox, uint32_t uid,
const char *fname,
enum maildir_uidlist_rec_flag *uidlist_flags,
bool *have_flags_r)
{
struct mail_index_view *view = mbox->flags_view;
struct maildir_keywords_sync_ctx *kw_ctx;
enum mail_flags flags;
ARRAY_TYPE(keyword_indexes) keywords;
const char *p;
uint32_t seq;
if (view == NULL || !mail_index_lookup_seq(view, uid, &seq)) {
*have_flags_r = FALSE;
return fname;
}
t_array_init(&keywords, 32);
mail_index_lookup_view_flags(view, seq, &flags, &keywords);
if (array_count(&keywords) == 0) {
*have_flags_r = (flags & MAIL_FLAGS_NONRECENT) != 0;
fname = maildir_filename_flags_set(fname, flags);
} else {
*have_flags_r = TRUE;
kw_ctx = maildir_keywords_sync_init_readonly(mbox->keywords,
mbox->box.index);
fname = maildir_filename_flags_kw_set(kw_ctx, fname,
flags, &keywords);
maildir_keywords_sync_deinit(&kw_ctx);
}
if (*have_flags_r) {
/* don't even bother looking into new/ dir */
*uidlist_flags &= MAILDIR_UIDLIST_REC_FLAG_NEW_DIR;
} else if ((*uidlist_flags & MAILDIR_UIDLIST_REC_FLAG_MOVED) == 0 &&
((*uidlist_flags & MAILDIR_UIDLIST_REC_FLAG_NEW_DIR) != 0 ||
mailbox_recent_flags_have_uid(&mbox->box, uid))) {
/* probably in new/ dir, drop ":2," from fname */
*uidlist_flags |= MAILDIR_UIDLIST_REC_FLAG_NEW_DIR;
p = strrchr(fname, MAILDIR_INFO_SEP);
if (p != NULL)
fname = t_strdup_until(fname, p);
}
return fname;
}
static int
index_list_mailbox_open_unchanged_view(struct mailbox *box,
struct mail_index_view **view_r,
uint32_t *seq_r)
{
struct index_mailbox_list *ilist;
struct mail_index_view *view;
uint32_t uid, seq;
int ret;
ilist = INDEX_LIST_CONTEXT(box->list);
if (ilist == NULL) {
/* indexing disabled */
return 0;
}
ret = mailbox_list_index_lookup(ilist->list_sync_view, box->name, &uid);
if (ret <= 0)
return ret;
/* make sure we're synced */
if (index_list_update_mail_index(ilist, box) < 0)
return -1;
/* found from list index. lookup the mail index record for it */
view = mail_index_view_open(ilist->mail_index);
if (!mail_index_lookup_seq(view, uid, &seq)) {
mail_index_view_close(&view);
return 0;
}
T_BEGIN {
ret = box->v.list_index_has_changed(box, view, seq);
} T_END;
if (ret != 0) {
/* error / mailbox has changed. we'll need to sync it. */
mail_index_view_close(&view);
return ret < 0 ? -1 : 0;
}
*view_r = view;
*seq_r = seq;
return 1;
}
static int imapc_mailbox_commit_delayed_expunges(struct imapc_mailbox *mbox)
{
struct mail_index_view *view = imapc_mailbox_get_sync_view(mbox);
struct mail_index_transaction *trans;
struct seq_range_iter iter;
unsigned int n;
uint32_t lseq, uid;
int ret;
trans = mail_index_transaction_begin(view,
MAIL_INDEX_TRANSACTION_FLAG_EXTERNAL);
seq_range_array_iter_init(&iter, &mbox->delayed_expunged_uids); n = 0;
while (seq_range_array_iter_nth(&iter, n++, &uid)) {
if (mail_index_lookup_seq(view, uid, &lseq))
mail_index_expunge(trans, lseq);
}
array_clear(&mbox->delayed_expunged_uids);
ret = mail_index_transaction_commit(&trans);
if (ret < 0)
mailbox_set_index_error(&mbox->box);
return ret;
}
static int mbox_mail_seek(struct index_mail *mail)
{
struct mbox_transaction_context *t =
(struct mbox_transaction_context *)mail->mail.mail.transaction;
struct mail *_mail = &mail->mail.mail;
struct mbox_mailbox *mbox = (struct mbox_mailbox *)_mail->box;
enum mbox_sync_flags sync_flags = 0;
int ret, try;
bool deleted;
if (_mail->expunged || mbox->syncing)
return -1;
if (_mail->lookup_abort != MAIL_LOOKUP_ABORT_NEVER) {
mail_set_aborted(_mail);
return -1;
}
if (mbox->mbox_stream != NULL &&
istream_raw_mbox_is_corrupted(mbox->mbox_stream)) {
/* clear the corruption by forcing a full resync */
sync_flags |= MBOX_SYNC_UNDIRTY | MBOX_SYNC_FORCE_SYNC;
}
for (try = 0; try < 2; try++) {
if ((sync_flags & MBOX_SYNC_FORCE_SYNC) != 0) {
/* dirty offsets are broken. make sure we can sync. */
mbox_prepare_resync(_mail);
}
if (mbox->mbox_lock_type == F_UNLCK) {
i_assert(t->read_lock_id == 0);
sync_flags |= MBOX_SYNC_LOCK_READING;
if (mbox_sync(mbox, sync_flags) < 0)
return -1;
t->read_lock_id = mbox_get_cur_lock_id(mbox);
i_assert(t->read_lock_id != 0);
/* refresh index file after mbox has been locked to
make sure we get only up-to-date mbox offsets. */
if (mail_index_refresh(mbox->box.index) < 0) {
mailbox_set_index_error(&mbox->box);
return -1;
}
i_assert(mbox->mbox_lock_type != F_UNLCK);
} else if (t->read_lock_id == 0) {
/* file is already locked by another transaction, but
we must keep it locked for the entire transaction,
so increase the lock counter. */
if (mbox_lock(mbox, mbox->mbox_lock_type,
&t->read_lock_id) < 0)
i_unreached();
}
if (mbox_file_open_stream(mbox) < 0)
return -1;
ret = mbox_file_seek(mbox, _mail->transaction->view,
_mail->seq, &deleted);
if (ret > 0) {
/* success */
break;
}
if (ret < 0) {
if (deleted)
mail_set_expunged(_mail);
return -1;
}
/* we'll need to re-sync it completely */
sync_flags |= MBOX_SYNC_UNDIRTY | MBOX_SYNC_FORCE_SYNC;
}
if (ret == 0) {
mail_storage_set_critical(&mbox->storage->storage,
"Losing sync for mail uid=%u in mbox file %s",
_mail->uid, mailbox_get_path(&mbox->box));
}
return 0;
}
static int mbox_mail_get_received_date(struct mail *_mail, time_t *date_r)
{
struct index_mail *mail = (struct index_mail *)_mail;
struct index_mail_data *data = &mail->data;
struct mbox_mailbox *mbox = (struct mbox_mailbox *)_mail->box;
if (index_mail_get_received_date(_mail, date_r) == 0)
return 0;
if (mbox_mail_seek(mail) < 0)
return -1;
data->received_date =
istream_raw_mbox_get_received_time(mbox->mbox_stream);
if (data->received_date == (time_t)-1) {
/* it's broken and conflicts with our "not found"
return value. change it. */
data->received_date = 0;
}
*date_r = data->received_date;
return 0;
}
static int mbox_mail_get_save_date(struct mail *_mail, time_t *date_r)
{
struct index_mail *mail = (struct index_mail *)_mail;
struct index_mail_data *data = &mail->data;
if (index_mail_get_save_date(_mail, date_r) == 0)
return 0;
/* no way to know this. save the current time into cache and use
that from now on. this works only as long as the index files
are permanent */
data->save_date = ioloop_time;
*date_r = data->save_date;
return 0;
}
static int
mbox_mail_get_md5_header(struct index_mail *mail, const char **value_r)
{
struct mail *_mail = &mail->mail.mail;
static uint8_t empty_md5[16] =
{ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 };
struct mbox_mailbox *mbox = (struct mbox_mailbox *)_mail->box;
const void *ext_data;
if (mail->data.guid != NULL) {
*value_r = mail->data.guid;
return 1;
}
mail_index_lookup_ext(_mail->transaction->view, _mail->seq,
mbox->md5hdr_ext_idx, &ext_data, NULL);
if (ext_data != NULL && memcmp(ext_data, empty_md5, 16) != 0) {
mail->data.guid = p_strdup(mail->mail.data_pool,
binary_to_hex(ext_data, 16));
*value_r = mail->data.guid;
return 1;
} else if (mail_index_is_expunged(_mail->transaction->view, _mail->seq)) {
mail_set_expunged(_mail);
return -1;
} else {
return 0;
}
}
static int
mbox_mail_get_special(struct mail *_mail, enum mail_fetch_field field,
const char **value_r)
{
struct index_mail *mail = (struct index_mail *)_mail;
struct mbox_mailbox *mbox = (struct mbox_mailbox *)_mail->box;
uoff_t offset;
bool move_offset;
int ret;
switch (field) {
case MAIL_FETCH_FROM_ENVELOPE:
if (mbox_mail_seek(mail) < 0)
return -1;
*value_r = istream_raw_mbox_get_sender(mbox->mbox_stream);
return 0;
case MAIL_FETCH_GUID:
case MAIL_FETCH_HEADER_MD5:
if ((ret = mbox_mail_get_md5_header(mail, value_r)) != 0)
return ret < 0 ? -1 : 0;
/* i guess in theory the empty_md5 is valid and can happen,
but it's almost guaranteed that it means the MD5 sum is
missing. recalculate it. */
if (mbox->mbox_lock_type == F_UNLCK ||
mbox->mbox_stream == NULL) {
offset = 0;
move_offset = FALSE;
} else {
offset = istream_raw_mbox_get_start_offset(mbox->mbox_stream);
move_offset = TRUE;
}
mbox->mbox_save_md5 = TRUE;
if (mbox_sync(mbox, MBOX_SYNC_FORCE_SYNC |
MBOX_SYNC_READONLY) < 0)
return -1;
if (move_offset) {
if (istream_raw_mbox_seek(mbox->mbox_stream,
offset) < 0) {
i_error("mbox %s sync lost during MD5 syncing",
_mail->box->name);
return -1;
}
}
if ((ret = mbox_mail_get_md5_header(mail, value_r)) == 0) {
i_error("mbox %s resyncing didn't save header MD5 values",
_mail->box->name);
return -1;
}
return ret < 0 ? -1 : 0;
default:
break;
}
return index_mail_get_special(_mail, field, value_r);
}
static int
mbox_mail_get_next_offset(struct index_mail *mail, uoff_t *next_offset_r)
{
struct mbox_mailbox *mbox = (struct mbox_mailbox *)mail->mail.mail.box;
struct mail_index_view *view;
const struct mail_index_header *hdr;
uint32_t seq;
int trailer_size;
int ret = 1;
*next_offset_r = (uoff_t)-1;
hdr = mail_index_get_header(mail->mail.mail.transaction->view);
if (mail->mail.mail.seq > hdr->messages_count) {
/* we're appending a new message */
return 0;
}
/* We can't really trust trans_view. The next message may already be
expunged from it. Also hdr.messages_count may be incorrect there.
So refresh the index to get the latest changes and get the next
message's offset using a new view. */
i_assert(mbox->mbox_lock_type != F_UNLCK);
if (mbox_sync_header_refresh(mbox) < 0)
return -1;
view = mail_index_view_open(mail->mail.mail.box->index);
hdr = mail_index_get_header(view);
if (!mail_index_lookup_seq(view, mail->mail.mail.uid, &seq))
i_panic("Message unexpectedly expunged from index");
if (seq < hdr->messages_count) {
if (mbox_file_lookup_offset(mbox, view, seq + 1,
next_offset_r) <= 0)
ret = -1;
} else if (mail->mail.mail.box->input != NULL) {
/* opened the mailbox as input stream. we can't trust the
sync_size, since it's wrong with compressed mailboxes */
ret = 0;
} else {
/* last message, use the synced mbox size */
trailer_size =
mbox->storage->storage.set->mail_save_crlf ? 2 : 1;
*next_offset_r = mbox->mbox_hdr.sync_size - trailer_size;
}
mail_index_view_close(&view);
return ret;
}
static int mbox_mail_get_physical_size(struct mail *_mail, uoff_t *size_r)
{
struct index_mail *mail = (struct index_mail *)_mail;
struct index_mail_data *data = &mail->data;
struct mbox_mailbox *mbox = (struct mbox_mailbox *)_mail->box;
struct istream *input;
struct message_size hdr_size;
uoff_t old_offset, body_offset, body_size, next_offset;
if (index_mail_get_physical_size(_mail, size_r) == 0)
return 0;
/* we want to return the header size as seen by mail_get_stream(). */
old_offset = data->stream == NULL ? 0 : data->stream->v_offset;
if (mail_get_stream(_mail, &hdr_size, NULL, &input) < 0)
return -1;
/* our header size varies, so don't do any caching */
if (istream_raw_mbox_get_body_offset(mbox->mbox_stream, &body_offset) < 0) {
mail_storage_set_critical(_mail->box->storage,
"mbox %s: Couldn't get body offset for uid=%u",
mailbox_get_path(&mbox->box), mail->mail.mail.uid);
return -1;
}
/* use the next message's offset to avoid reading through the entire
message body to find out its size */
if (mbox_mail_get_next_offset(mail, &next_offset) > 0)
body_size = next_offset - body_offset;
else
body_size = (uoff_t)-1;
/* verify that the calculated body size is correct */
if (istream_raw_mbox_get_body_size(mbox->mbox_stream,
body_size, &body_size) < 0) {
mail_storage_set_critical(_mail->box->storage,
"mbox %s: Couldn't get body size for uid=%u",
mailbox_get_path(&mbox->box), mail->mail.mail.uid);
return -1;
}
data->physical_size = hdr_size.physical_size + body_size;
*size_r = data->physical_size;
i_stream_seek(input, old_offset);
return 0;
}
static int mbox_mail_init_stream(struct index_mail *mail)
{
struct mbox_mailbox *mbox = (struct mbox_mailbox *)mail->mail.mail.box;
struct istream *raw_stream;
uoff_t hdr_offset, next_offset;
int ret;
if (mbox_mail_seek(mail) < 0)
return -1;
ret = mbox_mail_get_next_offset(mail, &next_offset);
if (ret < 0) {
if (mbox_mail_seek(mail) < 0)
return -1;
ret = mbox_mail_get_next_offset(mail, &next_offset);
if (ret < 0) {
i_warning("mbox %s: Can't find next message offset "
"for uid=%u", mailbox_get_path(&mbox->box),
mail->mail.mail.uid);
}
}
raw_stream = mbox->mbox_stream;
if (istream_raw_mbox_get_header_offset(raw_stream, &hdr_offset) < 0) {
mail_storage_set_critical(mbox->box.storage,
"mbox %s: Couldn't get header offset for uid=%u",
mailbox_get_path(&mbox->box), mail->mail.mail.uid);
return -1;
}
i_stream_seek(raw_stream, hdr_offset);
if (next_offset != (uoff_t)-1)
istream_raw_mbox_set_next_offset(raw_stream, next_offset);
raw_stream = i_stream_create_limit(raw_stream, (uoff_t)-1);
mail->data.stream =
i_stream_create_header_filter(raw_stream,
HEADER_FILTER_EXCLUDE | HEADER_FILTER_NO_CR,
mbox_hide_headers, mbox_hide_headers_count,
*null_header_filter_callback, (void *)NULL);
i_stream_unref(&raw_stream);
return 0;
}
static int mbox_mail_get_stream(struct mail *_mail, bool get_body ATTR_UNUSED,
struct message_size *hdr_size,
struct message_size *body_size,
struct istream **stream_r)
{
struct index_mail *mail = (struct index_mail *)_mail;
if (mail->data.stream == NULL) {
if (mbox_mail_init_stream(mail) < 0)
return -1;
}
return index_mail_init_stream(mail, hdr_size, body_size, stream_r);
}
static void mbox_mail_set_seq(struct mail *_mail, uint32_t seq, bool saving)
{
struct index_mail *mail = (struct index_mail *)_mail;
index_mail_set_seq(_mail, seq, saving);
mail->data.dont_cache_fetch_fields |= MAIL_FETCH_PHYSICAL_SIZE;
}
static bool mbox_mail_set_uid(struct mail *_mail, uint32_t uid)
{
struct index_mail *mail = (struct index_mail *)_mail;
bool ret;
ret = index_mail_set_uid(_mail, uid);
mail->data.dont_cache_fetch_fields |= MAIL_FETCH_PHYSICAL_SIZE;
return ret;
}
struct mail_vfuncs mbox_mail_vfuncs = {
index_mail_close,
index_mail_free,
mbox_mail_set_seq,
mbox_mail_set_uid,
index_mail_set_uid_cache_updates,
index_mail_prefetch,
index_mail_precache,
index_mail_add_temp_wanted_fields,
index_mail_get_flags,
index_mail_get_keywords,
index_mail_get_keyword_indexes,
index_mail_get_modseq,
index_mail_get_pvt_modseq,
index_mail_get_parts,
index_mail_get_date,
mbox_mail_get_received_date,
mbox_mail_get_save_date,
index_mail_get_virtual_size,
mbox_mail_get_physical_size,
index_mail_get_first_header,
index_mail_get_headers,
index_mail_get_header_stream,
mbox_mail_get_stream,
index_mail_get_binary_stream,
mbox_mail_get_special,
index_mail_get_real_mail,
index_mail_update_flags,
index_mail_update_keywords,
index_mail_update_modseq,
index_mail_update_pvt_modseq,
NULL,
index_mail_expunge,
index_mail_set_cache_corrupted,
index_mail_opened
};
