static uint32_t index_storage_find_first_pvt_unseen_seq(struct mailbox *box)
{
const struct mail_index_header *pvt_hdr;
const struct mail_index_record *pvt_rec;
uint32_t pvt_seq, pvt_count, shared_seq, seq2;
pvt_count = mail_index_view_get_messages_count(box->view_pvt);
mail_index_lookup_first(box->view_pvt, 0, MAIL_SEEN, &pvt_seq);
if (pvt_seq == 0)
pvt_seq = pvt_count+1;
for (; pvt_seq <= pvt_count; pvt_seq++) {
pvt_rec = mail_index_lookup(box->view_pvt, pvt_seq);
if ((pvt_rec->flags & MAIL_SEEN) == 0 &&
mail_index_lookup_seq(box->view, pvt_rec->uid, &shared_seq))
return shared_seq;
}
/* if shared index has any messages that don't exist in private index,
the first of them is the first unseen message */
pvt_hdr = mail_index_get_header(box->view_pvt);
if (mail_index_lookup_seq_range(box->view,
pvt_hdr->next_uid, (uint32_t)-1,
&shared_seq, &seq2))
return shared_seq;
return 0;
}
static void mbox_mailbox_close(struct mailbox *box)
{
struct mbox_mailbox *mbox = (struct mbox_mailbox *)box;
const struct mail_index_header *hdr;
enum mbox_sync_flags sync_flags = 0;
if (mbox->mbox_stream != NULL &&
istream_raw_mbox_is_corrupted(mbox->mbox_stream)) {
/* clear the corruption by forcing a full resync */
sync_flags |= MBOX_SYNC_UNDIRTY | MBOX_SYNC_FORCE_SYNC;
}
if (box->view != NULL) {
hdr = mail_index_get_header(box->view);
if ((hdr->flags & MAIL_INDEX_HDR_FLAG_HAVE_DIRTY) != 0 &&
!mbox_is_backend_readonly(mbox)) {
/* we've done changes to mbox which haven't been
written yet. do it now. */
sync_flags |= MBOX_SYNC_REWRITE;
}
}
if (sync_flags != 0 && !mbox->invalid_mbox_file)
(void)mbox_sync(mbox, sync_flags);
if (mbox->mbox_global_lock_id != 0)
mbox_unlock(mbox, mbox->mbox_global_lock_id);
if (mbox->keep_lock_to != NULL)
timeout_remove(&mbox->keep_lock_to);
mbox_file_close(mbox);
if (mbox->mbox_file_stream != NULL)
i_stream_destroy(&mbox->mbox_file_stream);
index_storage_mailbox_close(box);
}
static void
imapc_untagged_exists(const struct imapc_untagged_reply *reply,
struct imapc_mailbox *mbox)
{
struct mail_index_view *view;
uint32_t exists_count = reply->num;
const struct mail_index_header *hdr;
if (mbox == NULL)
return;
view = mbox->delayed_sync_view;
if (view == NULL)
view = imapc_mailbox_get_sync_view(mbox);
if (mbox->selecting) {
/* We don't know the latest flags, refresh them. */
mbox->sync_fetch_first_uid = 1;
} else if (mbox->sync_fetch_first_uid != 1) {
hdr = mail_index_get_header(view);
mbox->sync_fetch_first_uid = hdr->next_uid;
}
mbox->exists_count = exists_count;
mbox->exists_received = TRUE;
imapc_mailbox_idle_notify(mbox);
}
int sdbox_mailbox_create_indexes(struct mailbox *box,
const struct mailbox_update *update,
struct mail_index_transaction *trans)
{
struct sdbox_mailbox *mbox = SDBOX_MAILBOX(box);
struct mail_index_transaction *new_trans = NULL;
const struct mail_index_header *hdr;
uint32_t uid_validity, uid_next;
if (trans == NULL) {
new_trans = mail_index_transaction_begin(box->view, 0);
trans = new_trans;
}
hdr = mail_index_get_header(box->view);
if (update != NULL && update->uid_validity != 0)
uid_validity = update->uid_validity;
else if (hdr->uid_validity != 0)
uid_validity = hdr->uid_validity;
else {
/* set uidvalidity */
uid_validity = dbox_get_uidvalidity_next(box->list);
}
if (hdr->uid_validity != uid_validity) {
mail_index_update_header(trans,
offsetof(struct mail_index_header, uid_validity),
&uid_validity, sizeof(uid_validity), TRUE);
}
int index_storage_mailbox_update(struct mailbox *box,
const struct mailbox_update *update)
{
const struct mail_index_header *hdr;
struct mail_index_view *view;
struct mail_index_transaction *trans;
int ret;
if (mailbox_open(box) < 0)
return -1;
/* make sure we get the latest index info */
mail_index_refresh(box->index);
view = mail_index_view_open(box->index);
hdr = mail_index_get_header(view);
trans = mail_index_transaction_begin(view,
MAIL_INDEX_TRANSACTION_FLAG_EXTERNAL);
if (update->uid_validity != 0 &&
hdr->uid_validity != update->uid_validity) {
uint32_t uid_validity = update->uid_validity;
if (hdr->uid_validity != 0) {
/* UIDVALIDITY change requires index to be reset */
mail_index_reset(trans);
}
mail_index_update_header(trans,
offsetof(struct mail_index_header, uid_validity),
&uid_validity, sizeof(uid_validity), TRUE);
}
mdbox_write_index_header(struct mailbox *box,
const struct mailbox_update *update,
struct mail_index_transaction *trans)
{
struct mdbox_mailbox *mbox = (struct mdbox_mailbox *)box;
struct mail_index_transaction *new_trans = NULL;
struct mail_index_view *view;
const struct mail_index_header *hdr;
uint32_t uid_validity, uid_next;
if (mdbox_map_open_or_create(mbox->storage->map) < 0)
return -1;
if (trans == NULL) {
new_trans = mail_index_transaction_begin(box->view, 0);
trans = new_trans;
}
view = mail_index_view_open(box->index);
hdr = mail_index_get_header(view);
uid_validity = hdr->uid_validity;
if (update != NULL && update->uid_validity != 0)
uid_validity = update->uid_validity;
else if (uid_validity == 0) {
/* set uidvalidity */
uid_validity = dbox_get_uidvalidity_next(box->list);
}
if (hdr->uid_validity != uid_validity) {
mail_index_update_header(trans,
offsetof(struct mail_index_header, uid_validity),
&uid_validity, sizeof(uid_validity), TRUE);
}
void index_mail_init(struct index_mail *mail,
struct mailbox_transaction_context *_t,
enum mail_fetch_field wanted_fields,
struct mailbox_header_lookup_ctx *_wanted_headers)
{
struct index_transaction_context *t =
(struct index_transaction_context *)_t;
struct index_header_lookup_ctx *wanted_headers =
(struct index_header_lookup_ctx *)_wanted_headers;
const struct mail_index_header *hdr;
array_create(&mail->mail.module_contexts, mail->mail.pool,
sizeof(void *), 5);
mail->mail.v = *_t->box->mail_vfuncs;
mail->mail.mail.box = _t->box;
mail->mail.mail.transaction = &t->mailbox_ctx;
mail->mail.wanted_fields = wanted_fields;
mail->mail.wanted_headers = _wanted_headers;
hdr = mail_index_get_header(_t->box->view);
mail->uid_validity = hdr->uid_validity;
t->mail_ref_count++;
mail->data_pool = pool_alloconly_create("index_mail", 16384);
mail->ibox = INDEX_STORAGE_CONTEXT(_t->box);
mail->trans = t;
mail->wanted_fields = wanted_fields;
if (wanted_headers != NULL) {
mail->wanted_headers = wanted_headers;
mailbox_header_lookup_ref(_wanted_headers);
}
}
unsigned int index_mailbox_get_recent_count(struct mailbox *box)
{
struct index_mailbox_context *ibox = INDEX_STORAGE_CONTEXT(box);
const struct mail_index_header *hdr;
const struct seq_range *range;
unsigned int i, count, recent_count;
if (!array_is_created(&ibox->recent_flags))
return 0;
hdr = mail_index_get_header(box->view);
recent_count = ibox->recent_flags_count;
range = array_get(&ibox->recent_flags, &count);
for (i = count; i > 0; ) {
i--;
if (range[i].seq2 < hdr->next_uid)
break;
if (range[i].seq1 >= hdr->next_uid) {
/* completely invisible to this view */
recent_count -= range[i].seq2 - range[i].seq1 + 1;
} else {
/* partially invisible */
recent_count -= range[i].seq2 - hdr->next_uid + 1;
break;
}
}
return recent_count;
}
static void
index_mailbox_expunge_unseen_recent(struct index_mailbox_sync_context *ctx)
{
struct index_mailbox_context *ibox =
INDEX_STORAGE_CONTEXT(ctx->ctx.box);
struct mail_index_view *view = ctx->ctx.box->view;
const struct mail_index_header *hdr;
uint32_t seq, start_uid, uid;
if (!array_is_created(&ibox->recent_flags))
return;
/* expunges array contained expunges for the messages that were already
visible in this view, but append+expunge would be invisible.
recent_flags may however contain the append UID, so we'll have to
remove it separately */
hdr = mail_index_get_header(view);
if (ctx->messages_count == 0)
uid = 0;
else if (ctx->messages_count <= hdr->messages_count)
mail_index_lookup_uid(view, ctx->messages_count, &uid);
else {
i_assert(mail_index_view_is_inconsistent(view));
return;
}
for (seq = ctx->messages_count + 1; seq <= hdr->messages_count; seq++) {
start_uid = uid;
mail_index_lookup_uid(view, seq, &uid);
if (start_uid + 1 > uid - 1)
continue;
ibox->recent_flags_count -=
seq_range_array_remove_range(&ibox->recent_flags,
start_uid + 1, uid - 1);
}
if (uid + 1 < hdr->next_uid) {
ibox->recent_flags_count -=
seq_range_array_remove_range(&ibox->recent_flags,
uid + 1,
hdr->next_uid - 1);
}
#ifdef DEBUG
if (!mail_index_view_is_inconsistent(view)) {
const struct seq_range *range;
unsigned int i, count;
range = array_get(&ibox->recent_flags, &count);
for (i = 0; i < count; i++) {
for (uid = range[i].seq1; uid <= range[i].seq2; uid++) {
if (uid >= hdr->next_uid)
break;
(void)mail_index_lookup_seq(view, uid, &seq);
i_assert(seq != 0);
}
}
}
#endif
}
static int mdbox_sync_index(struct mdbox_sync_context *ctx)
{
struct mailbox *box = &ctx->mbox->box;
const struct mail_index_header *hdr;
struct mail_index_sync_rec sync_rec;
uint32_t seq1, seq2;
int ret = 0;
hdr = mail_index_get_header(ctx->sync_view);
if (hdr->uid_validity == 0) {
/* newly created index file */
mail_storage_set_critical(box->storage,
"Mailbox %s: Corrupted index, uidvalidity=0",
box->vname);
return 0;
}
/* mark the newly seen messages as recent */
if (mail_index_lookup_seq_range(ctx->sync_view, hdr->first_recent_uid,
hdr->next_uid, &seq1, &seq2)) {
index_mailbox_set_recent_seq(&ctx->mbox->box, ctx->sync_view,
seq1, seq2);
}
/* handle syncing records without map being locked. */
if (mdbox_map_atomic_is_locked(ctx->atomic)) {
ctx->map_trans = mdbox_map_transaction_begin(ctx->atomic, FALSE);
i_array_init(&ctx->expunged_seqs, 64);
}
while (mail_index_sync_next(ctx->index_sync_ctx, &sync_rec)) {
if ((ret = mdbox_sync_rec(ctx, &sync_rec)) < 0)
break;
}
/* write refcount changes to map index. transaction commit updates the
log head, while tail is left behind. */
if (mdbox_map_atomic_is_locked(ctx->atomic)) {
if (ret == 0)
ret = mdbox_map_transaction_commit(ctx->map_trans);
/* write changes to mailbox index */
if (ret == 0)
ret = dbox_sync_mark_expunges(ctx);
/* finish the map changes and unlock the map. this also updates
map's tail -> head. */
if (ret < 0)
mdbox_map_atomic_set_failed(ctx->atomic);
mdbox_map_transaction_free(&ctx->map_trans);
array_free(&ctx->expunged_seqs);
}
if (box->v.sync_notify != NULL)
box->v.sync_notify(box, 0, 0);
return ret == 0 ? 1 :
(ctx->mbox->storage->corrupted ? 0 : -1);
}
void mail_cache_decision_state_update(struct mail_cache_view *view,
uint32_t seq, unsigned int field)
{
struct mail_cache *cache = view->cache;
enum mail_cache_decision_type dec;
const struct mail_index_header *hdr;
uint32_t uid;
i_assert(field < cache->fields_count);
if (view->no_decision_updates)
return;
dec = cache->fields[field].field.decision;
if (dec == (MAIL_CACHE_DECISION_NO | MAIL_CACHE_DECISION_FORCED)) {
/* don't update last_used */
return;
}
if (ioloop_time - cache->fields[field].field.last_used > 3600*24) {
/* update last_used about once a day */
cache->fields[field].field.last_used = (uint32_t)ioloop_time;
if (cache->field_file_map[field] != (uint32_t)-1)
cache->field_header_write_pending = TRUE;
}
if (dec != MAIL_CACHE_DECISION_TEMP) {
/* a) forced decision
b) not cached, mail_cache_decision_add() will handle this
c) permanently cached already, okay. */
return;
}
mail_index_lookup_uid(view->view, seq, &uid);
hdr = mail_index_get_header(view->view);
/* see if we want to change decision from TEMP to YES */
if (uid < cache->fields[field].uid_highwater ||
uid < hdr->day_first_uid[7]) {
/* a) nonordered access within this session. if client doesn't
request messages in growing order, we assume it doesn't
have a permanent local cache.
b) accessing message older than one week. assume it's a
client with no local cache. if it was just a new client
generating the local cache for the first time, we'll
drop back to TEMP within few months. */
cache->fields[field].field.decision = MAIL_CACHE_DECISION_YES;
cache->fields[field].decision_dirty = TRUE;
if (cache->field_file_map[field] != (uint32_t)-1)
cache->field_header_write_pending = TRUE;
} else {
cache->fields[field].uid_highwater = uid;
}
}
static void index_mail_drop_recent_flag(struct mail *mail)
{
const struct mail_index_header *hdr;
uint32_t first_recent_uid = mail->uid + 1;
hdr = mail_index_get_header(mail->transaction->view);
if (hdr->first_recent_uid < first_recent_uid) {
mail_index_update_header(mail->transaction->itrans,
offsetof(struct mail_index_header, first_recent_uid),
&first_recent_uid, sizeof(first_recent_uid), FALSE);
}
static void
index_rebuild_header(struct index_rebuild_context *ctx,
index_rebuild_generate_uidvalidity_t *gen_uidvalidity)
{
const struct mail_index_header *hdr, *backup_hdr, *trans_hdr;
struct mail_index *index = mail_index_view_get_index(ctx->view);
struct mail_index_modseq_header modseq_hdr;
struct mail_index_view *trans_view;
uint32_t uid_validity, next_uid;
uint64_t modseq;
hdr = mail_index_get_header(ctx->view);
backup_hdr = ctx->backup_view == NULL ? NULL :
mail_index_get_header(ctx->backup_view);
trans_view = mail_index_transaction_open_updated_view(ctx->trans);
trans_hdr = mail_index_get_header(trans_view);
/* set uidvalidity */
if (hdr->uid_validity != 0)
uid_validity = hdr->uid_validity;
else if (backup_hdr != NULL && backup_hdr->uid_validity != 0)
uid_validity = backup_hdr->uid_validity;
else
uid_validity = gen_uidvalidity(ctx->box->list);
mail_index_update_header(ctx->trans,
offsetof(struct mail_index_header, uid_validity),
&uid_validity, sizeof(uid_validity), TRUE);
/* set next-uid */
if (hdr->next_uid != 0)
next_uid = hdr->next_uid;
else if (backup_hdr != NULL && backup_hdr->next_uid != 0)
next_uid = backup_hdr->next_uid;
else
next_uid = 1;
if (next_uid > trans_hdr->next_uid) {
mail_index_update_header(ctx->trans,
offsetof(struct mail_index_header, next_uid),
&next_uid, sizeof(next_uid), FALSE);
}
static void imapc_sync_uid_next(struct imapc_sync_context *ctx)
{
struct imapc_mailbox *mbox = ctx->mbox;
const struct mail_index_header *hdr;
uint32_t uid_next = mbox->sync_uid_next;
if (uid_next < mbox->min_append_uid)
uid_next = mbox->min_append_uid;
hdr = mail_index_get_header(ctx->sync_view);
if (hdr->next_uid < uid_next) {
mail_index_update_header(ctx->trans,
offsetof(struct mail_index_header, next_uid),
&uid_next, sizeof(uid_next), FALSE);
}
static void mbox_save_init_sync(struct mailbox_transaction_context *t)
{
struct mbox_mailbox *mbox = (struct mbox_mailbox *)t->box;
struct mbox_save_context *ctx = (struct mbox_save_context *)t->save_ctx;
const struct mail_index_header *hdr;
struct mail_index_view *view;
/* open a new view to get the header. this is required if we just
synced the mailbox so we can get updated next_uid. */
(void)mail_index_refresh(mbox->box.index);
view = mail_index_view_open(mbox->box.index);
hdr = mail_index_get_header(view);
ctx->next_uid = hdr->next_uid;
ctx->uid_validity = hdr->uid_validity;
ctx->synced = TRUE;
mail_index_view_close(&view);
}
static void imapc_sync_uid_validity(struct imapc_sync_context *ctx)
{
struct imapc_mailbox *mbox = ctx->mbox;
const struct mail_index_header *hdr;
hdr = mail_index_get_header(ctx->sync_view);
if (hdr->uid_validity != mbox->sync_uid_validity &&
mbox->sync_uid_validity != 0) {
if (hdr->uid_validity != 0) {
/* uidvalidity changed, reset the entire mailbox */
mail_index_reset(ctx->trans);
mbox->sync_fetch_first_uid = 1;
}
mail_index_update_header(ctx->trans,
offsetof(struct mail_index_header, uid_validity),
&mbox->sync_uid_validity,
sizeof(mbox->sync_uid_validity), TRUE);
}
}
static void cydir_sync_index(struct cydir_sync_context *ctx)
{
struct mailbox *box = &ctx->mbox->box;
const struct mail_index_header *hdr;
struct mail_index_sync_rec sync_rec;
uint32_t seq1, seq2;
hdr = mail_index_get_header(ctx->sync_view);
if (hdr->uid_validity != 0)
ctx->uid_validity = hdr->uid_validity;
else
cydir_sync_set_uidvalidity(ctx);
/* mark the newly seen messages as recent */
if (mail_index_lookup_seq_range(ctx->sync_view, hdr->first_recent_uid,
hdr->next_uid, &seq1, &seq2)) {
mailbox_recent_flags_set_seqs(&ctx->mbox->box, ctx->sync_view,
seq1, seq2);
}
while (mail_index_sync_next(ctx->index_sync_ctx, &sync_rec)) {
if (!mail_index_lookup_seq_range(ctx->sync_view,
sync_rec.uid1, sync_rec.uid2,
&seq1, &seq2)) {
/* already expunged, nothing to do. */
continue;
}
switch (sync_rec.type) {
case MAIL_INDEX_SYNC_TYPE_EXPUNGE:
cydir_sync_expunge(ctx, seq1, seq2);
break;
case MAIL_INDEX_SYNC_TYPE_FLAGS:
case MAIL_INDEX_SYNC_TYPE_KEYWORD_ADD:
case MAIL_INDEX_SYNC_TYPE_KEYWORD_REMOVE:
/* FIXME: should be bother calling sync_notify()? */
break;
}
}
if (box->v.sync_notify != NULL)
box->v.sync_notify(box, 0, 0);
}
void index_sync_update_recent_count(struct mailbox *box)
{
struct index_mailbox_context *ibox = INDEX_STORAGE_CONTEXT(box);
const struct mail_index_header *hdr;
uint32_t seq1, seq2;
hdr = mail_index_get_header(box->view);
if (hdr->first_recent_uid > ibox->recent_flags_prev_uid ||
hdr->next_uid > ibox->recent_flags_last_check_nextuid) {
ibox->recent_flags_last_check_nextuid = hdr->next_uid;
if (mail_index_lookup_seq_range(box->view,
hdr->first_recent_uid,
hdr->next_uid,
&seq1, &seq2)) {
index_mailbox_set_recent_seq(box, box->view,
seq1, seq2);
}
}
}
void index_storage_get_status(struct mailbox *box,
enum mailbox_status_items items,
struct mailbox_status *status_r)
{
const struct mail_index_header *hdr;
i_assert(box->opened);
memset(status_r, 0, sizeof(struct mailbox_status));
/* we can get most of the status items without any trouble */
hdr = mail_index_get_header(box->view);
status_r->messages = hdr->messages_count;
if ((items & STATUS_RECENT) != 0) {
status_r->recent = index_mailbox_get_recent_count(box);
i_assert(status_r->recent <= status_r->messages);
}
status_r->unseen = hdr->messages_count - hdr->seen_messages_count;
status_r->uidvalidity = hdr->uid_validity;
status_r->uidnext = hdr->next_uid;
status_r->nonpermanent_modseqs = mail_index_is_in_memory(box->index);
if ((items & STATUS_HIGHESTMODSEQ) != 0) {
status_r->highest_modseq =
mail_index_modseq_get_highest(box->view);
if (status_r->highest_modseq == 0) {
/* modseqs not enabled yet, but we can't return 0 */
status_r->highest_modseq = 1;
}
}
if ((items & STATUS_FIRST_UNSEEN_SEQ) != 0) {
mail_index_lookup_first(box->view, 0, MAIL_SEEN,
&status_r->first_unseen_seq);
}
if ((items & STATUS_KEYWORDS) != 0)
status_r->keywords = mail_index_get_keywords(box->index);
if ((items & STATUS_CACHE_FIELDS) != 0)
index_storage_get_status_cache_fields(box, status_r);
if ((items & STATUS_VIRTUAL_SIZE) != 0)
index_storage_get_status_virtual_size(box, status_r);
}
static void
get_last_cached_seq(struct mailbox *box, uint32_t *last_cached_seq_r)
{
const struct mail_index_header *hdr;
struct mail_cache_view *cache_view;
uint32_t seq;
*last_cached_seq_r = 0;
if (!mail_cache_exists(box->cache))
return;
cache_view = mail_cache_view_open(box->cache, box->view);
hdr = mail_index_get_header(box->view);
for (seq = hdr->messages_count; seq > 0; seq--) {
if (mail_cache_field_exists_any(cache_view, seq)) {
*last_cached_seq_r = seq;
break;
}
}
mail_cache_view_close(&cache_view);
}
void index_mail_cache_add_idx(struct index_mail *mail, unsigned int field_idx,
const void *data, size_t data_size)
{
const struct mail_storage_settings *set =
mail->mail.mail.box->storage->set;
const struct mail_index_header *hdr;
if (set->mail_cache_min_mail_count > 0) {
/* First check if we've configured caching not to be used with
low enough message count. */
hdr = mail_index_get_header(mail->mail.mail.box->view);
if (hdr->messages_count < set->mail_cache_min_mail_count)
return;
}
if (!mail->data.no_caching &&
mail->data.dont_cache_field_idx != field_idx) {
mail_cache_add(mail->trans->cache_trans, mail->data.seq,
field_idx, data, data_size);
}
}
int mailbox_list_index_parse(struct mailbox_list *list,
struct mail_index_view *view, bool force)
{
struct mailbox_list_index *ilist = INDEX_LIST_CONTEXT(list);
const struct mail_index_header *hdr;
const char *error;
hdr = mail_index_get_header(view);
if (!force &&
hdr->log_file_seq == ilist->sync_log_file_seq &&
hdr->log_file_head_offset == ilist->sync_log_file_offset) {
/* nothing changed */
return 0;
}
mailbox_list_index_reset(ilist);
ilist->sync_log_file_seq = hdr->log_file_seq;
ilist->sync_log_file_offset = hdr->log_file_head_offset;
if (mailbox_list_index_parse_header(ilist, view) < 0) {
mailbox_list_set_critical(list,
"Corrupted mailbox list index header %s", ilist->path);
if (ilist->has_backing_store) {
mail_index_mark_corrupted(ilist->index);
return -1;
}
}
if (mailbox_list_index_parse_records(ilist, view, &error) < 0) {
mailbox_list_set_critical(list,
"Corrupted mailbox list index %s: %s",
ilist->path, error);
if (ilist->has_backing_store) {
mail_index_mark_corrupted(ilist->index);
return -1;
}
/* FIXME: find any missing mailboxes, add them and write the
index back. */
}
return 0;
}
static int
index_mailbox_get_first_save_date(struct mailbox *box,
struct mailbox_metadata *metadata_r)
{
const struct mail_index_header *hdr;
struct mailbox_transaction_context *t;
struct mail *mail;
uint32_t seq;
int ret = -1;
hdr = mail_index_get_header(box->view);
if (hdr->messages_count == 0) {
metadata_r->first_save_date = (time_t)-1;
return 0;
}
t = mailbox_transaction_begin(box, 0, __func__);
mail = mail_alloc(t, 0, NULL);
for (seq = 1; seq <= hdr->messages_count; seq++) {
mail_set_seq(mail, seq);
if (mail_get_save_date(mail, &metadata_r->first_save_date) == 0) {
ret = 0;
break;
}
if (mailbox_get_last_mail_error(box) != MAIL_ERROR_EXPUNGED) {
/* failed */
break;
}
}
mail_free(&mail);
(void)mailbox_transaction_commit(&t);
if (seq > hdr->messages_count) {
/* all messages were expunged after all */
metadata_r->first_save_date = (time_t)-1;
return 0;
}
return ret;
}
static void
pop3c_sync_messages(struct pop3c_mailbox *mbox,
struct mail_index_view *sync_view,
struct mail_index_transaction *sync_trans,
struct mail_cache_view *cache_view)
{
struct index_mailbox_context *ibox =
INDEX_STORAGE_CONTEXT(&mbox->box);
const struct mail_index_header *hdr;
struct mail_cache_transaction_ctx *cache_trans;
string_t *str;
uint32_t seq, seq1, seq2, iseq, uid;
unsigned int cache_idx = ibox->cache_fields[MAIL_CACHE_POP3_UIDL].idx;
i_assert(mbox->msg_uids == NULL);
/* set our uidvalidity */
hdr = mail_index_get_header(sync_view);
if (hdr->uid_validity == 0) {
uint32_t uid_validity = ioloop_time;
mail_index_update_header(sync_trans,
offsetof(struct mail_index_header, uid_validity),
&uid_validity, sizeof(uid_validity), TRUE);
}
static int mbox_mail_seek(struct index_mail *mail)
{
struct mbox_transaction_context *t =
(struct mbox_transaction_context *)mail->mail.mail.transaction;
struct mail *_mail = &mail->mail.mail;
struct mbox_mailbox *mbox = (struct mbox_mailbox *)_mail->box;
enum mbox_sync_flags sync_flags = 0;
int ret, try;
bool deleted;
if (_mail->expunged || mbox->syncing)
return -1;
if (_mail->lookup_abort != MAIL_LOOKUP_ABORT_NEVER) {
mail_set_aborted(_mail);
return -1;
}
if (mbox->mbox_stream != NULL &&
istream_raw_mbox_is_corrupted(mbox->mbox_stream)) {
/* clear the corruption by forcing a full resync */
sync_flags |= MBOX_SYNC_UNDIRTY | MBOX_SYNC_FORCE_SYNC;
}
for (try = 0; try < 2; try++) {
if ((sync_flags & MBOX_SYNC_FORCE_SYNC) != 0) {
/* dirty offsets are broken. make sure we can sync. */
mbox_prepare_resync(_mail);
}
if (mbox->mbox_lock_type == F_UNLCK) {
i_assert(t->read_lock_id == 0);
sync_flags |= MBOX_SYNC_LOCK_READING;
if (mbox_sync(mbox, sync_flags) < 0)
return -1;
t->read_lock_id = mbox_get_cur_lock_id(mbox);
i_assert(t->read_lock_id != 0);
/* refresh index file after mbox has been locked to
make sure we get only up-to-date mbox offsets. */
if (mail_index_refresh(mbox->box.index) < 0) {
mailbox_set_index_error(&mbox->box);
return -1;
}
i_assert(mbox->mbox_lock_type != F_UNLCK);
} else if (t->read_lock_id == 0) {
/* file is already locked by another transaction, but
we must keep it locked for the entire transaction,
so increase the lock counter. */
if (mbox_lock(mbox, mbox->mbox_lock_type,
&t->read_lock_id) < 0)
i_unreached();
}
if (mbox_file_open_stream(mbox) < 0)
return -1;
ret = mbox_file_seek(mbox, _mail->transaction->view,
_mail->seq, &deleted);
if (ret > 0) {
/* success */
break;
}
if (ret < 0) {
if (deleted)
mail_set_expunged(_mail);
return -1;
}
/* we'll need to re-sync it completely */
sync_flags |= MBOX_SYNC_UNDIRTY | MBOX_SYNC_FORCE_SYNC;
}
if (ret == 0) {
mail_storage_set_critical(&mbox->storage->storage,
"Losing sync for mail uid=%u in mbox file %s",
_mail->uid, mailbox_get_path(&mbox->box));
}
return 0;
}
static int mbox_mail_get_received_date(struct mail *_mail, time_t *date_r)
{
struct index_mail *mail = (struct index_mail *)_mail;
struct index_mail_data *data = &mail->data;
struct mbox_mailbox *mbox = (struct mbox_mailbox *)_mail->box;
if (index_mail_get_received_date(_mail, date_r) == 0)
return 0;
if (mbox_mail_seek(mail) < 0)
return -1;
data->received_date =
istream_raw_mbox_get_received_time(mbox->mbox_stream);
if (data->received_date == (time_t)-1) {
/* it's broken and conflicts with our "not found"
return value. change it. */
data->received_date = 0;
}
*date_r = data->received_date;
return 0;
}
static int mbox_mail_get_save_date(struct mail *_mail, time_t *date_r)
{
struct index_mail *mail = (struct index_mail *)_mail;
struct index_mail_data *data = &mail->data;
if (index_mail_get_save_date(_mail, date_r) == 0)
return 0;
/* no way to know this. save the current time into cache and use
that from now on. this works only as long as the index files
are permanent */
data->save_date = ioloop_time;
*date_r = data->save_date;
return 0;
}
static int
mbox_mail_get_md5_header(struct index_mail *mail, const char **value_r)
{
struct mail *_mail = &mail->mail.mail;
static uint8_t empty_md5[16] =
{ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 };
struct mbox_mailbox *mbox = (struct mbox_mailbox *)_mail->box;
const void *ext_data;
if (mail->data.guid != NULL) {
*value_r = mail->data.guid;
return 1;
}
mail_index_lookup_ext(_mail->transaction->view, _mail->seq,
mbox->md5hdr_ext_idx, &ext_data, NULL);
if (ext_data != NULL && memcmp(ext_data, empty_md5, 16) != 0) {
mail->data.guid = p_strdup(mail->mail.data_pool,
binary_to_hex(ext_data, 16));
*value_r = mail->data.guid;
return 1;
} else if (mail_index_is_expunged(_mail->transaction->view, _mail->seq)) {
mail_set_expunged(_mail);
return -1;
} else {
return 0;
}
}
static int
mbox_mail_get_special(struct mail *_mail, enum mail_fetch_field field,
const char **value_r)
{
struct index_mail *mail = (struct index_mail *)_mail;
struct mbox_mailbox *mbox = (struct mbox_mailbox *)_mail->box;
uoff_t offset;
bool move_offset;
int ret;
switch (field) {
case MAIL_FETCH_FROM_ENVELOPE:
if (mbox_mail_seek(mail) < 0)
return -1;
*value_r = istream_raw_mbox_get_sender(mbox->mbox_stream);
return 0;
case MAIL_FETCH_GUID:
case MAIL_FETCH_HEADER_MD5:
if ((ret = mbox_mail_get_md5_header(mail, value_r)) != 0)
return ret < 0 ? -1 : 0;
/* i guess in theory the empty_md5 is valid and can happen,
but it's almost guaranteed that it means the MD5 sum is
missing. recalculate it. */
if (mbox->mbox_lock_type == F_UNLCK ||
mbox->mbox_stream == NULL) {
offset = 0;
move_offset = FALSE;
} else {
offset = istream_raw_mbox_get_start_offset(mbox->mbox_stream);
move_offset = TRUE;
}
mbox->mbox_save_md5 = TRUE;
if (mbox_sync(mbox, MBOX_SYNC_FORCE_SYNC |
MBOX_SYNC_READONLY) < 0)
return -1;
if (move_offset) {
if (istream_raw_mbox_seek(mbox->mbox_stream,
offset) < 0) {
i_error("mbox %s sync lost during MD5 syncing",
_mail->box->name);
return -1;
}
}
if ((ret = mbox_mail_get_md5_header(mail, value_r)) == 0) {
i_error("mbox %s resyncing didn't save header MD5 values",
_mail->box->name);
return -1;
}
return ret < 0 ? -1 : 0;
default:
break;
}
return index_mail_get_special(_mail, field, value_r);
}
static int
mbox_mail_get_next_offset(struct index_mail *mail, uoff_t *next_offset_r)
{
struct mbox_mailbox *mbox = (struct mbox_mailbox *)mail->mail.mail.box;
struct mail_index_view *view;
const struct mail_index_header *hdr;
uint32_t seq;
int trailer_size;
int ret = 1;
*next_offset_r = (uoff_t)-1;
hdr = mail_index_get_header(mail->mail.mail.transaction->view);
if (mail->mail.mail.seq > hdr->messages_count) {
/* we're appending a new message */
return 0;
}
/* We can't really trust trans_view. The next message may already be
expunged from it. Also hdr.messages_count may be incorrect there.
So refresh the index to get the latest changes and get the next
message's offset using a new view. */
i_assert(mbox->mbox_lock_type != F_UNLCK);
if (mbox_sync_header_refresh(mbox) < 0)
return -1;
view = mail_index_view_open(mail->mail.mail.box->index);
hdr = mail_index_get_header(view);
if (!mail_index_lookup_seq(view, mail->mail.mail.uid, &seq))
i_panic("Message unexpectedly expunged from index");
if (seq < hdr->messages_count) {
if (mbox_file_lookup_offset(mbox, view, seq + 1,
next_offset_r) <= 0)
ret = -1;
} else if (mail->mail.mail.box->input != NULL) {
/* opened the mailbox as input stream. we can't trust the
sync_size, since it's wrong with compressed mailboxes */
ret = 0;
} else {
/* last message, use the synced mbox size */
trailer_size =
mbox->storage->storage.set->mail_save_crlf ? 2 : 1;
*next_offset_r = mbox->mbox_hdr.sync_size - trailer_size;
}
mail_index_view_close(&view);
return ret;
}
static int mbox_mail_get_physical_size(struct mail *_mail, uoff_t *size_r)
{
struct index_mail *mail = (struct index_mail *)_mail;
struct index_mail_data *data = &mail->data;
struct mbox_mailbox *mbox = (struct mbox_mailbox *)_mail->box;
struct istream *input;
struct message_size hdr_size;
uoff_t old_offset, body_offset, body_size, next_offset;
if (index_mail_get_physical_size(_mail, size_r) == 0)
return 0;
/* we want to return the header size as seen by mail_get_stream(). */
old_offset = data->stream == NULL ? 0 : data->stream->v_offset;
if (mail_get_stream(_mail, &hdr_size, NULL, &input) < 0)
return -1;
/* our header size varies, so don't do any caching */
if (istream_raw_mbox_get_body_offset(mbox->mbox_stream, &body_offset) < 0) {
mail_storage_set_critical(_mail->box->storage,
"mbox %s: Couldn't get body offset for uid=%u",
mailbox_get_path(&mbox->box), mail->mail.mail.uid);
return -1;
}
/* use the next message's offset to avoid reading through the entire
message body to find out its size */
if (mbox_mail_get_next_offset(mail, &next_offset) > 0)
body_size = next_offset - body_offset;
else
body_size = (uoff_t)-1;
/* verify that the calculated body size is correct */
if (istream_raw_mbox_get_body_size(mbox->mbox_stream,
body_size, &body_size) < 0) {
mail_storage_set_critical(_mail->box->storage,
"mbox %s: Couldn't get body size for uid=%u",
mailbox_get_path(&mbox->box), mail->mail.mail.uid);
return -1;
}
data->physical_size = hdr_size.physical_size + body_size;
*size_r = data->physical_size;
i_stream_seek(input, old_offset);
return 0;
}
static int mbox_mail_init_stream(struct index_mail *mail)
{
struct mbox_mailbox *mbox = (struct mbox_mailbox *)mail->mail.mail.box;
struct istream *raw_stream;
uoff_t hdr_offset, next_offset;
int ret;
if (mbox_mail_seek(mail) < 0)
return -1;
ret = mbox_mail_get_next_offset(mail, &next_offset);
if (ret < 0) {
if (mbox_mail_seek(mail) < 0)
return -1;
ret = mbox_mail_get_next_offset(mail, &next_offset);
if (ret < 0) {
i_warning("mbox %s: Can't find next message offset "
"for uid=%u", mailbox_get_path(&mbox->box),
mail->mail.mail.uid);
}
}
raw_stream = mbox->mbox_stream;
if (istream_raw_mbox_get_header_offset(raw_stream, &hdr_offset) < 0) {
mail_storage_set_critical(mbox->box.storage,
"mbox %s: Couldn't get header offset for uid=%u",
mailbox_get_path(&mbox->box), mail->mail.mail.uid);
return -1;
}
i_stream_seek(raw_stream, hdr_offset);
if (next_offset != (uoff_t)-1)
istream_raw_mbox_set_next_offset(raw_stream, next_offset);
raw_stream = i_stream_create_limit(raw_stream, (uoff_t)-1);
mail->data.stream =
i_stream_create_header_filter(raw_stream,
HEADER_FILTER_EXCLUDE | HEADER_FILTER_NO_CR,
mbox_hide_headers, mbox_hide_headers_count,
*null_header_filter_callback, (void *)NULL);
i_stream_unref(&raw_stream);
return 0;
}
static int mbox_mail_get_stream(struct mail *_mail, bool get_body ATTR_UNUSED,
struct message_size *hdr_size,
struct message_size *body_size,
struct istream **stream_r)
{
struct index_mail *mail = (struct index_mail *)_mail;
if (mail->data.stream == NULL) {
if (mbox_mail_init_stream(mail) < 0)
return -1;
}
return index_mail_init_stream(mail, hdr_size, body_size, stream_r);
}
static void mbox_mail_set_seq(struct mail *_mail, uint32_t seq, bool saving)
{
struct index_mail *mail = (struct index_mail *)_mail;
index_mail_set_seq(_mail, seq, saving);
mail->data.dont_cache_fetch_fields |= MAIL_FETCH_PHYSICAL_SIZE;
}
static bool mbox_mail_set_uid(struct mail *_mail, uint32_t uid)
{
struct index_mail *mail = (struct index_mail *)_mail;
bool ret;
ret = index_mail_set_uid(_mail, uid);
mail->data.dont_cache_fetch_fields |= MAIL_FETCH_PHYSICAL_SIZE;
return ret;
}
struct mail_vfuncs mbox_mail_vfuncs = {
index_mail_close,
index_mail_free,
mbox_mail_set_seq,
mbox_mail_set_uid,
index_mail_set_uid_cache_updates,
index_mail_prefetch,
index_mail_precache,
index_mail_add_temp_wanted_fields,
index_mail_get_flags,
index_mail_get_keywords,
index_mail_get_keyword_indexes,
index_mail_get_modseq,
index_mail_get_pvt_modseq,
index_mail_get_parts,
index_mail_get_date,
mbox_mail_get_received_date,
mbox_mail_get_save_date,
index_mail_get_virtual_size,
mbox_mail_get_physical_size,
index_mail_get_first_header,
index_mail_get_headers,
index_mail_get_header_stream,
mbox_mail_get_stream,
index_mail_get_binary_stream,
mbox_mail_get_special,
index_mail_get_real_mail,
index_mail_update_flags,
index_mail_update_keywords,
index_mail_update_modseq,
index_mail_update_pvt_modseq,
NULL,
index_mail_expunge,
index_mail_set_cache_corrupted,
index_mail_opened
};
static int mdbox_map_open_internal(struct mdbox_map *map, bool create_missing)
{
enum mail_index_open_flags open_flags;
struct mailbox_permissions perm;
int ret = 0;
if (map->view != NULL) {
/* already opened */
return 1;
}
mailbox_list_get_root_permissions(map->root_list, &perm);
mail_index_set_permissions(map->index, perm.file_create_mode,
perm.file_create_gid,
perm.file_create_gid_origin);
open_flags = MAIL_INDEX_OPEN_FLAG_NEVER_IN_MEMORY |
mail_storage_settings_to_index_flags(MAP_STORAGE(map)->set);
if (create_missing) {
if ((ret = mdbox_map_mkdir_storage(map)) < 0)
return -1;
if (ret > 0) {
/* storage/ directory already existed.
the index should exist also. */
} else {
open_flags |= MAIL_INDEX_OPEN_FLAG_CREATE;
}
}
ret = mail_index_open(map->index, open_flags);
if (ret == 0 && create_missing) {
/* storage/ already existed, but indexes didn't. we'll need to
take extra steps to make sure we won't overwrite any m.*
files that may already exist. */
map->verify_existing_file_ids = TRUE;
open_flags |= MAIL_INDEX_OPEN_FLAG_CREATE;
ret = mail_index_open(map->index, open_flags);
}
if (ret < 0) {
mail_storage_set_internal_error(MAP_STORAGE(map));
mail_index_reset_error(map->index);
return -1;
}
if (ret == 0) {
/* index not found - for now just return failure */
i_assert(!create_missing);
return 0;
}
map->view = mail_index_view_open(map->index);
mdbox_map_cleanup(map);
if (mail_index_get_header(map->view)->uid_validity == 0) {
if (mdbox_map_generate_uid_validity(map) < 0 ||
mdbox_map_refresh(map) < 0) {
mail_storage_set_internal_error(MAP_STORAGE(map));
mail_index_reset_error(map->index);
mail_index_close(map->index);
return -1;
}
}
return 1;
}
static int
virtual_size_add_new(struct mailbox *box,
struct index_vsize_header *vsize_hdr)
{
struct index_mailbox_context *ibox = INDEX_STORAGE_CONTEXT(box);
const struct mail_index_header *hdr;
struct mailbox_transaction_context *trans;
struct mail_search_context *search_ctx;
struct mail_search_args *search_args;
struct mail *mail;
uint32_t seq1, seq2;
uoff_t vsize;
int ret = 0;
hdr = mail_index_get_header(box->view);
if (vsize_hdr->highest_uid == 0)
seq2 = 0;
else if (!mail_index_lookup_seq_range(box->view, 1,
vsize_hdr->highest_uid,
&seq1, &seq2))
seq2 = 0;
if (vsize_hdr->message_count != seq2) {
if (vsize_hdr->message_count < seq2) {
mail_storage_set_critical(box->storage,
"vsize-hdr has invalid message-count (%u < %u)",
vsize_hdr->message_count, seq2);
} else {
/* some messages have been expunged, rescan */
}
memset(vsize_hdr, 0, sizeof(*vsize_hdr));
seq2 = 0;
}
search_args = mail_search_build_init();
mail_search_build_add_seqset(search_args, seq2 + 1,
hdr->messages_count);
trans = mailbox_transaction_begin(box, 0);
search_ctx = mailbox_search_init(trans, search_args, NULL,
MAIL_FETCH_VIRTUAL_SIZE, NULL);
while (mailbox_search_next(search_ctx, &mail)) {
if (mail_get_virtual_size(mail, &vsize) < 0) {
if (mail->expunged)
continue;
ret = -1;
break;
}
vsize_hdr->vsize += vsize;
vsize_hdr->highest_uid = mail->uid;
vsize_hdr->message_count++;
}
if (mailbox_search_deinit(&search_ctx) < 0)
ret = -1;
mail_search_args_unref(&search_args);
if (ret == 0) {
/* success, cache all */
vsize_hdr->highest_uid = hdr->next_uid - 1;
} else {
/* search failed, cache only up to highest seen uid */
}
mail_index_update_header_ext(trans->itrans, ibox->vsize_hdr_ext_id,
0, vsize_hdr, sizeof(*vsize_hdr));
(void)mailbox_transaction_commit(&trans);
return ret;
}
void index_storage_get_open_status(struct mailbox *box,
enum mailbox_status_items items,
struct mailbox_status *status_r)
{
const struct mail_index_header *hdr;
/* we can get most of the status items without any trouble */
hdr = mail_index_get_header(box->view);
status_r->messages = hdr->messages_count;
if ((items & STATUS_RECENT) != 0) {
if ((box->flags & MAILBOX_FLAG_DROP_RECENT) != 0) {
/* recent flags are set and dropped by the previous
sync while index was locked. if we updated the
recent flags here we'd have a race condition. */
i_assert(box->synced);
} else {
/* make sure recent count is set, in case we haven't
synced yet */
index_sync_update_recent_count(box);
}
status_r->recent = index_mailbox_get_recent_count(box);
i_assert(status_r->recent <= status_r->messages);
}
if ((items & STATUS_UNSEEN) != 0) {
if (box->view_pvt == NULL ||
(mailbox_get_private_flags_mask(box) & MAIL_SEEN) == 0) {
status_r->unseen = hdr->messages_count -
hdr->seen_messages_count;
} else {
status_r->unseen = index_storage_count_pvt_unseen(box);
}
}
status_r->uidvalidity = hdr->uid_validity;
status_r->uidnext = hdr->next_uid;
status_r->first_recent_uid = hdr->first_recent_uid;
if ((items & STATUS_HIGHESTMODSEQ) != 0) {
status_r->nonpermanent_modseqs =
mail_index_is_in_memory(box->index);
status_r->no_modseq_tracking =
!mail_index_have_modseq_tracking(box->index);
status_r->highest_modseq =
mail_index_modseq_get_highest(box->view);
if (status_r->highest_modseq == 0) {
/* modseqs not enabled yet, but we can't return 0 */
status_r->highest_modseq = 1;
}
}
if ((items & STATUS_HIGHESTPVTMODSEQ) != 0 && box->view_pvt != NULL) {
status_r->highest_pvt_modseq =
mail_index_modseq_get_highest(box->view_pvt);
if (status_r->highest_pvt_modseq == 0) {
/* modseqs not enabled yet, but we can't return 0 */
status_r->highest_pvt_modseq = 1;
}
}
if ((items & STATUS_FIRST_UNSEEN_SEQ) != 0) {
if (box->view_pvt == NULL ||
(mailbox_get_private_flags_mask(box) & MAIL_SEEN) == 0) {
mail_index_lookup_first(box->view, 0, MAIL_SEEN,
&status_r->first_unseen_seq);
} else {
status_r->first_unseen_seq =
index_storage_find_first_pvt_unseen_seq(box);
}
}
if ((items & STATUS_LAST_CACHED_SEQ) != 0)
get_last_cached_seq(box, &status_r->last_cached_seq);
if ((items & STATUS_KEYWORDS) != 0)
status_r->keywords = mail_index_get_keywords(box->index);
if ((items & STATUS_PERMANENT_FLAGS) != 0) {
if (!mailbox_is_readonly(box)) {
status_r->permanent_flags = MAIL_FLAGS_NONRECENT;
status_r->permanent_keywords = TRUE;
status_r->allow_new_keywords =
!box->disallow_new_keywords;
}
}
}
int maildir_sync_index(struct maildir_index_sync_context *ctx,
bool partial)
{
struct maildir_mailbox *mbox = ctx->mbox;
struct mail_index_view *view = ctx->view;
struct mail_index_view *view2;
struct maildir_uidlist_iter_ctx *iter;
struct mail_index_transaction *trans = ctx->trans;
const struct mail_index_header *hdr;
struct mail_index_header empty_hdr;
const struct mail_index_record *rec;
uint32_t seq, seq2, uid, prev_uid;
enum maildir_uidlist_rec_flag uflags;
const char *filename;
uint32_t uid_validity, next_uid, hdr_next_uid, first_recent_uid;
uint32_t first_uid;
unsigned int changes = 0;
int ret = 0;
time_t time_before_sync;
guid_128_t expunged_guid_128;
enum mail_flags private_flags_mask;
bool expunged, full_rescan = FALSE;
i_assert(!mbox->syncing_commit);
first_uid = 1;
hdr = mail_index_get_header(view);
uid_validity = maildir_uidlist_get_uid_validity(mbox->uidlist);
if (uid_validity != hdr->uid_validity &&
uid_validity != 0 && hdr->uid_validity != 0) {
/* uidvalidity changed and index isn't being synced for the
first time, reset the index so we can add all messages as
new */
i_warning("Maildir %s: UIDVALIDITY changed (%u -> %u)",
mailbox_get_path(&ctx->mbox->box),
hdr->uid_validity, uid_validity);
mail_index_reset(trans);
mailbox_recent_flags_reset(&mbox->box);
first_uid = hdr->messages_count + 1;
memset(&empty_hdr, 0, sizeof(empty_hdr));
empty_hdr.next_uid = 1;
hdr = &empty_hdr;
}
hdr_next_uid = hdr->next_uid;
ctx->mbox->box.tmp_sync_view = view;
private_flags_mask = mailbox_get_private_flags_mask(&mbox->box);
time_before_sync = time(NULL);
mbox->syncing_commit = TRUE;
seq = prev_uid = 0; first_recent_uid = I_MAX(hdr->first_recent_uid, 1);
i_array_init(&ctx->keywords, MAILDIR_MAX_KEYWORDS);
i_array_init(&ctx->idx_keywords, MAILDIR_MAX_KEYWORDS);
iter = maildir_uidlist_iter_init(mbox->uidlist);
while (maildir_uidlist_iter_next(iter, &uid, &uflags, &filename)) {
maildir_filename_flags_get(ctx->keywords_sync_ctx, filename,
&ctx->flags, &ctx->keywords);
i_assert(uid > prev_uid);
prev_uid = uid;
/* the private flags are kept only in indexes. don't use them
at all even for newly seen mails */
ctx->flags &= ~private_flags_mask;
again:
seq++;
ctx->uid = uid;
if (seq > hdr->messages_count) {
if (uid < hdr_next_uid) {
if (maildir_handle_uid_insertion(ctx, uflags,
filename,
uid) < 0)
ret = -1;
seq--;
continue;
}
/* Trust uidlist recent flags only for newly added
messages. When saving/copying messages with flags
they're stored to cur/ and uidlist treats them
as non-recent. */
if ((uflags & MAILDIR_UIDLIST_REC_FLAG_RECENT) == 0) {
if (uid >= first_recent_uid)
first_recent_uid = uid + 1;
}
hdr_next_uid = uid + 1;
mail_index_append(trans, uid, &seq);
mail_index_update_flags(trans, seq, MODIFY_REPLACE,
ctx->flags);
if (array_count(&ctx->keywords) > 0) {
struct mail_keywords *kw;
kw = mail_index_keywords_create_from_indexes(
mbox->box.index, &ctx->keywords);
mail_index_update_keywords(trans, seq,
MODIFY_REPLACE, kw);
mail_index_keywords_unref(&kw);
}
continue;
}
rec = mail_index_lookup(view, seq);
if (uid > rec->uid) {
/* already expunged (no point in showing guid in the
expunge record anymore) */
mail_index_expunge(ctx->trans, seq);
goto again;
}
if (uid < rec->uid) {
if (maildir_handle_uid_insertion(ctx, uflags,
filename, uid) < 0)
ret = -1;
seq--;
continue;
}
index_sync_changes_read(ctx->sync_changes, ctx->uid, &expunged,
expunged_guid_128);
if (expunged) {
if (!maildir_expunge_is_valid_guid(ctx, ctx->uid,
filename,
expunged_guid_128))
continue;
if (maildir_file_do(mbox, ctx->uid,
maildir_expunge, ctx) >= 0) {
/* successful expunge */
mail_index_expunge(ctx->trans, seq);
}
if ((++changes % MAILDIR_SLOW_MOVE_COUNT) == 0)
maildir_sync_notify(ctx->maildir_sync_ctx);
continue;
}
/* the private flags are stored only in indexes, keep them */
ctx->flags |= rec->flags & private_flags_mask;
if (index_sync_changes_have(ctx->sync_changes)) {
/* apply flag changes to maildir */
if (maildir_file_do(mbox, ctx->uid,
maildir_sync_flags, ctx) < 0)
ctx->flags |= MAIL_INDEX_MAIL_FLAG_DIRTY;
if ((++changes % MAILDIR_SLOW_MOVE_COUNT) == 0)
maildir_sync_notify(ctx->maildir_sync_ctx);
}
if ((uflags & MAILDIR_UIDLIST_REC_FLAG_NONSYNCED) != 0) {
/* partial syncing */
if ((uflags & MAILDIR_UIDLIST_REC_FLAG_NEW_DIR) != 0) {
/* we last saw this mail in new/, but it's
not there anymore. possibly expunged,
make sure. */
full_rescan = TRUE;
}
continue;
}
if ((rec->flags & MAIL_INDEX_MAIL_FLAG_DIRTY) != 0) {
/* we haven't been able to update maildir with this
record's flag changes. don't sync them. */
continue;
}
if (ctx->flags != (rec->flags & MAIL_FLAGS_NONRECENT)) {
mail_index_update_flags(trans, seq, MODIFY_REPLACE,
ctx->flags);
}
maildir_sync_mail_keywords(ctx, seq);
}
maildir_uidlist_iter_deinit(&iter);
if (!partial) {
/* expunge the rest */
for (seq++; seq <= hdr->messages_count; seq++)
mail_index_expunge(ctx->trans, seq);
}
/* add \Recent flags. use updated view so it contains newly
appended messages. */
view2 = mail_index_transaction_open_updated_view(trans);
if (mail_index_lookup_seq_range(view2, first_recent_uid, (uint32_t)-1,
&seq, &seq2) && seq2 >= first_uid) {
if (seq < first_uid) {
/* UIDVALIDITY changed, skip over the old messages */
seq = first_uid;
}
mailbox_recent_flags_set_seqs(&mbox->box, view2, seq, seq2);
}
mail_index_view_close(&view2);
if (ctx->uidlist_sync_ctx != NULL) {
if (maildir_uidlist_sync_deinit(&ctx->uidlist_sync_ctx,
TRUE) < 0)
ret = -1;
}
if (mbox->box.v.sync_notify != NULL)
mbox->box.v.sync_notify(&mbox->box, 0, 0);
ctx->mbox->box.tmp_sync_view = NULL;
/* check cur/ mtime later. if we came here from saving messages they
could still be moved to cur/ directory. */
ctx->update_maildir_hdr_cur = TRUE;
mbox->maildir_hdr.cur_check_time = time_before_sync;
if (uid_validity == 0) {
uid_validity = hdr->uid_validity != 0 ? hdr->uid_validity :
maildir_get_uidvalidity_next(mbox->box.list);
maildir_uidlist_set_uid_validity(mbox->uidlist, uid_validity);
}
maildir_uidlist_set_next_uid(mbox->uidlist, hdr_next_uid, FALSE);
if (uid_validity != hdr->uid_validity) {
mail_index_update_header(trans,
offsetof(struct mail_index_header, uid_validity),
&uid_validity, sizeof(uid_validity), TRUE);
}
void index_mail_set_seq(struct mail *_mail, uint32_t seq)
{
struct index_mail *mail = (struct index_mail *)_mail;
struct index_mail_data *data = &mail->data;
struct mail_cache_field *cache_fields = mail->ibox->cache_fields;
struct mail_cache_view *cache_view = mail->trans->cache_view;
const struct mail_index_header *hdr;
struct istream *input;
if (data->seq == seq)
return;
index_mail_reset(mail);
data->seq = seq;
mail->mail.mail.seq = seq;
mail_index_lookup_uid(_mail->transaction->view, seq,
&mail->mail.mail.uid);
if (mail_index_view_is_inconsistent(_mail->transaction->view)) {
mail_set_expunged(&mail->mail.mail);
return;
}
if ((mail->wanted_fields & (MAIL_FETCH_NUL_STATE |
MAIL_FETCH_IMAP_BODY |
MAIL_FETCH_IMAP_BODYSTRUCTURE)) != 0) {
(void)index_mail_get_fixed_field(mail, MAIL_CACHE_FLAGS,
&data->cache_flags,
sizeof(data->cache_flags));
mail->mail.mail.has_nuls =
(data->cache_flags & MAIL_CACHE_FLAG_HAS_NULS) != 0;
mail->mail.mail.has_no_nuls =
(data->cache_flags & MAIL_CACHE_FLAG_HAS_NO_NULS) != 0;
}
/* see if wanted_fields can tell us if we need to read/parse
header/body */
if ((mail->wanted_fields & MAIL_FETCH_MESSAGE_PARTS) != 0) {
unsigned int cache_field =
cache_fields[MAIL_CACHE_MESSAGE_PARTS].idx;
if (mail_cache_field_exists(cache_view, seq,
cache_field) <= 0) {
data->access_part |= PARSE_HDR | PARSE_BODY;
data->save_message_parts = TRUE;
}
}
if ((mail->wanted_fields & MAIL_FETCH_IMAP_ENVELOPE) != 0)
check_envelope(mail);
if ((mail->wanted_fields & MAIL_FETCH_IMAP_BODY) != 0 &&
(data->cache_flags & MAIL_CACHE_FLAG_TEXT_PLAIN_7BIT_ASCII) == 0) {
/* we need either imap.body or imap.bodystructure */
unsigned int cache_field1 =
cache_fields[MAIL_CACHE_IMAP_BODY].idx;
unsigned int cache_field2 =
cache_fields[MAIL_CACHE_IMAP_BODYSTRUCTURE].idx;
if (mail_cache_field_exists(cache_view,
seq, cache_field1) <= 0 &&
mail_cache_field_exists(cache_view,
seq, cache_field2) <= 0) {
data->access_part |= PARSE_HDR | PARSE_BODY;
data->save_bodystructure_header = TRUE;
data->save_bodystructure_body = TRUE;
}
}
if ((mail->wanted_fields & MAIL_FETCH_IMAP_BODYSTRUCTURE) != 0 &&
(data->cache_flags & MAIL_CACHE_FLAG_TEXT_PLAIN_7BIT_ASCII) == 0) {
unsigned int cache_field =
cache_fields[MAIL_CACHE_IMAP_BODYSTRUCTURE].idx;
if (mail_cache_field_exists(cache_view, seq,
cache_field) <= 0) {
data->access_part |= PARSE_HDR | PARSE_BODY;
data->save_bodystructure_header = TRUE;
data->save_bodystructure_body = TRUE;
}
}
if ((mail->wanted_fields & MAIL_FETCH_DATE) != 0) {
unsigned int cache_field =
cache_fields[MAIL_CACHE_SENT_DATE].idx;
if (mail_cache_field_exists(cache_view, seq,
cache_field) <= 0) {
data->access_part |= PARSE_HDR;
data->save_sent_date = TRUE;
}
}
if ((mail->wanted_fields & (MAIL_FETCH_STREAM_HEADER |
MAIL_FETCH_STREAM_BODY)) != 0) {
/* open stream immediately to set expunged flag if
it's already lost */
if ((mail->wanted_fields & MAIL_FETCH_STREAM_HEADER) != 0)
data->access_part |= READ_HDR;
if ((mail->wanted_fields & MAIL_FETCH_STREAM_BODY) != 0)
data->access_part |= READ_BODY;
/* open the stream only if we didn't get here from
mailbox_save_init() */
hdr = mail_index_get_header(_mail->box->view);
if (!_mail->saving && _mail->uid < hdr->next_uid)
(void)mail_get_stream(_mail, NULL, NULL, &input);
}
}
