static void notify_delay_callback(struct mailbox *box)
{
struct index_mailbox_context *ibox = INDEX_STORAGE_CONTEXT(box);
if (ibox->notify_delay_to != NULL)
timeout_remove(&ibox->notify_delay_to);
box->notify_callback(box, box->notify_context);
}
static int
dbox_get_cached_metadata(struct dbox_mail *mail, enum dbox_metadata_key key,
enum index_cache_field cache_field,
const char **value_r)
{
struct index_mail *imail = &mail->imail;
struct index_mailbox_context *ibox =
INDEX_STORAGE_CONTEXT(imail->mail.mail.box);
const char *value;
string_t *str;
uint32_t order;
str = str_new(imail->mail.data_pool, 64);
if (mail_cache_lookup_field(imail->mail.mail.transaction->cache_view,
str, imail->mail.mail.seq,
ibox->cache_fields[cache_field].idx) > 0) {
if (cache_field == MAIL_CACHE_POP3_ORDER) {
i_assert(str_len(str) == sizeof(order));
memcpy(&order, str_data(str), sizeof(order));
str_truncate(str, 0);
if (order != 0)
str_printfa(str, "%u", order);
else {
/* order=0 means it doesn't exist. we don't
want to return "0" though, because then the
mails get ordered to beginning, while
nonexistent are supposed to be ordered at
the end. */
}
}
*value_r = str_c(str);
return 0;
}
if (dbox_mail_metadata_get(mail, key, &value) < 0)
return -1;
if (value == NULL)
value = "";
if (cache_field != MAIL_CACHE_POP3_ORDER) {
index_mail_cache_add_idx(imail, ibox->cache_fields[cache_field].idx,
value, strlen(value)+1);
} else {
if (str_to_uint(value, &order) < 0)
order = 0;
index_mail_cache_add_idx(imail, ibox->cache_fields[cache_field].idx,
&order, sizeof(order));
}
/* don't return pointer to dbox metadata directly, since it may
change unexpectedly */
str_truncate(str, 0);
str_append(str, value);
*value_r = str_c(str);
return 0;
}
void index_mailbox_reset_uidvalidity(struct mailbox *box)
{
struct index_mailbox_context *ibox = INDEX_STORAGE_CONTEXT(box);
/* can't trust the currently cached recent flags anymore */
if (array_is_created(&ibox->recent_flags))
array_clear(&ibox->recent_flags);
ibox->recent_flags_count = 0;
ibox->recent_flags_prev_uid = 0;
}
static void notify_callback(struct mailbox *box)
{
struct index_mailbox_context *ibox = INDEX_STORAGE_CONTEXT(box);
timeout_reset(ibox->notify_to);
if (ibox->notify_delay_to == NULL) {
ibox->notify_delay_to =
timeout_add_short(NOTIFY_DELAY_MSECS,
notify_delay_callback, box);
}
}
static void index_transaction_index_rollback(struct mail_index_transaction *t)
{
struct index_transaction_context *it = MAIL_STORAGE_CONTEXT(t);
struct index_mailbox_context *ibox =
INDEX_STORAGE_CONTEXT(it->mailbox_ctx.box);
if (it->mailbox_ctx.save_ctx != NULL)
ibox->save_rollback(it->mailbox_ctx.save_ctx);
i_assert(it->mail_ref_count == 0);
it->super.rollback(t);
index_transaction_free(it);
}
static void
index_mailbox_expunge_recent(struct mailbox *box, uint32_t seq1, uint32_t seq2)
{
struct index_mailbox_context *ibox = INDEX_STORAGE_CONTEXT(box);
uint32_t uid;
if (!array_is_created(&ibox->recent_flags))
return;
for (; seq1 <= seq2; seq1++) {
mail_index_lookup_uid(box->view, seq1, &uid);
if (seq_range_array_remove(&ibox->recent_flags, uid))
ibox->recent_flags_count--;
}
}
void index_storage_lock_notify(struct mailbox *box,
enum mailbox_lock_notify_type notify_type,
unsigned int secs_left)
{
struct index_mailbox_context *ibox = INDEX_STORAGE_CONTEXT(box);
struct mail_storage *storage = box->storage;
const char *str;
time_t now;
/* if notify type changes, print the message immediately */
now = time(NULL);
if (ibox->last_notify_type == MAILBOX_LOCK_NOTIFY_NONE ||
ibox->last_notify_type == notify_type) {
if (ibox->last_notify_type == MAILBOX_LOCK_NOTIFY_NONE &&
notify_type == MAILBOX_LOCK_NOTIFY_MAILBOX_OVERRIDE) {
/* first override notification, show it */
} else {
if (now < ibox->next_lock_notify || secs_left < 15)
return;
}
}
ibox->next_lock_notify = now + LOCK_NOTIFY_INTERVAL;
ibox->last_notify_type = notify_type;
switch (notify_type) {
case MAILBOX_LOCK_NOTIFY_NONE:
break;
case MAILBOX_LOCK_NOTIFY_MAILBOX_ABORT:
if (storage->callbacks.notify_no == NULL)
break;
str = t_strdup_printf("Mailbox is locked, will abort in "
"%u seconds", secs_left);
storage->callbacks.
notify_no(box, str, storage->callback_context);
break;
case MAILBOX_LOCK_NOTIFY_MAILBOX_OVERRIDE:
if (storage->callbacks.notify_ok == NULL)
break;
str = t_strdup_printf("Stale mailbox lock file detected, "
"will override in %u seconds", secs_left);
storage->callbacks.
notify_ok(box, str, storage->callback_context);
break;
}
}
static void check_timeout(struct mailbox *box)
{
struct index_mailbox_context *ibox = INDEX_STORAGE_CONTEXT(box);
struct index_notify_file *file;
struct stat st;
bool notify = FALSE;
for (file = ibox->notify_files; file != NULL; file = file->next) {
if (stat(file->path, &st) == 0 &&
file->last_stamp != st.st_mtime) {
file->last_stamp = st.st_mtime;
notify = TRUE;
}
}
if (notify)
notify_delay_callback(box);
}
void index_mailbox_check_add(struct mailbox *box, const char *path)
{
struct index_mailbox_context *ibox = INDEX_STORAGE_CONTEXT(box);
const struct mail_storage_settings *set = box->storage->set;
struct index_notify_file *file;
struct stat st;
struct io *io = NULL;
struct index_notify_io *aio;
i_assert(set->mailbox_idle_check_interval > 0);
(void)io_add_notify(path, notify_callback, box, &io);
if (io != NULL) {
aio = i_new(struct index_notify_io, 1);
aio->io = io;
aio->next = ibox->notify_ios;
ibox->notify_ios = aio;
}
void index_sync_update_recent_count(struct mailbox *box)
{
struct index_mailbox_context *ibox = INDEX_STORAGE_CONTEXT(box);
const struct mail_index_header *hdr;
uint32_t seq1, seq2;
hdr = mail_index_get_header(box->view);
if (hdr->first_recent_uid > ibox->recent_flags_prev_uid ||
hdr->next_uid > ibox->recent_flags_last_check_nextuid) {
ibox->recent_flags_last_check_nextuid = hdr->next_uid;
if (mail_index_lookup_seq_range(box->view,
hdr->first_recent_uid,
hdr->next_uid,
&seq1, &seq2)) {
index_mailbox_set_recent_seq(box, box->view,
seq1, seq2);
}
}
}
void index_mailbox_set_recent_uid(struct mailbox *box, uint32_t uid)
{
struct index_mailbox_context *ibox = INDEX_STORAGE_CONTEXT(box);
if (uid <= ibox->recent_flags_prev_uid) {
if (seq_range_exists(&ibox->recent_flags, uid))
return;
mail_storage_set_critical(box->storage,
"Recent flags state corrupted for mailbox %s",
box->vname);
array_clear(&ibox->recent_flags);
ibox->recent_flags_count = 0;
}
ibox->recent_flags_prev_uid = uid;
seq_range_array_add_with_init(&ibox->recent_flags, 64, uid);
ibox->recent_flags_count++;
}
static void
parse_imap_keywords_list(struct mbox_sync_mail_context *ctx,
struct message_header_line *hdr, size_t pos)
{
struct mailbox *box = &ctx->sync_ctx->mbox->box;
struct index_mailbox_context *ibox = INDEX_STORAGE_CONTEXT(box);
const char *keyword, *error;
size_t keyword_start;
unsigned int idx, count;
count = 0;
while (pos < hdr->full_value_len) {
if (IS_LWSP_LF(hdr->full_value[pos])) {
pos++;
continue;
}
/* read the keyword */
keyword_start = pos;
for (; pos < hdr->full_value_len; pos++) {
if (IS_LWSP_LF(hdr->full_value[pos]))
break;
}
/* add it to index's keyword list if it's not there already */
keyword = t_strndup(hdr->full_value + keyword_start,
pos - keyword_start);
if (mailbox_keyword_is_valid(&ctx->sync_ctx->mbox->box,
keyword, &error)) {
mail_index_keyword_lookup_or_create(box->index,
keyword, &idx);
}
count++;
}
if (count != array_count(ibox->keyword_names)) {
/* need to update this list */
ctx->imapbase_rewrite = TRUE;
ctx->need_rewrite = TRUE;
}
}
static int
get_metadata_virtual_size(struct mailbox *box,
struct mailbox_metadata *metadata_r)
{
struct index_mailbox_context *ibox = INDEX_STORAGE_CONTEXT(box);
struct index_vsize_header vsize_hdr;
struct mailbox_status status;
const void *data;
size_t size;
int ret;
mailbox_get_open_status(box, STATUS_MESSAGES | STATUS_UIDNEXT, &status);
mail_index_get_header_ext(box->view, ibox->vsize_hdr_ext_id,
&data, &size);
if (size == sizeof(vsize_hdr))
memcpy(&vsize_hdr, data, sizeof(vsize_hdr));
else {
if (size != 0) {
mail_storage_set_critical(box->storage,
"vsize-hdr has invalid size: %"PRIuSIZE_T,
size);
}
memset(&vsize_hdr, 0, sizeof(vsize_hdr));
}
if (vsize_hdr.highest_uid + 1 == status.uidnext &&
vsize_hdr.message_count == status.messages) {
/* up to date */
metadata_r->virtual_size = vsize_hdr.vsize;
return 0;
}
if (vsize_hdr.highest_uid >= status.uidnext) {
mail_storage_set_critical(box->storage,
"vsize-hdr has invalid highest-uid (%u >= %u)",
vsize_hdr.highest_uid, status.uidnext);
memset(&vsize_hdr, 0, sizeof(vsize_hdr));
}
ret = virtual_size_add_new(box, &vsize_hdr);
metadata_r->virtual_size = vsize_hdr.vsize;
return ret;
}
static struct mailbox *
mbox_mailbox_alloc(struct mail_storage *storage, struct mailbox_list *list,
const char *name, enum mailbox_flags flags)
{
struct mbox_mailbox *mbox;
struct index_mailbox_context *ibox;
pool_t pool;
pool = pool_alloconly_create("mbox mailbox", 1024*3);
mbox = p_new(pool, struct mbox_mailbox, 1);
mbox->box = mbox_mailbox;
mbox->box.pool = pool;
mbox->box.storage = storage;
mbox->box.list = list;
mbox->box.mail_vfuncs = &mbox_mail_vfuncs;
index_storage_mailbox_alloc(&mbox->box, name, flags, MBOX_INDEX_PREFIX);
ibox = INDEX_STORAGE_CONTEXT(&mbox->box);
ibox->save_commit_pre = mbox_transaction_save_commit_pre;
ibox->save_commit_post = mbox_transaction_save_commit_post;
ibox->save_rollback = mbox_transaction_save_rollback;
mbox->storage = (struct mbox_storage *)storage;
mbox->mbox_fd = -1;
mbox->mbox_lock_type = F_UNLCK;
mbox->mbox_ext_idx =
mail_index_ext_register(mbox->box.index, "mbox",
sizeof(mbox->mbox_hdr),
sizeof(uint64_t), sizeof(uint64_t));
mbox->md5hdr_ext_idx =
mail_index_ext_register(mbox->box.index, "header-md5",
0, 16, 1);
if ((storage->flags & MAIL_STORAGE_FLAG_KEEP_HEADER_MD5) != 0)
mbox->mbox_save_md5 = TRUE;
return &mbox->box;
}
static void index_cache_register_defaults(struct mailbox *box)
{
struct index_mailbox_context *ibox = INDEX_STORAGE_CONTEXT(box);
const struct mail_storage_settings *set = box->storage->set;
struct mail_cache *cache = box->cache;
ibox->cache_fields = i_malloc(sizeof(global_cache_fields));
memcpy(ibox->cache_fields, global_cache_fields,
sizeof(global_cache_fields));
mail_cache_register_fields(cache, ibox->cache_fields,
MAIL_INDEX_CACHE_FIELD_COUNT);
set_cache_decisions(cache, "mail_cache_fields",
set->mail_cache_fields,
MAIL_CACHE_DECISION_TEMP);
set_cache_decisions(cache, "mail_always_cache_fields",
set->mail_always_cache_fields,
MAIL_CACHE_DECISION_YES |
MAIL_CACHE_DECISION_FORCED);
set_cache_decisions(cache, "mail_never_cache_fields",
set->mail_never_cache_fields,
MAIL_CACHE_DECISION_NO |
MAIL_CACHE_DECISION_FORCED);
}
void index_storage_mailbox_close(struct mailbox *box)
{
struct index_mailbox_context *ibox = INDEX_STORAGE_CONTEXT(box);
mailbox_watch_remove_all(box);
if (box->input != NULL)
i_stream_unref(&box->input);
if (box->view_pvt != NULL)
mail_index_view_close(&box->view_pvt);
if (box->index_pvt != NULL)
mail_index_close(box->index_pvt);
if (box->view != NULL) {
mail_index_view_close(&box->view);
mail_index_close(box->index);
}
box->cache = NULL;
ibox->keyword_names = NULL;
i_free_and_null(ibox->cache_fields);
ibox->sync_last_check = 0;
}
static void
index_storage_get_status_virtual_size(struct mailbox *box,
struct mailbox_status *status_r)
{
struct index_mailbox_context *ibox = INDEX_STORAGE_CONTEXT(box);
struct index_vsize_header vsize_hdr;
const void *data;
size_t size;
mail_index_get_header_ext(box->view, ibox->vsize_hdr_ext_id,
&data, &size);
if (size == sizeof(vsize_hdr))
memcpy(&vsize_hdr, data, sizeof(vsize_hdr));
else {
if (size != 0) {
mail_storage_set_critical(box->storage,
"vsize-hdr has invalid size: %"PRIuSIZE_T,
size);
}
memset(&vsize_hdr, 0, sizeof(vsize_hdr));
}
if (vsize_hdr.highest_uid + 1 == status_r->uidnext &&
vsize_hdr.message_count == status_r->messages) {
/* up to date */
status_r->virtual_size = vsize_hdr.vsize;
return;
}
if (vsize_hdr.highest_uid >= status_r->uidnext) {
mail_storage_set_critical(box->storage,
"vsize-hdr has invalid highest-uid (%u >= %u)",
vsize_hdr.highest_uid, status_r->uidnext);
memset(&vsize_hdr, 0, sizeof(vsize_hdr));
}
index_storage_virtual_size_add_new(box, &vsize_hdr);
status_r->virtual_size = vsize_hdr.vsize;
}
static void
pop3c_sync_messages(struct pop3c_mailbox *mbox,
struct mail_index_view *sync_view,
struct mail_index_transaction *sync_trans,
struct mail_cache_view *cache_view)
{
struct index_mailbox_context *ibox =
INDEX_STORAGE_CONTEXT(&mbox->box);
const struct mail_index_header *hdr;
struct mail_cache_transaction_ctx *cache_trans;
string_t *str;
uint32_t seq, seq1, seq2, iseq, uid;
unsigned int cache_idx = ibox->cache_fields[MAIL_CACHE_POP3_UIDL].idx;
i_assert(mbox->msg_uids == NULL);
/* set our uidvalidity */
hdr = mail_index_get_header(sync_view);
if (hdr->uid_validity == 0) {
uint32_t uid_validity = ioloop_time;
mail_index_update_header(sync_trans,
offsetof(struct mail_index_header, uid_validity),
&uid_validity, sizeof(uid_validity), TRUE);
}
static int maildir_get_pop3_state(struct index_mail *mail)
{
struct mailbox *box = mail->mail.mail.box;
struct index_mailbox_context *ibox = INDEX_STORAGE_CONTEXT(box);
const struct mail_cache_field *fields;
unsigned int i, count, psize_idx, vsize_idx;
enum mail_cache_decision_type dec, vsize_dec;
enum mail_fetch_field allowed_pop3_fields;
bool not_pop3_only = FALSE;
if (mail->pop3_state_set)
return mail->pop3_state;
/* if this mail itself has non-pop3 fields we know we're not
pop3-only */
allowed_pop3_fields = MAIL_FETCH_FLAGS | MAIL_FETCH_STREAM_HEADER |
MAIL_FETCH_STREAM_BODY | MAIL_FETCH_STORAGE_ID |
MAIL_FETCH_VIRTUAL_SIZE;
if (mail->data.wanted_headers != NULL ||
(mail->data.wanted_fields & ~allowed_pop3_fields) != 0)
not_pop3_only = TRUE;
/* get vsize decisions */
psize_idx = ibox->cache_fields[MAIL_CACHE_PHYSICAL_FULL_SIZE].idx;
vsize_idx = ibox->cache_fields[MAIL_CACHE_VIRTUAL_FULL_SIZE].idx;
if (not_pop3_only) {
vsize_dec = mail_cache_field_get_decision(box->cache,
vsize_idx);
vsize_dec &= ~MAIL_CACHE_DECISION_FORCED;
} else {
/* also check if there are any non-[pv]size cached fields */
vsize_dec = MAIL_CACHE_DECISION_NO;
fields = mail_cache_register_get_list(box->cache,
pool_datastack_create(),
&count);
for (i = 0; i < count; i++) {
dec = fields[i].decision & ~MAIL_CACHE_DECISION_FORCED;
if (fields[i].idx == vsize_idx)
vsize_dec = dec;
else if (dec != MAIL_CACHE_DECISION_NO &&
fields[i].idx != psize_idx)
not_pop3_only = TRUE;
}
}
if (!not_pop3_only) {
/* either nothing is cached, or only vsize is cached. */
mail->pop3_state = 1;
} else if (vsize_dec != MAIL_CACHE_DECISION_YES &&
(box->flags & MAILBOX_FLAG_POP3_SESSION) == 0) {
/* if virtual size isn't cached permanently,
POP3 isn't being used */
mail->pop3_state = -1;
} else {
/* possibly a mixed pop3/imap */
mail->pop3_state = 0;
}
mail->pop3_state_set = TRUE;
return mail->pop3_state;
}
int index_storage_mailbox_open(struct mailbox *box, bool move_to_memory)
{
struct index_mailbox_context *ibox = INDEX_STORAGE_CONTEXT(box);
enum mail_index_open_flags index_flags;
int ret;
i_assert(!box->opened);
index_flags = ibox->index_flags;
if (move_to_memory)
ibox->index_flags &= ~MAIL_INDEX_OPEN_FLAG_CREATE;
if (index_storage_mailbox_alloc_index(box) < 0)
return -1;
/* make sure mail_index_set_permissions() has been called */
(void)mailbox_get_permissions(box);
ret = mail_index_open(box->index, index_flags);
if (ret <= 0 || move_to_memory) {
if ((index_flags & MAIL_INDEX_OPEN_FLAG_NEVER_IN_MEMORY) != 0) {
i_assert(ret <= 0);
mailbox_set_index_error(box);
return -1;
}
if (mail_index_move_to_memory(box->index) < 0) {
/* try opening once more. it should be created
directly into memory now. */
if (mail_index_open_or_create(box->index,
index_flags) < 0)
i_panic("in-memory index creation failed");
}
}
if ((index_flags & MAIL_INDEX_OPEN_FLAG_NEVER_IN_MEMORY) != 0) {
if (mail_index_is_in_memory(box->index)) {
mail_storage_set_critical(box->storage,
"Couldn't create index file");
mail_index_close(box->index);
return -1;
}
}
if ((box->flags & MAILBOX_FLAG_OPEN_DELETED) == 0) {
if (mail_index_is_deleted(box->index)) {
mailbox_set_deleted(box);
mail_index_close(box->index);
return -1;
}
}
box->cache = mail_index_get_cache(box->index);
index_cache_register_defaults(box);
box->view = mail_index_view_open(box->index);
ibox->keyword_names = mail_index_get_keywords(box->index);
ibox->vsize_hdr_ext_id =
mail_index_ext_register(box->index, "hdr-vsize",
sizeof(struct index_vsize_header), 0,
sizeof(uint64_t));
box->opened = TRUE;
if ((box->enabled_features & MAILBOX_FEATURE_CONDSTORE) != 0)
mail_index_modseq_enable(box->index);
index_thread_mailbox_opened(box);
hook_mailbox_opened(box);
return 0;
}
static int
virtual_size_add_new(struct mailbox *box,
struct index_vsize_header *vsize_hdr)
{
struct index_mailbox_context *ibox = INDEX_STORAGE_CONTEXT(box);
const struct mail_index_header *hdr;
struct mailbox_transaction_context *trans;
struct mail_search_context *search_ctx;
struct mail_search_args *search_args;
struct mail *mail;
uint32_t seq1, seq2;
uoff_t vsize;
int ret = 0;
hdr = mail_index_get_header(box->view);
if (vsize_hdr->highest_uid == 0)
seq2 = 0;
else if (!mail_index_lookup_seq_range(box->view, 1,
vsize_hdr->highest_uid,
&seq1, &seq2))
seq2 = 0;
if (vsize_hdr->message_count != seq2) {
if (vsize_hdr->message_count < seq2) {
mail_storage_set_critical(box->storage,
"vsize-hdr has invalid message-count (%u < %u)",
vsize_hdr->message_count, seq2);
} else {
/* some messages have been expunged, rescan */
}
memset(vsize_hdr, 0, sizeof(*vsize_hdr));
seq2 = 0;
}
search_args = mail_search_build_init();
mail_search_build_add_seqset(search_args, seq2 + 1,
hdr->messages_count);
trans = mailbox_transaction_begin(box, 0);
search_ctx = mailbox_search_init(trans, search_args, NULL,
MAIL_FETCH_VIRTUAL_SIZE, NULL);
while (mailbox_search_next(search_ctx, &mail)) {
if (mail_get_virtual_size(mail, &vsize) < 0) {
if (mail->expunged)
continue;
ret = -1;
break;
}
vsize_hdr->vsize += vsize;
vsize_hdr->highest_uid = mail->uid;
vsize_hdr->message_count++;
}
if (mailbox_search_deinit(&search_ctx) < 0)
ret = -1;
mail_search_args_unref(&search_args);
if (ret == 0) {
/* success, cache all */
vsize_hdr->highest_uid = hdr->next_uid - 1;
} else {
/* search failed, cache only up to highest seen uid */
}
mail_index_update_header_ext(trans->itrans, ibox->vsize_hdr_ext_id,
0, vsize_hdr, sizeof(*vsize_hdr));
(void)mailbox_transaction_commit(&trans);
return ret;
}
/* @UNSAFE */
data = buffer_get_space_unsafe(ctx->header, pos, need);
for (i = 0; flags_list[i].chr != 0; i++) {
if ((ctx->mail.flags & flags_list[i].flag) != 0)
*data++ = flags_list[i].chr;
}
ctx->mail.flags ^= MBOX_NONRECENT_KLUDGE;
}
static void
keywords_append(struct mbox_sync_context *sync_ctx, string_t *dest,
const ARRAY_TYPE(keyword_indexes) *keyword_indexes_arr)
{
struct index_mailbox_context *ibox =
INDEX_STORAGE_CONTEXT(&sync_ctx->mbox->box);
const char *const *keyword_names;
const unsigned int *keyword_indexes;
unsigned int i, idx_count, keywords_count;
size_t last_break;
keyword_names = array_get(ibox->keyword_names,
&keywords_count);
keyword_indexes = array_get(keyword_indexes_arr, &idx_count);
for (i = 0, last_break = str_len(dest); i < idx_count; i++) {
i_assert(keyword_indexes[i] < keywords_count);
/* wrap the line whenever it gets too long */
if (str_len(dest) - last_break < KEYWORD_WRAP_LINE_LENGTH) {
if (i > 0)