static int raw_mail_stat(struct mail *mail)
{
struct raw_mailbox *mbox = (struct raw_mailbox *)mail->box;
const struct stat *st;
if (mail->lookup_abort == MAIL_LOOKUP_ABORT_NOT_IN_CACHE) {
mail_set_aborted(mail);
return -1;
}
mail->transaction->stats.fstat_lookup_count++;
if (i_stream_stat(mail->box->input, TRUE, &st) < 0) {
mail_storage_set_critical(mail->box->storage,
"stat(%s) failed: %m",
i_stream_get_name(mail->box->input));
return -1;
}
if (mbox->mtime != (time_t)-1)
mbox->mtime = st->st_mtime;
if (mbox->ctime != (time_t)-1)
mbox->ctime = st->st_ctime;
mbox->size = st->st_size;
return 0;
}
int mdbox_mail_open(struct dbox_mail *mail, uoff_t *offset_r,
struct dbox_file **file_r)
{
struct mail *_mail = &mail->imail.mail.mail;
struct mdbox_mailbox *mbox = (struct mdbox_mailbox *)_mail->box;
uint32_t prev_file_id = 0, map_uid = 0;
bool deleted;
if (_mail->lookup_abort != MAIL_LOOKUP_ABORT_NEVER) {
mail_set_aborted(_mail);
return -1;
}
do {
if (mail->open_file != NULL) {
/* already open */
} else if (!_mail->saving) {
if (mdbox_mail_lookup(mbox, _mail->transaction->view,
_mail->seq, &map_uid) < 0)
return -1;
if (dbox_mail_open_init(mail, map_uid) < 0)
return -1;
} else {
/* mail is being saved in this transaction */
mail->open_file =
mdbox_save_file_get_file(_mail->transaction,
_mail->seq,
&mail->offset);
}
if (!dbox_file_is_open(mail->open_file))
mail->imail.mail.stats_open_lookup_count++;
if (dbox_file_open(mail->open_file, &deleted) <= 0)
return -1;
if (deleted) {
/* either it's expunged now or moved to another file. */
struct mdbox_file *mfile =
(struct mdbox_file *)mail->open_file;
if (mfile->file_id == prev_file_id) {
dbox_mail_set_expunged(mail, map_uid);
return -1;
}
prev_file_id = mfile->file_id;
if (mdbox_map_refresh(mbox->storage->map) < 0)
return -1;
dbox_file_unref(&mail->open_file);
}
} while (mail->open_file == NULL);
*file_r = mail->open_file;
*offset_r = mail->offset;
return 0;
}
int mail_get_hdr_stream(struct mail *mail, struct message_size *hdr_size,
struct istream **stream_r)
{
struct mail_private *p = (struct mail_private *)mail;
int ret;
if (mail->lookup_abort != MAIL_LOOKUP_ABORT_NEVER) {
mail_set_aborted(mail);
return -1;
}
T_BEGIN {
ret = p->v.get_stream(mail, FALSE, hdr_size, NULL, stream_r);
} T_END;
return ret;
}
static int maildir_mail_stat(struct mail *mail, struct stat *st_r)
{
struct maildir_mailbox *mbox = (struct maildir_mailbox *)mail->box;
struct index_mail *imail = (struct index_mail *)mail;
const char *path;
int fd, ret;
if (mail->lookup_abort == MAIL_LOOKUP_ABORT_NOT_IN_CACHE) {
mail_set_aborted(mail);
return -1;
}
if (imail->data.access_part != 0 &&
imail->data.stream == NULL) {
/* we're going to open the mail anyway */
struct istream *input;
(void)mail_get_stream(mail, NULL, NULL, &input);
}
if (imail->data.stream != NULL &&
(fd = i_stream_get_fd(imail->data.stream)) != -1) {
mail->transaction->stats.fstat_lookup_count++;
if (fstat(fd, st_r) < 0) {
mail_storage_set_critical(mail->box->storage,
"fstat(%s) failed: %m",
i_stream_get_name(imail->data.stream));
return -1;
}
} else if (!mail->saving) {
mail->transaction->stats.stat_lookup_count++;
ret = maildir_file_do(mbox, mail->uid, do_stat, st_r);
if (ret <= 0) {
if (ret == 0)
mail_set_expunged(mail);
return -1;
}
} else {
mail->transaction->stats.stat_lookup_count++;
path = maildir_save_file_get_path(mail->transaction, mail->seq);
if (stat(path, st_r) < 0) {
mail_storage_set_critical(mail->box->storage,
"stat(%s) failed: %m", path);
return -1;
}
}
return 0;
}
static int
imapc_mail_get_stream(struct mail *_mail, bool get_body,
struct message_size *hdr_size,
struct message_size *body_size, struct istream **stream_r)
{
struct imapc_mail *mail = (struct imapc_mail *)_mail;
struct index_mail_data *data = &mail->imail.data;
enum mail_fetch_field fetch_field;
if (get_body && !mail->body_fetched &&
mail->imail.data.stream != NULL) {
/* we've fetched the header, but we need the body now too */
index_mail_close_streams(&mail->imail);
}
if (data->stream == NULL) {
if (!data->initialized) {
/* coming here from mail_set_seq() */
mail_set_aborted(_mail);
return -1;
}
fetch_field = get_body ||
(data->access_part & READ_BODY) != 0 ?
MAIL_FETCH_STREAM_BODY : MAIL_FETCH_STREAM_HEADER;
if (imapc_mail_fetch(_mail, fetch_field) < 0)
return -1;
if (data->stream == NULL) {
if (imapc_mail_failed(_mail, "BODY[]") < 0)
return -1;
i_assert(data->stream == NULL);
/* this could be either a temporary server bug, or the
server may permanently just not return anything for
this mail. the latter happens at least with Exchange
when trying to fetch calendar "mails", so we'll just
return them as empty mails instead of disconnecting
the client. */
mail->body_fetched = TRUE;
data->stream = i_stream_create_from_data(&uchar_nul, 0);
imapc_mail_init_stream(mail, TRUE);
}
}
return index_mail_init_stream(&mail->imail, hdr_size, body_size,
stream_r);
}
int mail_get_binary_stream(struct mail *mail, const struct message_part *part,
bool include_hdr, uoff_t *size_r,
bool *binary_r, struct istream **stream_r)
{
struct mail_private *p = (struct mail_private *)mail;
int ret;
if (mail->lookup_abort != MAIL_LOOKUP_ABORT_NEVER) {
mail_set_aborted(mail);
return -1;
}
T_BEGIN {
ret = p->v.get_binary_stream(mail, part, include_hdr,
size_r, NULL, binary_r, stream_r);
} T_END;
return ret;
}
int mail_get_stream_because(struct mail *mail, struct message_size *hdr_size,
struct message_size *body_size,
const char *reason, struct istream **stream_r)
{
struct mail_private *p = (struct mail_private *)mail;
int ret;
if (mail->lookup_abort != MAIL_LOOKUP_ABORT_NEVER) {
mail_set_aborted(mail);
return -1;
}
T_BEGIN {
p->get_stream_reason = reason;
ret = p->v.get_stream(mail, TRUE, hdr_size, body_size, stream_r);
p->get_stream_reason = "";
} T_END;
return ret;
}
static int
imapc_mail_get_stream(struct mail *_mail, bool get_body,
struct message_size *hdr_size,
struct message_size *body_size, struct istream **stream_r)
{
struct imapc_mail *mail = (struct imapc_mail *)_mail;
struct index_mail_data *data = &mail->imail.data;
enum mail_fetch_field fetch_field;
if (get_body && !mail->body_fetched &&
mail->imail.data.stream != NULL) {
/* we've fetched the header, but we need the body now too */
index_mail_close_streams(&mail->imail);
}
if (data->stream == NULL) {
if (!data->initialized) {
/* coming here from mail_set_seq() */
mail_set_aborted(_mail);
return -1;
}
fetch_field = get_body ||
(data->access_part & READ_BODY) != 0 ?
MAIL_FETCH_STREAM_BODY : MAIL_FETCH_STREAM_HEADER;
if (imapc_mail_fetch(_mail, fetch_field, NULL) < 0)
return -1;
if (data->stream == NULL) {
if (imapc_mail_failed(_mail, "BODY[]"))
return -1;
i_assert(data->stream == NULL);
/* return the broken email as empty */
mail->body_fetched = TRUE;
data->stream = i_stream_create_from_data(NULL, 0);
imapc_mail_init_stream(mail);
}
}
return index_mail_init_stream(&mail->imail, hdr_size, body_size,
stream_r);
}
static int cydir_mail_stat(struct mail *mail, struct stat *st_r)
{
struct mail_private *p = (struct mail_private *)mail;
const char *path;
if (mail->lookup_abort == MAIL_LOOKUP_ABORT_NOT_IN_CACHE)
return mail_set_aborted(mail);
p->stats_stat_lookup_count++;
path = cydir_mail_get_path(mail);
if (stat(path, st_r) < 0) {
if (errno == ENOENT)
mail_set_expunged(mail);
else {
mail_storage_set_critical(mail->box->storage,
"stat(%s) failed: %m", path);
}
return -1;
}
return 0;
}
static int raw_mail_stat(struct mail *mail)
{
struct raw_mailbox *mbox = (struct raw_mailbox *)mail->box;
struct mail_private *p = (struct mail_private *)mail;
const struct stat *st;
if (mail->lookup_abort == MAIL_LOOKUP_ABORT_NOT_IN_CACHE)
return mail_set_aborted(mail);
p->stats_fstat_lookup_count++;
st = i_stream_stat(mail->box->input, TRUE);
if (st == NULL) {
mail_storage_set_critical(mail->box->storage,
"stat(%s) failed: %m", mail->box->path);
return -1;
}
if (mbox->mtime != (time_t)-1)
mbox->mtime = st->st_mtime;
if (mbox->ctime != (time_t)-1)
mbox->ctime = st->st_ctime;
mbox->size = st->st_size;
return 0;
}
static int mbox_mail_seek(struct index_mail *mail)
{
struct mbox_transaction_context *t =
(struct mbox_transaction_context *)mail->mail.mail.transaction;
struct mail *_mail = &mail->mail.mail;
struct mbox_mailbox *mbox = (struct mbox_mailbox *)_mail->box;
enum mbox_sync_flags sync_flags = 0;
int ret, try;
bool deleted;
if (_mail->expunged || mbox->syncing)
return -1;
if (_mail->lookup_abort != MAIL_LOOKUP_ABORT_NEVER) {
mail_set_aborted(_mail);
return -1;
}
if (mbox->mbox_stream != NULL &&
istream_raw_mbox_is_corrupted(mbox->mbox_stream)) {
/* clear the corruption by forcing a full resync */
sync_flags |= MBOX_SYNC_UNDIRTY | MBOX_SYNC_FORCE_SYNC;
}
for (try = 0; try < 2; try++) {
if ((sync_flags & MBOX_SYNC_FORCE_SYNC) != 0) {
/* dirty offsets are broken. make sure we can sync. */
mbox_prepare_resync(_mail);
}
if (mbox->mbox_lock_type == F_UNLCK) {
i_assert(t->read_lock_id == 0);
sync_flags |= MBOX_SYNC_LOCK_READING;
if (mbox_sync(mbox, sync_flags) < 0)
return -1;
t->read_lock_id = mbox_get_cur_lock_id(mbox);
i_assert(t->read_lock_id != 0);
/* refresh index file after mbox has been locked to
make sure we get only up-to-date mbox offsets. */
if (mail_index_refresh(mbox->box.index) < 0) {
mailbox_set_index_error(&mbox->box);
return -1;
}
i_assert(mbox->mbox_lock_type != F_UNLCK);
} else if (t->read_lock_id == 0) {
/* file is already locked by another transaction, but
we must keep it locked for the entire transaction,
so increase the lock counter. */
if (mbox_lock(mbox, mbox->mbox_lock_type,
&t->read_lock_id) < 0)
i_unreached();
}
if (mbox_file_open_stream(mbox) < 0)
return -1;
ret = mbox_file_seek(mbox, _mail->transaction->view,
_mail->seq, &deleted);
if (ret > 0) {
/* success */
break;
}
if (ret < 0) {
if (deleted)
mail_set_expunged(_mail);
return -1;
}
/* we'll need to re-sync it completely */
sync_flags |= MBOX_SYNC_UNDIRTY | MBOX_SYNC_FORCE_SYNC;
}
if (ret == 0) {
mail_storage_set_critical(&mbox->storage->storage,
"Losing sync for mail uid=%u in mbox file %s",
_mail->uid, mailbox_get_path(&mbox->box));
}
return 0;
}
static int mbox_mail_get_received_date(struct mail *_mail, time_t *date_r)
{
struct index_mail *mail = (struct index_mail *)_mail;
struct index_mail_data *data = &mail->data;
struct mbox_mailbox *mbox = (struct mbox_mailbox *)_mail->box;
if (index_mail_get_received_date(_mail, date_r) == 0)
return 0;
if (mbox_mail_seek(mail) < 0)
return -1;
data->received_date =
istream_raw_mbox_get_received_time(mbox->mbox_stream);
if (data->received_date == (time_t)-1) {
/* it's broken and conflicts with our "not found"
return value. change it. */
data->received_date = 0;
}
*date_r = data->received_date;
return 0;
}
static int mbox_mail_get_save_date(struct mail *_mail, time_t *date_r)
{
struct index_mail *mail = (struct index_mail *)_mail;
struct index_mail_data *data = &mail->data;
if (index_mail_get_save_date(_mail, date_r) == 0)
return 0;
/* no way to know this. save the current time into cache and use
that from now on. this works only as long as the index files
are permanent */
data->save_date = ioloop_time;
*date_r = data->save_date;
return 0;
}
static int
mbox_mail_get_md5_header(struct index_mail *mail, const char **value_r)
{
struct mail *_mail = &mail->mail.mail;
static uint8_t empty_md5[16] =
{ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 };
struct mbox_mailbox *mbox = (struct mbox_mailbox *)_mail->box;
const void *ext_data;
if (mail->data.guid != NULL) {
*value_r = mail->data.guid;
return 1;
}
mail_index_lookup_ext(_mail->transaction->view, _mail->seq,
mbox->md5hdr_ext_idx, &ext_data, NULL);
if (ext_data != NULL && memcmp(ext_data, empty_md5, 16) != 0) {
mail->data.guid = p_strdup(mail->mail.data_pool,
binary_to_hex(ext_data, 16));
*value_r = mail->data.guid;
return 1;
} else if (mail_index_is_expunged(_mail->transaction->view, _mail->seq)) {
mail_set_expunged(_mail);
return -1;
} else {
return 0;
}
}
static int
mbox_mail_get_special(struct mail *_mail, enum mail_fetch_field field,
const char **value_r)
{
struct index_mail *mail = (struct index_mail *)_mail;
struct mbox_mailbox *mbox = (struct mbox_mailbox *)_mail->box;
uoff_t offset;
bool move_offset;
int ret;
switch (field) {
case MAIL_FETCH_FROM_ENVELOPE:
if (mbox_mail_seek(mail) < 0)
return -1;
*value_r = istream_raw_mbox_get_sender(mbox->mbox_stream);
return 0;
case MAIL_FETCH_GUID:
case MAIL_FETCH_HEADER_MD5:
if ((ret = mbox_mail_get_md5_header(mail, value_r)) != 0)
return ret < 0 ? -1 : 0;
/* i guess in theory the empty_md5 is valid and can happen,
but it's almost guaranteed that it means the MD5 sum is
missing. recalculate it. */
if (mbox->mbox_lock_type == F_UNLCK ||
mbox->mbox_stream == NULL) {
offset = 0;
move_offset = FALSE;
} else {
offset = istream_raw_mbox_get_start_offset(mbox->mbox_stream);
move_offset = TRUE;
}
mbox->mbox_save_md5 = TRUE;
if (mbox_sync(mbox, MBOX_SYNC_FORCE_SYNC |
MBOX_SYNC_READONLY) < 0)
return -1;
if (move_offset) {
if (istream_raw_mbox_seek(mbox->mbox_stream,
offset) < 0) {
i_error("mbox %s sync lost during MD5 syncing",
_mail->box->name);
return -1;
}
}
if ((ret = mbox_mail_get_md5_header(mail, value_r)) == 0) {
i_error("mbox %s resyncing didn't save header MD5 values",
_mail->box->name);
return -1;
}
return ret < 0 ? -1 : 0;
default:
break;
}
return index_mail_get_special(_mail, field, value_r);
}
static int
mbox_mail_get_next_offset(struct index_mail *mail, uoff_t *next_offset_r)
{
struct mbox_mailbox *mbox = (struct mbox_mailbox *)mail->mail.mail.box;
struct mail_index_view *view;
const struct mail_index_header *hdr;
uint32_t seq;
int trailer_size;
int ret = 1;
*next_offset_r = (uoff_t)-1;
hdr = mail_index_get_header(mail->mail.mail.transaction->view);
if (mail->mail.mail.seq > hdr->messages_count) {
/* we're appending a new message */
return 0;
}
/* We can't really trust trans_view. The next message may already be
expunged from it. Also hdr.messages_count may be incorrect there.
So refresh the index to get the latest changes and get the next
message's offset using a new view. */
i_assert(mbox->mbox_lock_type != F_UNLCK);
if (mbox_sync_header_refresh(mbox) < 0)
return -1;
view = mail_index_view_open(mail->mail.mail.box->index);
hdr = mail_index_get_header(view);
if (!mail_index_lookup_seq(view, mail->mail.mail.uid, &seq))
i_panic("Message unexpectedly expunged from index");
if (seq < hdr->messages_count) {
if (mbox_file_lookup_offset(mbox, view, seq + 1,
next_offset_r) <= 0)
ret = -1;
} else if (mail->mail.mail.box->input != NULL) {
/* opened the mailbox as input stream. we can't trust the
sync_size, since it's wrong with compressed mailboxes */
ret = 0;
} else {
/* last message, use the synced mbox size */
trailer_size =
mbox->storage->storage.set->mail_save_crlf ? 2 : 1;
*next_offset_r = mbox->mbox_hdr.sync_size - trailer_size;
}
mail_index_view_close(&view);
return ret;
}
static int mbox_mail_get_physical_size(struct mail *_mail, uoff_t *size_r)
{
struct index_mail *mail = (struct index_mail *)_mail;
struct index_mail_data *data = &mail->data;
struct mbox_mailbox *mbox = (struct mbox_mailbox *)_mail->box;
struct istream *input;
struct message_size hdr_size;
uoff_t old_offset, body_offset, body_size, next_offset;
if (index_mail_get_physical_size(_mail, size_r) == 0)
return 0;
/* we want to return the header size as seen by mail_get_stream(). */
old_offset = data->stream == NULL ? 0 : data->stream->v_offset;
if (mail_get_stream(_mail, &hdr_size, NULL, &input) < 0)
return -1;
/* our header size varies, so don't do any caching */
if (istream_raw_mbox_get_body_offset(mbox->mbox_stream, &body_offset) < 0) {
mail_storage_set_critical(_mail->box->storage,
"mbox %s: Couldn't get body offset for uid=%u",
mailbox_get_path(&mbox->box), mail->mail.mail.uid);
return -1;
}
/* use the next message's offset to avoid reading through the entire
message body to find out its size */
if (mbox_mail_get_next_offset(mail, &next_offset) > 0)
body_size = next_offset - body_offset;
else
body_size = (uoff_t)-1;
/* verify that the calculated body size is correct */
if (istream_raw_mbox_get_body_size(mbox->mbox_stream,
body_size, &body_size) < 0) {
mail_storage_set_critical(_mail->box->storage,
"mbox %s: Couldn't get body size for uid=%u",
mailbox_get_path(&mbox->box), mail->mail.mail.uid);
return -1;
}
data->physical_size = hdr_size.physical_size + body_size;
*size_r = data->physical_size;
i_stream_seek(input, old_offset);
return 0;
}
static int mbox_mail_init_stream(struct index_mail *mail)
{
struct mbox_mailbox *mbox = (struct mbox_mailbox *)mail->mail.mail.box;
struct istream *raw_stream;
uoff_t hdr_offset, next_offset;
int ret;
if (mbox_mail_seek(mail) < 0)
return -1;
ret = mbox_mail_get_next_offset(mail, &next_offset);
if (ret < 0) {
if (mbox_mail_seek(mail) < 0)
return -1;
ret = mbox_mail_get_next_offset(mail, &next_offset);
if (ret < 0) {
i_warning("mbox %s: Can't find next message offset "
"for uid=%u", mailbox_get_path(&mbox->box),
mail->mail.mail.uid);
}
}
raw_stream = mbox->mbox_stream;
if (istream_raw_mbox_get_header_offset(raw_stream, &hdr_offset) < 0) {
mail_storage_set_critical(mbox->box.storage,
"mbox %s: Couldn't get header offset for uid=%u",
mailbox_get_path(&mbox->box), mail->mail.mail.uid);
return -1;
}
i_stream_seek(raw_stream, hdr_offset);
if (next_offset != (uoff_t)-1)
istream_raw_mbox_set_next_offset(raw_stream, next_offset);
raw_stream = i_stream_create_limit(raw_stream, (uoff_t)-1);
mail->data.stream =
i_stream_create_header_filter(raw_stream,
HEADER_FILTER_EXCLUDE | HEADER_FILTER_NO_CR,
mbox_hide_headers, mbox_hide_headers_count,
*null_header_filter_callback, (void *)NULL);
i_stream_unref(&raw_stream);
return 0;
}
static int mbox_mail_get_stream(struct mail *_mail, bool get_body ATTR_UNUSED,
struct message_size *hdr_size,
struct message_size *body_size,
struct istream **stream_r)
{
struct index_mail *mail = (struct index_mail *)_mail;
if (mail->data.stream == NULL) {
if (mbox_mail_init_stream(mail) < 0)
return -1;
}
return index_mail_init_stream(mail, hdr_size, body_size, stream_r);
}
static void mbox_mail_set_seq(struct mail *_mail, uint32_t seq, bool saving)
{
struct index_mail *mail = (struct index_mail *)_mail;
index_mail_set_seq(_mail, seq, saving);
mail->data.dont_cache_fetch_fields |= MAIL_FETCH_PHYSICAL_SIZE;
}
static bool mbox_mail_set_uid(struct mail *_mail, uint32_t uid)
{
struct index_mail *mail = (struct index_mail *)_mail;
bool ret;
ret = index_mail_set_uid(_mail, uid);
mail->data.dont_cache_fetch_fields |= MAIL_FETCH_PHYSICAL_SIZE;
return ret;
}
struct mail_vfuncs mbox_mail_vfuncs = {
index_mail_close,
index_mail_free,
mbox_mail_set_seq,
mbox_mail_set_uid,
index_mail_set_uid_cache_updates,
index_mail_prefetch,
index_mail_precache,
index_mail_add_temp_wanted_fields,
index_mail_get_flags,
index_mail_get_keywords,
index_mail_get_keyword_indexes,
index_mail_get_modseq,
index_mail_get_pvt_modseq,
index_mail_get_parts,
index_mail_get_date,
mbox_mail_get_received_date,
mbox_mail_get_save_date,
index_mail_get_virtual_size,
mbox_mail_get_physical_size,
index_mail_get_first_header,
index_mail_get_headers,
index_mail_get_header_stream,
mbox_mail_get_stream,
index_mail_get_binary_stream,
mbox_mail_get_special,
index_mail_get_real_mail,
index_mail_update_flags,
index_mail_update_keywords,
index_mail_update_modseq,
index_mail_update_pvt_modseq,
NULL,
index_mail_expunge,
index_mail_set_cache_corrupted,
index_mail_opened
};
