int index_storage_mailbox_open(struct mailbox *box, bool move_to_memory)
{
struct index_mailbox_context *ibox = INDEX_STORAGE_CONTEXT(box);
enum mail_index_open_flags index_flags;
int ret;
i_assert(!box->opened);
index_flags = ibox->index_flags;
if (move_to_memory)
ibox->index_flags &= ~MAIL_INDEX_OPEN_FLAG_CREATE;
if (index_storage_mailbox_alloc_index(box) < 0)
return -1;
/* make sure mail_index_set_permissions() has been called */
(void)mailbox_get_permissions(box);
ret = mail_index_open(box->index, index_flags);
if (ret <= 0 || move_to_memory) {
if ((index_flags & MAIL_INDEX_OPEN_FLAG_NEVER_IN_MEMORY) != 0) {
i_assert(ret <= 0);
mailbox_set_index_error(box);
return -1;
}
if (mail_index_move_to_memory(box->index) < 0) {
/* try opening once more. it should be created
directly into memory now. */
if (mail_index_open_or_create(box->index,
index_flags) < 0)
i_panic("in-memory index creation failed");
}
}
if ((index_flags & MAIL_INDEX_OPEN_FLAG_NEVER_IN_MEMORY) != 0) {
if (mail_index_is_in_memory(box->index)) {
mail_storage_set_critical(box->storage,
"Couldn't create index file");
mail_index_close(box->index);
return -1;
}
}
if ((box->flags & MAILBOX_FLAG_OPEN_DELETED) == 0) {
if (mail_index_is_deleted(box->index)) {
mailbox_set_deleted(box);
mail_index_close(box->index);
return -1;
}
}
box->cache = mail_index_get_cache(box->index);
index_cache_register_defaults(box);
box->view = mail_index_view_open(box->index);
ibox->keyword_names = mail_index_get_keywords(box->index);
box->vsize_hdr_ext_id =
mail_index_ext_register(box->index, "hdr-vsize",
sizeof(struct mailbox_index_vsize), 0,
sizeof(uint64_t));
box->opened = TRUE;
if ((box->enabled_features & MAILBOX_FEATURE_CONDSTORE) != 0)
mail_index_modseq_enable(box->index);
index_thread_mailbox_opened(box);
hook_mailbox_opened(box);
return 0;
}
int mdbox_sync_begin(struct mdbox_mailbox *mbox, enum mdbox_sync_flags flags,
struct mdbox_map_atomic_context *atomic,
struct mdbox_sync_context **ctx_r)
{
struct mail_storage *storage = mbox->box.storage;
struct mdbox_sync_context *ctx;
enum mail_index_sync_flags sync_flags;
int ret;
bool rebuild, storage_rebuilt = FALSE;
*ctx_r = NULL;
/* avoid race conditions with mailbox creation, don't check for dbox
headers until syncing has locked the mailbox */
rebuild = mbox->storage->corrupted ||
(flags & MDBOX_SYNC_FLAG_FORCE_REBUILD) != 0;
if (rebuild && (flags & MDBOX_SYNC_FLAG_NO_REBUILD) == 0) {
if (mdbox_storage_rebuild_in_context(mbox->storage, atomic) < 0)
return -1;
index_mailbox_reset_uidvalidity(&mbox->box);
storage_rebuilt = TRUE;
}
ctx = i_new(struct mdbox_sync_context, 1);
ctx->mbox = mbox;
ctx->flags = flags;
ctx->atomic = atomic;
sync_flags = index_storage_get_sync_flags(&mbox->box);
if (!rebuild && (flags & MDBOX_SYNC_FLAG_FORCE) == 0)
sync_flags |= MAIL_INDEX_SYNC_FLAG_REQUIRE_CHANGES;
if ((flags & MDBOX_SYNC_FLAG_FSYNC) != 0)
sync_flags |= MAIL_INDEX_SYNC_FLAG_FSYNC;
/* don't write unnecessary dirty flag updates */
sync_flags |= MAIL_INDEX_SYNC_FLAG_AVOID_FLAG_UPDATES;
ret = mdbox_sync_try_begin(ctx, sync_flags);
if (ret <= 0) {
/* failed / nothing to do */
i_free(ctx);
return ret;
}
if ((ret = mdbox_sync_index(ctx)) <= 0) {
mail_index_sync_rollback(&ctx->index_sync_ctx);
i_free_and_null(ctx);
if (ret < 0)
return -1;
/* corrupted */
if (storage_rebuilt) {
mail_storage_set_critical(storage,
"mdbox %s: Storage keeps breaking",
mailbox_get_path(&mbox->box));
return -1;
}
/* we'll need to rebuild storage.
try again from the beginning. */
mdbox_storage_set_corrupted(mbox->storage);
if ((flags & MDBOX_SYNC_FLAG_NO_REBUILD) != 0) {
mail_storage_set_critical(storage,
"mdbox %s: Can't rebuild storage",
mailbox_get_path(&mbox->box));
return -1;
}
return mdbox_sync_begin(mbox, flags, atomic, ctx_r);
}
*ctx_r = ctx;
return 0;
}
static int ATTR_NULL(2) ATTR_NOWARN_UNUSED_RESULT
mbox_dotlock_privileged_op(struct mbox_mailbox *mbox,
struct dotlock_settings *set,
enum mbox_dotlock_op op)
{
const char *box_path, *dir, *fname;
int ret = -1, orig_dir_fd, orig_errno;
orig_dir_fd = open(".", O_RDONLY);
if (orig_dir_fd == -1) {
mail_storage_set_critical(&mbox->storage->storage,
"open(.) failed: %m");
return -1;
}
/* allow dotlocks to be created only for files we can read while we're
unprivileged. to make sure there are no race conditions we first
have to chdir to the mbox file's directory and then use relative
paths. unless this is done, users could:
- create *.lock files to any directory writable by the
privileged group
- DoS other users by dotlocking their mailboxes infinitely
*/
box_path = mailbox_get_path(&mbox->box);
fname = strrchr(box_path, '/');
if (fname == NULL) {
/* already relative */
fname = box_path;
} else {
dir = t_strdup_until(box_path, fname);
if (chdir(dir) < 0) {
mail_storage_set_critical(&mbox->storage->storage,
"chdir(%s) failed: %m", dir);
i_close_fd(&orig_dir_fd);
return -1;
}
fname++;
}
if (op == MBOX_DOTLOCK_OP_LOCK) {
if (access(fname, R_OK) < 0) {
mail_storage_set_critical(&mbox->storage->storage,
"access(%s) failed: %m", box_path);
i_close_fd(&orig_dir_fd);
return -1;
}
}
if (restrict_access_use_priv_gid() < 0) {
i_close_fd(&orig_dir_fd);
return -1;
}
switch (op) {
case MBOX_DOTLOCK_OP_LOCK:
/* we're now privileged - avoid doing as much as possible */
ret = file_dotlock_create(set, fname, 0, &mbox->mbox_dotlock);
if (ret > 0)
mbox->mbox_used_privileges = TRUE;
else if (ret < 0 && errno == EACCES) {
const char *errmsg =
eacces_error_get_creating("file_dotlock_create",
fname);
mail_storage_set_critical(&mbox->storage->storage,
"%s", errmsg);
} else {
mbox_set_syscall_error(mbox, "file_dotlock_create()");
}
break;
case MBOX_DOTLOCK_OP_UNLOCK:
/* we're now privileged - avoid doing as much as possible */
ret = file_dotlock_delete(&mbox->mbox_dotlock);
if (ret < 0)
mbox_set_syscall_error(mbox, "file_dotlock_delete()");
mbox->mbox_used_privileges = FALSE;
break;
case MBOX_DOTLOCK_OP_TOUCH:
ret = file_dotlock_touch(mbox->mbox_dotlock);
if (ret < 0)
mbox_set_syscall_error(mbox, "file_dotlock_touch()");
break;
}
orig_errno = errno;
restrict_access_drop_priv_gid();
if (fchdir(orig_dir_fd) < 0) {
mail_storage_set_critical(&mbox->storage->storage,
"fchdir() failed: %m");
}
i_close_fd(&orig_dir_fd);
errno = orig_errno;
return ret;
}
static int mdbox_sync_index(struct mdbox_sync_context *ctx)
{
struct mailbox *box = &ctx->mbox->box;
const struct mail_index_header *hdr;
struct mail_index_sync_rec sync_rec;
uint32_t seq1, seq2;
int ret = 0;
hdr = mail_index_get_header(ctx->sync_view);
if (hdr->uid_validity == 0) {
/* newly created index file */
if (hdr->next_uid == 1) {
/* could be just a race condition where we opened the
mailbox between mkdir and index creation. fix this
silently. */
if (mdbox_mailbox_create_indexes(box, NULL, ctx->trans) < 0)
return -1;
return 1;
}
mail_storage_set_critical(box->storage,
"Mailbox %s: Broken index: missing UIDVALIDITY",
box->vname);
return 0;
}
/* mark the newly seen messages as recent */
if (mail_index_lookup_seq_range(ctx->sync_view, hdr->first_recent_uid,
hdr->next_uid, &seq1, &seq2)) {
index_mailbox_set_recent_seq(&ctx->mbox->box, ctx->sync_view,
seq1, seq2);
}
/* handle syncing records without map being locked. */
if (mdbox_map_atomic_is_locked(ctx->atomic)) {
ctx->map_trans = mdbox_map_transaction_begin(ctx->atomic, FALSE);
i_array_init(&ctx->expunged_seqs, 64);
}
while (mail_index_sync_next(ctx->index_sync_ctx, &sync_rec)) {
if ((ret = mdbox_sync_rec(ctx, &sync_rec)) < 0)
break;
}
/* write refcount changes to map index. transaction commit updates the
log head, while tail is left behind. */
if (mdbox_map_atomic_is_locked(ctx->atomic)) {
if (ret == 0)
ret = mdbox_map_transaction_commit(ctx->map_trans);
/* write changes to mailbox index */
if (ret == 0)
ret = dbox_sync_mark_expunges(ctx);
/* finish the map changes and unlock the map. this also updates
map's tail -> head. */
if (ret < 0)
mdbox_map_atomic_set_failed(ctx->atomic);
mdbox_map_transaction_free(&ctx->map_trans);
array_free(&ctx->expunged_seqs);
}
if (box->v.sync_notify != NULL)
box->v.sync_notify(box, 0, 0);
return ret == 0 ? 1 :
(ctx->mbox->storage->corrupted ? 0 : -1);
}
static int maildir_save_finish_real(struct mail_save_context *_ctx)
{
struct maildir_save_context *ctx = (struct maildir_save_context *)_ctx;
struct mail_storage *storage = &ctx->mbox->storage->storage;
const char *path;
off_t real_size;
uoff_t size;
int output_errno;
ctx->last_save_finished = TRUE;
if (ctx->failed && ctx->fd == -1) {
/* tmp file creation failed */
return -1;
}
path = t_strconcat(ctx->tmpdir, "/", ctx->file_last->tmp_name, NULL);
if (!ctx->failed && o_stream_nfinish(_ctx->data.output) < 0) {
if (!mail_storage_set_error_from_errno(storage)) {
mail_storage_set_critical(storage,
"write(%s) failed: %m", path);
}
ctx->failed = TRUE;
}
if (_ctx->data.save_date != (time_t)-1) {
/* we can't change ctime, but we can add the date to cache */
struct index_mail *mail = (struct index_mail *)_ctx->dest_mail;
uint32_t t = _ctx->data.save_date;
index_mail_cache_add(mail, MAIL_CACHE_SAVE_DATE, &t, sizeof(t));
}
if (maildir_save_finish_received_date(ctx, path) < 0)
ctx->failed = TRUE;
if (ctx->cur_dest_mail != NULL) {
index_mail_cache_parse_deinit(ctx->cur_dest_mail,
ctx->ctx.data.received_date,
!ctx->failed);
}
i_stream_unref(&ctx->input);
/* remember the size in case we want to add it to filename */
ctx->file_last->size = _ctx->data.output->offset;
if (ctx->cur_dest_mail == NULL ||
mail_get_virtual_size(ctx->cur_dest_mail,
&ctx->file_last->vsize) < 0)
ctx->file_last->vsize = (uoff_t)-1;
output_errno = _ctx->data.output->last_failed_errno;
o_stream_destroy(&_ctx->data.output);
if (storage->set->parsed_fsync_mode != FSYNC_MODE_NEVER &&
!ctx->failed) {
if (fsync(ctx->fd) < 0) {
if (!mail_storage_set_error_from_errno(storage)) {
mail_storage_set_critical(storage,
"fsync(%s) failed: %m", path);
}
ctx->failed = TRUE;
}
}
real_size = lseek(ctx->fd, 0, SEEK_END);
if (real_size == (off_t)-1) {
mail_storage_set_critical(storage,
"lseek(%s) failed: %m", path);
} else if (real_size != (off_t)ctx->file_last->size &&
(!maildir_filename_get_size(ctx->file_last->dest_basename,
MAILDIR_EXTRA_FILE_SIZE, &size) ||
size != ctx->file_last->size)) {
/* e.g. zlib plugin was used. the "physical size" must be in
the maildir filename, since stat() will return wrong size */
ctx->file_last->preserve_filename = FALSE;
/* preserve the GUID if needed */
if (ctx->file_last->guid == NULL)
ctx->file_last->guid = ctx->file_last->dest_basename;
/* reset the base name as well, just in case there's a
,W=vsize */
ctx->file_last->dest_basename = ctx->file_last->tmp_name;
}
if (close(ctx->fd) < 0) {
if (!mail_storage_set_error_from_errno(storage)) {
mail_storage_set_critical(storage,
"close(%s) failed: %m", path);
}
ctx->failed = TRUE;
}
ctx->fd = -1;
if (ctx->failed) {
/* delete the tmp file */
if (unlink(path) < 0 && errno != ENOENT) {
mail_storage_set_critical(storage,
"unlink(%s) failed: %m", path);
}
errno = output_errno;
if (ENOQUOTA(errno)) {
mail_storage_set_error(storage,
MAIL_ERROR_NOQUOTA, MAIL_ERRSTR_NO_QUOTA);
} else if (errno != 0) {
mail_storage_set_critical(storage,
"write(%s) failed: %m", path);
}
maildir_save_remove_last_filename(ctx);
return -1;
}
ctx->file_last = NULL;
return 0;
}
static int
cmd_append_catenate_mpurl(struct client_command_context *cmd,
const char *caturl, struct imap_msgpart_url *mpurl)
{
struct cmd_append_context *ctx = cmd->context;
struct imap_msgpart_open_result mpresult;
uoff_t newsize;
const char *error;
int ret;
/* catenate URL */
ret = imap_msgpart_url_read_part(mpurl, &mpresult, &error);
if (ret < 0) {
client_send_box_error(cmd, ctx->box);
return -1;
}
if (ret == 0) {
/* invalid url, abort */
client_send_tagline(cmd,
t_strdup_printf("NO [BADURL %s] %s.", caturl, error));
return -1;
}
if (mpresult.size == 0) {
/* empty input */
return 0;
}
newsize = ctx->cat_msg_size + mpresult.size;
if (newsize < ctx->cat_msg_size) {
client_send_tagline(cmd,
"NO [TOOBIG] Composed message grows too big.");
return -1;
}
ctx->cat_msg_size = newsize;
/* add this input stream to chain */
i_stream_chain_append(ctx->catchain, mpresult.input);
/* save by reading the chain stream */
while (!i_stream_is_eof(mpresult.input)) {
ret = i_stream_read(mpresult.input);
i_assert(ret != 0); /* we can handle only blocking input here */
if (mailbox_save_continue(ctx->save_ctx) < 0 || ret == -1)
break;
}
if (mpresult.input->stream_errno != 0) {
errno = mpresult.input->stream_errno;
mail_storage_set_critical(ctx->box->storage,
"read(%s) failed: %s (for CATENATE URL %s)",
i_stream_get_name(mpresult.input),
i_stream_get_error(mpresult.input), caturl);
client_send_box_error(cmd, ctx->box);
ret = -1;
} else if (!mpresult.input->eof) {
/* save failed */
client_send_box_error(cmd, ctx->box);
ret = -1;
} else {
/* all the input must be consumed, so istream-chain's read()
unreferences the stream and we can free its parent mail */
i_assert(!i_stream_have_bytes_left(mpresult.input));
ret = 0;
}
return ret;
}
static int mbox_mail_seek(struct index_mail *mail)
{
struct mbox_transaction_context *t =
(struct mbox_transaction_context *)mail->mail.mail.transaction;
struct mail *_mail = &mail->mail.mail;
struct mbox_mailbox *mbox = (struct mbox_mailbox *)_mail->box;
enum mbox_sync_flags sync_flags = 0;
int ret, try;
bool deleted;
if (_mail->expunged || mbox->syncing)
return -1;
if (_mail->lookup_abort != MAIL_LOOKUP_ABORT_NEVER) {
mail_set_aborted(_mail);
return -1;
}
if (mbox->mbox_stream != NULL &&
istream_raw_mbox_is_corrupted(mbox->mbox_stream)) {
/* clear the corruption by forcing a full resync */
sync_flags |= MBOX_SYNC_UNDIRTY | MBOX_SYNC_FORCE_SYNC;
}
for (try = 0; try < 2; try++) {
if ((sync_flags & MBOX_SYNC_FORCE_SYNC) != 0) {
/* dirty offsets are broken. make sure we can sync. */
mbox_prepare_resync(_mail);
}
if (mbox->mbox_lock_type == F_UNLCK) {
i_assert(t->read_lock_id == 0);
sync_flags |= MBOX_SYNC_LOCK_READING;
if (mbox_sync(mbox, sync_flags) < 0)
return -1;
t->read_lock_id = mbox_get_cur_lock_id(mbox);
i_assert(t->read_lock_id != 0);
/* refresh index file after mbox has been locked to
make sure we get only up-to-date mbox offsets. */
if (mail_index_refresh(mbox->box.index) < 0) {
mailbox_set_index_error(&mbox->box);
return -1;
}
i_assert(mbox->mbox_lock_type != F_UNLCK);
} else if (t->read_lock_id == 0) {
/* file is already locked by another transaction, but
we must keep it locked for the entire transaction,
so increase the lock counter. */
if (mbox_lock(mbox, mbox->mbox_lock_type,
&t->read_lock_id) < 0)
i_unreached();
}
if (mbox_file_open_stream(mbox) < 0)
return -1;
ret = mbox_file_seek(mbox, _mail->transaction->view,
_mail->seq, &deleted);
if (ret > 0) {
/* success */
break;
}
if (ret < 0) {
if (deleted)
mail_set_expunged(_mail);
return -1;
}
/* we'll need to re-sync it completely */
sync_flags |= MBOX_SYNC_UNDIRTY | MBOX_SYNC_FORCE_SYNC;
}
if (ret == 0) {
mail_storage_set_critical(&mbox->storage->storage,
"Losing sync for mail uid=%u in mbox file %s",
_mail->uid, mailbox_get_path(&mbox->box));
}
return 0;
}
static int mbox_mail_get_received_date(struct mail *_mail, time_t *date_r)
{
struct index_mail *mail = (struct index_mail *)_mail;
struct index_mail_data *data = &mail->data;
struct mbox_mailbox *mbox = (struct mbox_mailbox *)_mail->box;
if (index_mail_get_received_date(_mail, date_r) == 0)
return 0;
if (mbox_mail_seek(mail) < 0)
return -1;
data->received_date =
istream_raw_mbox_get_received_time(mbox->mbox_stream);
if (data->received_date == (time_t)-1) {
/* it's broken and conflicts with our "not found"
return value. change it. */
data->received_date = 0;
}
*date_r = data->received_date;
return 0;
}
static int mbox_mail_get_save_date(struct mail *_mail, time_t *date_r)
{
struct index_mail *mail = (struct index_mail *)_mail;
struct index_mail_data *data = &mail->data;
if (index_mail_get_save_date(_mail, date_r) == 0)
return 0;
/* no way to know this. save the current time into cache and use
that from now on. this works only as long as the index files
are permanent */
data->save_date = ioloop_time;
*date_r = data->save_date;
return 0;
}
static int
mbox_mail_get_md5_header(struct index_mail *mail, const char **value_r)
{
struct mail *_mail = &mail->mail.mail;
static uint8_t empty_md5[16] =
{ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 };
struct mbox_mailbox *mbox = (struct mbox_mailbox *)_mail->box;
const void *ext_data;
if (mail->data.guid != NULL) {
*value_r = mail->data.guid;
return 1;
}
mail_index_lookup_ext(_mail->transaction->view, _mail->seq,
mbox->md5hdr_ext_idx, &ext_data, NULL);
if (ext_data != NULL && memcmp(ext_data, empty_md5, 16) != 0) {
mail->data.guid = p_strdup(mail->mail.data_pool,
binary_to_hex(ext_data, 16));
*value_r = mail->data.guid;
return 1;
} else if (mail_index_is_expunged(_mail->transaction->view, _mail->seq)) {
mail_set_expunged(_mail);
return -1;
} else {
return 0;
}
}
static int
mbox_mail_get_special(struct mail *_mail, enum mail_fetch_field field,
const char **value_r)
{
struct index_mail *mail = (struct index_mail *)_mail;
struct mbox_mailbox *mbox = (struct mbox_mailbox *)_mail->box;
uoff_t offset;
bool move_offset;
int ret;
switch (field) {
case MAIL_FETCH_FROM_ENVELOPE:
if (mbox_mail_seek(mail) < 0)
return -1;
*value_r = istream_raw_mbox_get_sender(mbox->mbox_stream);
return 0;
case MAIL_FETCH_GUID:
case MAIL_FETCH_HEADER_MD5:
if ((ret = mbox_mail_get_md5_header(mail, value_r)) != 0)
return ret < 0 ? -1 : 0;
/* i guess in theory the empty_md5 is valid and can happen,
but it's almost guaranteed that it means the MD5 sum is
missing. recalculate it. */
if (mbox->mbox_lock_type == F_UNLCK ||
mbox->mbox_stream == NULL) {
offset = 0;
move_offset = FALSE;
} else {
offset = istream_raw_mbox_get_start_offset(mbox->mbox_stream);
move_offset = TRUE;
}
mbox->mbox_save_md5 = TRUE;
if (mbox_sync(mbox, MBOX_SYNC_FORCE_SYNC |
MBOX_SYNC_READONLY) < 0)
return -1;
if (move_offset) {
if (istream_raw_mbox_seek(mbox->mbox_stream,
offset) < 0) {
i_error("mbox %s sync lost during MD5 syncing",
_mail->box->name);
return -1;
}
}
if ((ret = mbox_mail_get_md5_header(mail, value_r)) == 0) {
i_error("mbox %s resyncing didn't save header MD5 values",
_mail->box->name);
return -1;
}
return ret < 0 ? -1 : 0;
default:
break;
}
return index_mail_get_special(_mail, field, value_r);
}
static int
mbox_mail_get_next_offset(struct index_mail *mail, uoff_t *next_offset_r)
{
struct mbox_mailbox *mbox = (struct mbox_mailbox *)mail->mail.mail.box;
struct mail_index_view *view;
const struct mail_index_header *hdr;
uint32_t seq;
int trailer_size;
int ret = 1;
*next_offset_r = (uoff_t)-1;
hdr = mail_index_get_header(mail->mail.mail.transaction->view);
if (mail->mail.mail.seq > hdr->messages_count) {
/* we're appending a new message */
return 0;
}
/* We can't really trust trans_view. The next message may already be
expunged from it. Also hdr.messages_count may be incorrect there.
So refresh the index to get the latest changes and get the next
message's offset using a new view. */
i_assert(mbox->mbox_lock_type != F_UNLCK);
if (mbox_sync_header_refresh(mbox) < 0)
return -1;
view = mail_index_view_open(mail->mail.mail.box->index);
hdr = mail_index_get_header(view);
if (!mail_index_lookup_seq(view, mail->mail.mail.uid, &seq))
i_panic("Message unexpectedly expunged from index");
if (seq < hdr->messages_count) {
if (mbox_file_lookup_offset(mbox, view, seq + 1,
next_offset_r) <= 0)
ret = -1;
} else if (mail->mail.mail.box->input != NULL) {
/* opened the mailbox as input stream. we can't trust the
sync_size, since it's wrong with compressed mailboxes */
ret = 0;
} else {
/* last message, use the synced mbox size */
trailer_size =
mbox->storage->storage.set->mail_save_crlf ? 2 : 1;
*next_offset_r = mbox->mbox_hdr.sync_size - trailer_size;
}
mail_index_view_close(&view);
return ret;
}
static int mbox_mail_get_physical_size(struct mail *_mail, uoff_t *size_r)
{
struct index_mail *mail = (struct index_mail *)_mail;
struct index_mail_data *data = &mail->data;
struct mbox_mailbox *mbox = (struct mbox_mailbox *)_mail->box;
struct istream *input;
struct message_size hdr_size;
uoff_t old_offset, body_offset, body_size, next_offset;
if (index_mail_get_physical_size(_mail, size_r) == 0)
return 0;
/* we want to return the header size as seen by mail_get_stream(). */
old_offset = data->stream == NULL ? 0 : data->stream->v_offset;
if (mail_get_stream(_mail, &hdr_size, NULL, &input) < 0)
return -1;
/* our header size varies, so don't do any caching */
if (istream_raw_mbox_get_body_offset(mbox->mbox_stream, &body_offset) < 0) {
mail_storage_set_critical(_mail->box->storage,
"mbox %s: Couldn't get body offset for uid=%u",
mailbox_get_path(&mbox->box), mail->mail.mail.uid);
return -1;
}
/* use the next message's offset to avoid reading through the entire
message body to find out its size */
if (mbox_mail_get_next_offset(mail, &next_offset) > 0)
body_size = next_offset - body_offset;
else
body_size = (uoff_t)-1;
/* verify that the calculated body size is correct */
if (istream_raw_mbox_get_body_size(mbox->mbox_stream,
body_size, &body_size) < 0) {
mail_storage_set_critical(_mail->box->storage,
"mbox %s: Couldn't get body size for uid=%u",
mailbox_get_path(&mbox->box), mail->mail.mail.uid);
return -1;
}
data->physical_size = hdr_size.physical_size + body_size;
*size_r = data->physical_size;
i_stream_seek(input, old_offset);
return 0;
}
static int mbox_mail_init_stream(struct index_mail *mail)
{
struct mbox_mailbox *mbox = (struct mbox_mailbox *)mail->mail.mail.box;
struct istream *raw_stream;
uoff_t hdr_offset, next_offset;
int ret;
if (mbox_mail_seek(mail) < 0)
return -1;
ret = mbox_mail_get_next_offset(mail, &next_offset);
if (ret < 0) {
if (mbox_mail_seek(mail) < 0)
return -1;
ret = mbox_mail_get_next_offset(mail, &next_offset);
if (ret < 0) {
i_warning("mbox %s: Can't find next message offset "
"for uid=%u", mailbox_get_path(&mbox->box),
mail->mail.mail.uid);
}
}
raw_stream = mbox->mbox_stream;
if (istream_raw_mbox_get_header_offset(raw_stream, &hdr_offset) < 0) {
mail_storage_set_critical(mbox->box.storage,
"mbox %s: Couldn't get header offset for uid=%u",
mailbox_get_path(&mbox->box), mail->mail.mail.uid);
return -1;
}
i_stream_seek(raw_stream, hdr_offset);
if (next_offset != (uoff_t)-1)
istream_raw_mbox_set_next_offset(raw_stream, next_offset);
raw_stream = i_stream_create_limit(raw_stream, (uoff_t)-1);
mail->data.stream =
i_stream_create_header_filter(raw_stream,
HEADER_FILTER_EXCLUDE | HEADER_FILTER_NO_CR,
mbox_hide_headers, mbox_hide_headers_count,
*null_header_filter_callback, (void *)NULL);
i_stream_unref(&raw_stream);
return 0;
}
static int mbox_mail_get_stream(struct mail *_mail, bool get_body ATTR_UNUSED,
struct message_size *hdr_size,
struct message_size *body_size,
struct istream **stream_r)
{
struct index_mail *mail = (struct index_mail *)_mail;
if (mail->data.stream == NULL) {
if (mbox_mail_init_stream(mail) < 0)
return -1;
}
return index_mail_init_stream(mail, hdr_size, body_size, stream_r);
}
static void mbox_mail_set_seq(struct mail *_mail, uint32_t seq, bool saving)
{
struct index_mail *mail = (struct index_mail *)_mail;
index_mail_set_seq(_mail, seq, saving);
mail->data.dont_cache_fetch_fields |= MAIL_FETCH_PHYSICAL_SIZE;
}
static bool mbox_mail_set_uid(struct mail *_mail, uint32_t uid)
{
struct index_mail *mail = (struct index_mail *)_mail;
bool ret;
ret = index_mail_set_uid(_mail, uid);
mail->data.dont_cache_fetch_fields |= MAIL_FETCH_PHYSICAL_SIZE;
return ret;
}
struct mail_vfuncs mbox_mail_vfuncs = {
index_mail_close,
index_mail_free,
mbox_mail_set_seq,
mbox_mail_set_uid,
index_mail_set_uid_cache_updates,
index_mail_prefetch,
index_mail_precache,
index_mail_add_temp_wanted_fields,
index_mail_get_flags,
index_mail_get_keywords,
index_mail_get_keyword_indexes,
index_mail_get_modseq,
index_mail_get_pvt_modseq,
index_mail_get_parts,
index_mail_get_date,
mbox_mail_get_received_date,
mbox_mail_get_save_date,
index_mail_get_virtual_size,
mbox_mail_get_physical_size,
index_mail_get_first_header,
index_mail_get_headers,
index_mail_get_header_stream,
mbox_mail_get_stream,
index_mail_get_binary_stream,
mbox_mail_get_special,
index_mail_get_real_mail,
index_mail_update_flags,
index_mail_update_keywords,
index_mail_update_modseq,
index_mail_update_pvt_modseq,
NULL,
index_mail_expunge,
index_mail_set_cache_corrupted,
index_mail_opened
};
int dbox_file_fix(struct dbox_file *file, uoff_t start_offset)
{
struct ostream *output;
const char *dir, *p, *temp_path, *broken_path;
bool deleted;
int fd, ret;
i_assert(dbox_file_is_open(file));
p = strrchr(file->cur_path, '/');
i_assert(p != NULL);
dir = t_strdup_until(file->cur_path, p);
temp_path = t_strdup_printf("%s/%s", dir, dbox_generate_tmp_filename());
fd = file->storage->v.file_create_fd(file, temp_path, FALSE);
if (fd == -1)
return -1;
output = o_stream_create_fd_file(fd, 0, FALSE);
ret = dbox_file_fix_write_stream(file, start_offset, temp_path, output);
o_stream_unref(&output);
if (close(fd) < 0) {
mail_storage_set_critical(&file->storage->storage,
"close(%s) failed: %m", temp_path);
ret = -1;
}
if (ret < 0) {
if (unlink(temp_path) < 0) {
mail_storage_set_critical(&file->storage->storage,
"unlink(%s) failed: %m", temp_path);
}
return -1;
}
/* keep a copy of the original file in case someone wants to look
at it */
broken_path = t_strconcat(file->cur_path,
DBOX_MAIL_FILE_BROKEN_COPY_SUFFIX, NULL);
if (link(file->cur_path, broken_path) < 0) {
mail_storage_set_critical(&file->storage->storage,
"link(%s, %s) failed: %m",
file->cur_path, broken_path);
} else {
i_warning("dbox: Copy of the broken file saved to %s",
broken_path);
}
if (rename(temp_path, file->cur_path) < 0) {
mail_storage_set_critical(&file->storage->storage,
"rename(%s, %s) failed: %m",
temp_path, file->cur_path);
return -1;
}
/* file was successfully recreated - reopen it */
dbox_file_close(file);
if (dbox_file_open(file, &deleted) <= 0) {
mail_storage_set_critical(&file->storage->storage,
"dbox_file_fix(%s): reopening file failed",
file->cur_path);
return -1;
}
return 0;
}
static int
dbox_file_fix_write_stream(struct dbox_file *file, uoff_t start_offset,
const char *temp_path, struct ostream *output)
{
struct dbox_message_header msg_hdr;
uoff_t offset, msg_size, hdr_offset, body_offset;
bool pre, write_header, have_guid;
struct message_size body;
struct istream *body_input;
uint8_t guid_128[MAIL_GUID_128_SIZE];
int ret;
i_stream_seek(file->input, 0);
if (start_offset > 0) {
/* copy the valid data */
if (stream_copy(file, output, temp_path, start_offset) < 0)
return -1;
} else {
/* the file header is broken. recreate it */
if (dbox_file_header_write(file, output) < 0) {
dbox_file_set_syscall_error(file, "write()");
return -1;
}
}
while ((ret = dbox_file_find_next_magic(file, &offset, &pre)) > 0) {
msg_size = offset - file->input->v_offset;
if (msg_size < 256 && pre) {
/* probably some garbage or some broken headers.
we most likely don't miss anything by skipping
over this data. */
i_stream_skip(file->input, msg_size);
hdr_offset = file->input->v_offset;
ret = dbox_file_read_mail_header(file, &msg_size);
if (ret <= 0) {
if (ret < 0)
return -1;
dbox_file_skip_broken_header(file);
body_offset = file->input->v_offset;
msg_size = (uoff_t)-1;
} else {
i_stream_skip(file->input,
file->msg_header_size);
body_offset = file->input->v_offset;
i_stream_skip(file->input, msg_size);
}
ret = dbox_file_find_next_magic(file, &offset, &pre);
if (ret <= 0)
break;
if (!pre && msg_size == offset - body_offset) {
/* msg header ok, copy it */
i_stream_seek(file->input, hdr_offset);
if (stream_copy(file, output, temp_path,
file->msg_header_size) < 0)
return -1;
write_header = FALSE;
} else {
/* msg header is broken. write our own. */
i_stream_seek(file->input, body_offset);
if (msg_size != (uoff_t)-1) {
/* previous magic find might have
skipped too much. seek back and
make sure */
ret = dbox_file_find_next_magic(file, &offset, &pre);
if (ret <= 0)
break;
}
write_header = TRUE;
msg_size = offset - body_offset;
}
} else {
/* treat this data as a separate message. */
write_header = TRUE;
body_offset = file->input->v_offset;
}
/* write msg header */
if (write_header) {
dbox_msg_header_fill(&msg_hdr, msg_size);
(void)o_stream_send(output, &msg_hdr, sizeof(msg_hdr));
}
/* write msg body */
i_assert(file->input->v_offset == body_offset);
if (stream_copy(file, output, temp_path, msg_size) < 0)
return -1;
i_assert(file->input->v_offset == offset);
/* get message body size */
i_stream_seek(file->input, body_offset);
body_input = i_stream_create_limit(file->input, msg_size);
ret = message_get_body_size(body_input, &body, NULL);
i_stream_unref(&body_input);
if (ret < 0) {
errno = output->stream_errno;
mail_storage_set_critical(&file->storage->storage,
"read(%s) failed: %m", file->cur_path);
return -1;
}
/* write msg metadata. */
i_assert(file->input->v_offset == offset);
ret = dbox_file_metadata_skip_header(file);
if (ret < 0)
return -1;
o_stream_send_str(output, DBOX_MAGIC_POST);
if (ret == 0)
have_guid = FALSE;
else
dbox_file_copy_metadata(file, output, &have_guid);
if (!have_guid) {
mail_generate_guid_128(guid_128);
o_stream_send_str(output,
t_strdup_printf("%c%s\n", DBOX_METADATA_GUID,
binary_to_hex(guid_128, sizeof(guid_128))));
}
o_stream_send_str(output,
t_strdup_printf("%c%llx\n", DBOX_METADATA_VIRTUAL_SIZE,
(unsigned long long)body.virtual_size));
o_stream_send_str(output, "\n");
if (output->stream_errno != 0) {
errno = output->stream_errno;
mail_storage_set_critical(&file->storage->storage,
"write(%s) failed: %m", temp_path);
return -1;
}
}
return ret;
}
void dbox_file_set_syscall_error(struct dbox_file *file, const char *function)
{
mail_storage_set_critical(&file->storage->storage,
"%s failed for file %s: %m",
function, file->cur_path);
}
