void mail_transaction_log_file_unlock(struct mail_transaction_log_file *file,
const char *lock_reason)
{
unsigned int lock_time;
if (!file->locked)
return;
file->locked = FALSE;
file->locked_sync_offset_updated = FALSE;
if (MAIL_TRANSACTION_LOG_FILE_IN_MEMORY(file))
return;
lock_time = time(NULL) - file->lock_created;
if (lock_time >= MAIL_TRANSACTION_LOG_LOCK_WARN_SECS && lock_reason != NULL) {
i_warning("Transaction log file %s was locked for %u seconds (%s)",
file->filepath, lock_time, lock_reason);
}
if (file->log->index->lock_method == FILE_LOCK_METHOD_DOTLOCK) {
(void)mail_transaction_log_file_undotlock(file);
return;
}
file_unlock(&file->file_lock);
}
static int
mail_transaction_log_refresh(struct mail_transaction_log *log, bool nfs_flush)
{
struct mail_transaction_log_file *file;
struct stat st;
i_assert(log->head != NULL);
if (MAIL_TRANSACTION_LOG_FILE_IN_MEMORY(log->head))
return 0;
if (nfs_flush && log->nfs_flush)
nfs_flush_file_handle_cache(log->filepath);
if (nfs_safe_stat(log->filepath, &st) < 0) {
if (errno != ENOENT) {
mail_index_file_set_syscall_error(log->index,
log->filepath,
"stat()");
return -1;
}
/* see if the whole directory got deleted */
if (nfs_safe_stat(log->index->dir, &st) < 0 &&
errno == ENOENT) {
log->index->index_deleted = TRUE;
return -1;
}
/* the file should always exist at this point. if it doesn't,
someone deleted it manually while the index was open. try to
handle this nicely by creating a new log file. */
file = log->head;
if (mail_transaction_log_create(log, FALSE) < 0)
return -1;
i_assert(file->refcount > 0);
file->refcount--;
log->index->need_recreate = TRUE;
return 0;
} else if (log->head->st_ino == st.st_ino &&
CMP_DEV_T(log->head->st_dev, st.st_dev)) {
/* NFS: log files get rotated to .log.2 files instead
of being unlinked, so we don't bother checking if
the existing file has already been unlinked here
(in which case inodes could match but point to
different files) */
return 0;
}
file = mail_transaction_log_file_alloc(log, log->filepath);
if (mail_transaction_log_file_open(file, FALSE) <= 0) {
mail_transaction_log_file_free(&file);
return -1;
}
i_assert(!file->locked);
if (--log->head->refcount == 0)
mail_transaction_logs_clean(log);
mail_transaction_log_set_head(log, file);
return 0;
}
static void
mail_transaction_log_mark_corrupted(struct mail_transaction_log_file *file)
{
unsigned int offset =
offsetof(struct mail_transaction_log_header, indexid);
int flags;
if (MAIL_TRANSACTION_LOG_FILE_IN_MEMORY(file) ||
file->log->index->readonly)
return;
/* indexid=0 marks the log file as corrupted. we opened the file with
O_APPEND, and now we need to drop it for pwrite() to work (at least
in Linux) */
flags = fcntl(file->fd, F_GETFL, 0);
if (flags < 0) {
mail_index_file_set_syscall_error(file->log->index,
file->filepath, "fcntl(F_GETFL)");
return;
}
if (fcntl(file->fd, F_SETFL, flags & ~O_APPEND) < 0) {
mail_index_file_set_syscall_error(file->log->index,
file->filepath, "fcntl(F_SETFL)");
return;
}
if (pwrite_full(file->fd, &file->hdr.indexid,
sizeof(file->hdr.indexid), offset) < 0) {
mail_index_file_set_syscall_error(file->log->index,
file->filepath, "pwrite()");
}
}
void mail_transaction_log_file_move_to_memory(struct mail_transaction_log_file
*file)
{
buffer_t *buf;
if (MAIL_TRANSACTION_LOG_FILE_IN_MEMORY(file))
return;
if (file->mmap_base != NULL) {
/* just copy to memory */
i_assert(file->buffer_offset == 0);
buf = buffer_create_dynamic(default_pool, file->mmap_size);
buffer_append(buf, file->mmap_base, file->mmap_size);
buffer_free(&file->buffer);
file->buffer = buf;
/* and lose the mmap */
if (munmap(file->mmap_base, file->mmap_size) < 0)
log_file_set_syscall_error(file, "munmap()");
file->mmap_base = NULL;
} else if (file->buffer_offset != 0) {
/* we don't have the full log in the memory. read it. */
(void)mail_transaction_log_file_read(file, 0, FALSE);
}
file->last_size = 0;
if (close(file->fd) < 0)
log_file_set_syscall_error(file, "close()");
file->fd = -1;
i_free(file->filepath);
file->filepath = i_strdup(file->log->filepath);
}
int mail_transaction_log_file_lock(struct mail_transaction_log_file *file)
{
unsigned int lock_timeout_secs;
int ret;
if (file->locked)
return 0;
if (MAIL_TRANSACTION_LOG_FILE_IN_MEMORY(file)) {
file->locked = TRUE;
return 0;
}
if (file->log->index->lock_method == FILE_LOCK_METHOD_DOTLOCK)
return mail_transaction_log_file_dotlock(file);
if (file->log->index->readonly) {
mail_index_set_error(file->log->index,
"Index is read-only, can't write-lock %s",
file->filepath);
return -1;
}
i_assert(file->file_lock == NULL);
lock_timeout_secs = I_MIN(MAIL_TRANSACTION_LOG_LOCK_TIMEOUT,
file->log->index->max_lock_timeout_secs);
ret = mail_index_lock_fd(file->log->index, file->filepath, file->fd,
F_WRLCK, lock_timeout_secs,
&file->file_lock);
if (ret > 0) {
file->locked = TRUE;
file->lock_created = time(NULL);
return 0;
}
if (ret < 0) {
log_file_set_syscall_error(file, "mail_index_wait_lock_fd()");
return -1;
}
mail_index_set_error(file->log->index,
"Timeout (%us) while waiting for lock for "
"transaction log file %s%s",
lock_timeout_secs, file->filepath,
file_lock_find(file->fd, file->log->index->lock_method, F_WRLCK));
file->log->index->index_lock_timeout = TRUE;
return -1;
}
static int
mail_transaction_log_file_read_hdr(struct mail_transaction_log_file *file,
bool ignore_estale)
{
struct mail_transaction_log_file *f;
int ret;
i_assert(!MAIL_TRANSACTION_LOG_FILE_IN_MEMORY(file));
if (file->corrupted)
return 0;
ret = mail_transaction_log_file_read_header(file);
if (ret < 0) {
if (errno != ESTALE || !ignore_estale)
log_file_set_syscall_error(file, "pread()");
return -1;
}
if (file->hdr.major_version != MAIL_TRANSACTION_LOG_MAJOR_VERSION) {
/* incompatible version - fix silently */
return 0;
}
if (ret < MAIL_TRANSACTION_LOG_HEADER_MIN_SIZE) {
mail_transaction_log_file_set_corrupted(file,
"unexpected end of file while reading header");
return 0;
}
if (file->hdr.minor_version >= 2 || file->hdr.major_version > 1) {
/* we have compatibility flags */
enum mail_index_header_compat_flags compat_flags = 0;
#if !WORDS_BIGENDIAN
compat_flags |= MAIL_INDEX_COMPAT_LITTLE_ENDIAN;
#endif
if (file->hdr.compat_flags != compat_flags) {
/* architecture change */
mail_index_set_error(file->log->index,
"Rebuilding index file %s: "
"CPU architecture changed",
file->log->index->filepath);
return 0;
}
}
if (file->hdr.hdr_size < MAIL_TRANSACTION_LOG_HEADER_MIN_SIZE) {
mail_transaction_log_file_set_corrupted(file,
"Header size too small");
return 0;
}
if (file->hdr.hdr_size < sizeof(file->hdr)) {
/* @UNSAFE: smaller than we expected - zero out the fields we
shouldn't have filled */
memset(PTR_OFFSET(&file->hdr, file->hdr.hdr_size), 0,
sizeof(file->hdr) - file->hdr.hdr_size);
}
if (file->hdr.indexid == 0) {
/* corrupted */
file->corrupted = TRUE;
mail_index_set_error(file->log->index,
"Transaction log file %s: marked corrupted",
file->filepath);
return 0;
}
if (file->hdr.indexid != file->log->index->indexid) {
if (file->log->index->indexid != 0 &&
!file->log->index->initial_create) {
/* index file was probably just rebuilt and we don't
know about it yet */
mail_transaction_log_file_set_corrupted(file,
"indexid changed %u -> %u",
file->log->index->indexid, file->hdr.indexid);
return 0;
}
/* creating index file. since transaction log is created
first, use the indexid in it to create the main index
to avoid races. */
file->log->index->indexid = file->hdr.indexid;
}
/* make sure we already don't have a file with the same sequence
opened. it shouldn't happen unless the old log file was
corrupted. */
for (f = file->log->files; f != NULL; f = f->next) {
if (f->hdr.file_seq == file->hdr.file_seq) {
if (strcmp(f->filepath, f->log->head->filepath) != 0) {
/* old "f" is the .log.2 */
return mail_transaction_log_file_fail_dupe(f);
} else {
/* new "file" is probably the .log.2 */
return mail_transaction_log_file_fail_dupe(file);
}
}
}
file->sync_highest_modseq = file->hdr.initial_modseq;
return 1;
}
int mail_transaction_log_file_map(struct mail_transaction_log_file *file,
uoff_t start_offset, uoff_t end_offset)
{
struct mail_index *index = file->log->index;
uoff_t map_start_offset = start_offset;
size_t size;
int ret;
if (file->hdr.indexid == 0) {
/* corrupted */
return 0;
}
i_assert(start_offset >= file->hdr.hdr_size);
i_assert(start_offset <= end_offset);
i_assert(file->buffer == NULL || file->mmap_base != NULL ||
file->sync_offset >= file->buffer_offset + file->buffer->used);
if (file->locked_sync_offset_updated && file == file->log->head &&
end_offset == (uoff_t)-1) {
/* we're not interested of going further than sync_offset */
if (log_file_map_check_offsets(file, start_offset,
end_offset) == 0)
return 0;
i_assert(start_offset <= file->sync_offset);
end_offset = file->sync_offset;
}
if (file->buffer != NULL && file->buffer_offset <= start_offset) {
/* see if we already have it */
size = file->buffer->used;
if (file->buffer_offset + size >= end_offset)
return 1;
}
if (file->locked) {
/* set this only when we've synced to end of file while locked
(either end_offset=(uoff_t)-1 or we had to read anyway) */
file->locked_sync_offset_updated = TRUE;
}
if (MAIL_TRANSACTION_LOG_FILE_IN_MEMORY(file)) {
if (start_offset < file->buffer_offset || file->buffer == NULL) {
/* we had moved the log to memory but failed to read
the beginning of the log file */
mail_index_set_error(index,
"%s: Beginning of the log isn't available",
file->filepath);
return 0;
}
return log_file_map_check_offsets(file, start_offset,
end_offset);
}
if (start_offset > file->sync_offset)
mail_transaction_log_file_skip_to_head(file);
if (start_offset > file->sync_offset) {
/* although we could just skip over the unwanted data, we have
to sync everything so that modseqs are calculated
correctly */
map_start_offset = file->sync_offset;
}
if ((file->log->index->flags & MAIL_INDEX_OPEN_FLAG_MMAP_DISABLE) == 0)
ret = mail_transaction_log_file_map_mmap(file, map_start_offset);
else {
mail_transaction_log_file_munmap(file);
ret = mail_transaction_log_file_read(file, map_start_offset, FALSE);
}
i_assert(file->buffer == NULL || file->mmap_base != NULL ||
file->sync_offset >= file->buffer_offset + file->buffer->used);
if (ret <= 0)
return ret;
i_assert(file->buffer != NULL);
return log_file_map_check_offsets(file, start_offset, end_offset);
}
