static int o_stream_ssl_flush_buffer(struct ssl_ostream *sstream)
{
size_t pos = 0;
int ret = 1;
while (pos < sstream->buffer->used) {
/* we're writing plaintext data to OpenSSL, which it encrypts
and writes to bio_int's buffer. ssl_iostream_bio_sync()
reads it from there and adds to plain_output stream. */
ret = SSL_write(sstream->ssl_io->ssl,
CONST_PTR_OFFSET(sstream->buffer->data, pos),
sstream->buffer->used - pos);
if (ret <= 0) {
ret = openssl_iostream_handle_write_error(sstream->ssl_io,
ret, "SSL_write");
if (ret < 0) {
io_stream_set_error(&sstream->ostream.iostream,
"%s", sstream->ssl_io->last_error);
sstream->ostream.ostream.stream_errno = errno;
break;
}
if (ret == 0)
break;
} else {
pos += ret;
(void)openssl_iostream_bio_sync(sstream->ssl_io);
}
}
buffer_delete(sstream->buffer, 0, pos);
return ret <= 0 ? ret : 1;
}
static int
log_get_synced_record(struct mail_transaction_log_file *file, uoff_t *offset,
const struct mail_transaction_header **hdr_r)
{
const struct mail_transaction_header *hdr;
uint32_t trans_size;
hdr = CONST_PTR_OFFSET(file->buffer->data,
*offset - file->buffer_offset);
/* we've already synced this record at some point. it should
be valid. */
trans_size = mail_index_offset_to_uint32(hdr->size);
if (trans_size < sizeof(*hdr) ||
*offset - file->buffer_offset + trans_size > file->buffer->used) {
mail_transaction_log_file_set_corrupted(file,
"Transaction log corrupted unexpectedly at "
"%"PRIuUOFF_T": Invalid size %u (type=%x)",
*offset, trans_size, hdr->type);
return -1;
}
*offset += trans_size;
*hdr_r = hdr;
return 0;
}
int
mail_index_sync_keywords_reset(struct mail_index_sync_map_ctx *ctx,
const struct mail_transaction_header *hdr,
const struct mail_transaction_keyword_reset *r)
{
struct mail_index_map *map = ctx->view->map;
struct mail_index_record *rec;
const struct mail_index_ext *ext;
const struct mail_transaction_keyword_reset *end;
uint32_t ext_map_idx, seq1, seq2;
if (!mail_index_map_lookup_ext(map, MAIL_INDEX_EXT_KEYWORDS,
&ext_map_idx)) {
/* nothing to do */
return 1;
}
ext = array_idx(&map->extensions, ext_map_idx);
end = CONST_PTR_OFFSET(r, hdr->size);
for (; r != end; r++) {
if (!mail_index_lookup_seq_range(ctx->view, r->uid1, r->uid2,
&seq1, &seq2))
continue;
mail_index_modseq_reset_keywords(ctx->modseq_ctx, seq1, seq2);
for (; seq1 <= seq2; seq1++) {
rec = MAIL_INDEX_REC_AT_SEQ(map, seq1);
memset(PTR_OFFSET(rec, ext->record_offset),
0, ext->record_size);
}
}
return 1;
}
static void log_header_update(const struct mail_transaction_header_update *u)
{
const void *data = u + 1;
unsigned int offset = u->offset, size = u->size;
unsigned int i;
while (size > 0) {
/* don't bother trying to handle header updates that include
unknown/unexpected fields offsets/sizes */
for (i = 0; i < N_ELEMENTS(header_fields); i++) {
if (header_fields[i].offset == offset &&
header_fields[i].size <= size)
break;
}
if (i == N_ELEMENTS(header_fields)) {
printf(" - offset = %u, size = %u: ", offset, size);
print_data(data, size);
printf("\n");
break;
}
printf(" - %s = ", header_fields[i].name);
print_try_uint(data, header_fields[i].size);
printf("\n");
data = CONST_PTR_OFFSET(data, header_fields[i].size);
offset += header_fields[i].size;
size -= header_fields[i].size;
}
}
static int mail_index_recreate(struct mail_index *index)
{
struct mail_index_map *map = index->map;
struct ostream *output;
unsigned int base_size;
const char *path;
int ret = 0, fd;
i_assert(!MAIL_INDEX_IS_IN_MEMORY(index));
i_assert(map->hdr.indexid == index->indexid);
i_assert((map->hdr.flags & MAIL_INDEX_HDR_FLAG_CORRUPTED) == 0);
i_assert(index->indexid != 0);
fd = mail_index_create_tmp_file(index, index->filepath, &path);
if (fd == -1)
return -1;
output = o_stream_create_fd_file(fd, 0, FALSE);
o_stream_cork(output);
base_size = I_MIN(map->hdr.base_header_size, sizeof(map->hdr));
o_stream_nsend(output, &map->hdr, base_size);
o_stream_nsend(output, CONST_PTR_OFFSET(map->hdr_base, base_size),
map->hdr.header_size - base_size);
o_stream_nsend(output, map->rec_map->records,
map->rec_map->records_count * map->hdr.record_size);
o_stream_nflush(output);
if (o_stream_nfinish(output) < 0) {
mail_index_file_set_syscall_error(index, path, "write()");
ret = -1;
}
o_stream_destroy(&output);
if (ret == 0 && index->fsync_mode != FSYNC_MODE_NEVER) {
if (fdatasync(fd) < 0) {
mail_index_file_set_syscall_error(index, path,
"fdatasync()");
ret = -1;
}
}
if (close(fd) < 0) {
mail_index_file_set_syscall_error(index, path, "close()");
ret = -1;
}
if ((index->flags & MAIL_INDEX_OPEN_FLAG_KEEP_BACKUPS) != 0)
(void)mail_index_create_backup(index);
if (ret == 0 && rename(path, index->filepath) < 0) {
mail_index_set_error(index, "rename(%s, %s) failed: %m",
path, index->filepath);
ret = -1;
}
if (ret < 0)
i_unlink(path);
return ret;
}
void mail_transaction_update_modseq(const struct mail_transaction_header *hdr,
const void *data, uint64_t *cur_modseq)
{
uint32_t trans_size;
trans_size = mail_index_offset_to_uint32(hdr->size);
i_assert(trans_size != 0);
if (*cur_modseq != 0) {
/* tracking modseqs */
} else if ((hdr->type & MAIL_TRANSACTION_TYPE_MASK) ==
MAIL_TRANSACTION_EXT_INTRO) {
/* modseqs not tracked yet. see if this is a modseq
extension introduction. */
const struct mail_transaction_ext_intro *intro = data;
const unsigned int modseq_ext_len =
strlen(MAIL_INDEX_MODSEQ_EXT_NAME);
if (intro->name_size == modseq_ext_len &&
memcmp(intro + 1, MAIL_INDEX_MODSEQ_EXT_NAME,
modseq_ext_len) == 0) {
/* modseq tracking started */
*cur_modseq += 1;
}
return;
} else {
/* not tracking modseqs */
return;
}
switch (hdr->type & MAIL_TRANSACTION_TYPE_MASK) {
case MAIL_TRANSACTION_EXPUNGE | MAIL_TRANSACTION_EXPUNGE_PROT:
case MAIL_TRANSACTION_EXPUNGE_GUID | MAIL_TRANSACTION_EXPUNGE_PROT:
if ((hdr->type & MAIL_TRANSACTION_EXTERNAL) == 0) {
/* ignore expunge requests */
break;
}
case MAIL_TRANSACTION_APPEND:
case MAIL_TRANSACTION_FLAG_UPDATE:
case MAIL_TRANSACTION_KEYWORD_UPDATE:
case MAIL_TRANSACTION_KEYWORD_RESET:
case MAIL_TRANSACTION_ATTRIBUTE_UPDATE:
/* these changes increase modseq */
*cur_modseq += 1;
break;
case MAIL_TRANSACTION_MODSEQ_UPDATE: {
const struct mail_transaction_modseq_update *rec, *end;
end = CONST_PTR_OFFSET(data, trans_size - sizeof(*hdr));
for (rec = data; rec < end; rec++) {
uint64_t modseq = ((uint64_t)rec->modseq_high32 << 32) |
rec->modseq_low32;
if (*cur_modseq < modseq)
*cur_modseq = modseq;
}
}
}
}
static void squat_uidlist_map_blocks_set_pointers(struct squat_uidlist *uidlist)
{
const void *base;
size_t end_index_size, end_size;
base = CONST_PTR_OFFSET(uidlist->data, uidlist->hdr.block_list_offset +
sizeof(uint32_t));
end_index_size = uidlist->cur_block_count * sizeof(uint32_t);
end_size = end_index_size + uidlist->cur_block_count * sizeof(uint32_t);
if (end_size <= uidlist->data_size) {
uidlist->cur_block_end_indexes = base;
uidlist->cur_block_offsets =
CONST_PTR_OFFSET(base, end_index_size);
} else {
uidlist->cur_block_end_indexes = NULL;
uidlist->cur_block_offsets = NULL;
}
}
static int dump_record(int fd, buffer_t *buf)
{
struct fts_expunge_log_record rec;
off_t offset;
void *data;
const uint32_t *expunges, *uids;
ssize_t ret;
size_t data_size;
unsigned int i, uids_count;
offset = lseek(fd, 0, SEEK_CUR);
ret = read(fd, &rec, sizeof(rec));
if (ret == 0)
return 0;
if (ret != sizeof(rec))
i_fatal("rec read() %d != %d", (int)ret, (int)sizeof(rec));
if (rec.record_size < sizeof(rec) + sizeof(uint32_t) ||
rec.record_size > INT_MAX) {
i_fatal("Invalid record_size=%u at offset %"PRIuUOFF_T,
rec.record_size, offset);
}
data_size = rec.record_size - sizeof(rec);
buffer_set_used_size(buf, 0);
data = buffer_append_space_unsafe(buf, data_size);
ret = read(fd, data, data_size);
if (ret != (ssize_t)data_size)
i_fatal("rec read() %d != %d", (int)ret, (int)data_size);
printf("#%"PRIuUOFF_T":\n", offset);
printf(" checksum = %8x\n", rec.checksum);
printf(" size .... = %u\n", rec.record_size);
printf(" mailbox . = %s\n", guid_128_to_string(rec.guid));
expunges = CONST_PTR_OFFSET(data, data_size - sizeof(uint32_t));
printf(" expunges = %u\n", *expunges);
printf(" uids .... = ");
uids = data;
uids_count = (rec.record_size - sizeof(rec) - sizeof(uint32_t)) /
sizeof(uint32_t);
for (i = 0; i < uids_count; i += 2) {
if (i != 0)
printf(",");
if (uids[i] == uids[i+1])
printf("%u", uids[i]);
else
printf("%u-%u", uids[i], uids[i+1]);
}
printf("\n");
return 1;
}
static int mailbox_list_index_parse_header(struct mailbox_list_index *ilist,
struct mail_index_view *view)
{
const void *data, *p;
size_t i, len, size;
uint32_t id, prev_id = 0;
char *name;
mail_index_map_get_header_ext(view, view->map, ilist->ext_id, &data, &size);
if (size == 0)
return 0;
for (i = sizeof(struct mailbox_list_index_header); i < size; ) {
/* get id */
if (i + sizeof(id) > size)
return -1;
memcpy(&id, CONST_PTR_OFFSET(data, i), sizeof(id));
i += sizeof(id);
if (id <= prev_id) {
/* allow extra space in the end as long as last id=0 */
return id == 0 ? 0 : -1;
}
/* get name */
p = memchr(CONST_PTR_OFFSET(data, i), '\0', size-i);
if (p == NULL)
return -1;
len = (const char *)p -
(const char *)(CONST_PTR_OFFSET(data, i));
name = p_strndup(ilist->mailbox_pool,
CONST_PTR_OFFSET(data, i), len);
i += len + 1;
/* add id => name to hash table */
hash_table_insert(ilist->mailbox_names, POINTER_CAST(id), name);
ilist->highest_name_id = id;
}
i_assert(i == size);
return 0;
}
static int squat_uidlist_map_blocks(struct squat_uidlist *uidlist)
{
const struct squat_uidlist_file_header *hdr = &uidlist->hdr;
const void *base;
uint32_t block_count, blocks_offset, blocks_size, i, verify_count;
if (hdr->block_list_offset == 0) {
/* empty file */
uidlist->cur_block_count = 0;
return 1;
}
/* get number of blocks */
if (uidlist_file_cache_read(uidlist, hdr->block_list_offset,
sizeof(block_count)) < 0)
return -1;
blocks_offset = hdr->block_list_offset + sizeof(block_count);
if (blocks_offset > uidlist->data_size) {
squat_uidlist_set_corrupted(uidlist, "block list outside file");
return 0;
}
i_assert(uidlist->data != NULL);
base = CONST_PTR_OFFSET(uidlist->data, hdr->block_list_offset);
memcpy(&block_count, base, sizeof(block_count));
/* map the blocks */
blocks_size = block_count * sizeof(uint32_t)*2;
if (uidlist_file_cache_read(uidlist, blocks_offset, blocks_size) < 0)
return -1;
if (blocks_offset + blocks_size > uidlist->data_size) {
squat_uidlist_set_corrupted(uidlist, "block list outside file");
return 0;
}
uidlist->cur_block_count = block_count;
squat_uidlist_map_blocks_set_pointers(uidlist);
i_assert(uidlist->cur_block_end_indexes != NULL);
/* verify just a couple of the end indexes to make sure they
look correct */
verify_count = I_MIN(block_count, 8);
for (i = 1; i < verify_count; i++) {
if (unlikely(uidlist->cur_block_end_indexes[i-1] >=
uidlist->cur_block_end_indexes[i])) {
squat_uidlist_set_corrupted(uidlist,
"block list corrupted");
return 0;
}
}
return 1;
}
static size_t
o_stream_ssl_buffer(struct ssl_ostream *sstream, const struct const_iovec *iov,
unsigned int iov_count, size_t bytes_sent)
{
size_t avail, skip_left, size;
unsigned int i;
if (sstream->buffer == NULL)
sstream->buffer = buffer_create_dynamic(default_pool, 4096);
skip_left = bytes_sent;
for (i = 0; i < iov_count; i++) {
if (skip_left < iov[i].iov_len)
break;
skip_left -= iov[i].iov_len;
}
if (sstream->ostream.max_buffer_size == 0) {
/* we're requeted to use whatever space is available in
the buffer */
avail = buffer_get_size(sstream->buffer) - sstream->buffer->used;
} else {
avail = sstream->ostream.max_buffer_size > sstream->buffer->used ?
sstream->ostream.max_buffer_size - sstream->buffer->used : 0;
}
if (i < iov_count && skip_left > 0) {
size = I_MIN(iov[i].iov_len - skip_left, avail);
buffer_append(sstream->buffer,
CONST_PTR_OFFSET(iov[i].iov_base, skip_left),
size);
bytes_sent += size;
avail -= size;
if (size != iov[i].iov_len)
i = iov_count;
}
if (avail > 0)
o_stream_set_flush_pending(sstream->ssl_io->plain_output, TRUE);
for (; i < iov_count; i++) {
size = I_MIN(iov[i].iov_len, avail);
buffer_append(sstream->buffer, iov[i].iov_base, size);
bytes_sent += size;
avail -= size;
if (size != iov[i].iov_len)
break;
}
sstream->ostream.ostream.offset += bytes_sent;
return bytes_sent;
}
static int
index_list_update(struct index_mailbox_list *ilist, struct mailbox *box,
struct mail_index_view *view, uint32_t seq,
const struct mailbox_status *status)
{
struct index_list_mailbox *ibox = INDEX_LIST_STORAGE_CONTEXT(box);
struct mail_index_transaction *trans;
struct mail_index_transaction_commit_result result;
const void *data;
const uint32_t *counter_p;
uint32_t *ext_id_p;
unsigned int i;
bool expunged;
int ret = 0;
trans = mail_index_transaction_begin(view,
MAIL_INDEX_TRANSACTION_FLAG_EXTERNAL);
/* update counters */
for (i = 0; index_list_map[i].name != NULL; i++) {
ext_id_p = PTR_OFFSET(ilist, index_list_map[i].eid_offset);
mail_index_lookup_ext(view, seq, *ext_id_p, &data, &expunged);
if (expunged) {
ret = -1;
break;
}
counter_p = CONST_PTR_OFFSET(status,
index_list_map[i].status_offset);
if (data == NULL ||
*(const uint32_t *)data != *counter_p) {
mail_index_update_ext(trans, seq, *ext_id_p,
counter_p, NULL);
}
}
if (box->v.list_index_update_sync(box, trans, seq) < 0)
ret = -1;
if (ret < 0) {
mail_index_transaction_rollback(&trans);
return -1;
}
if (mail_index_transaction_commit_full(&trans, &result) < 0)
return -1;
ibox->log_seq = result.log_file_seq;
ibox->log_offset = result.log_file_offset;
return 0;
}
static void test_quoted_printable_decode(void)
{
static struct test_quoted_printable_decode_data data[] = {
{ "foo \r\nbar=", "foo\r\nbar", 1, 0 },
{ "foo\t=\nbar", "foo\tbar", 0, 0 },
{ "foo = \n=01", "foo \001", 0, 0 },
{ "foo =\t\r\nbar", "foo bar", 0, 0 },
{ "foo =\r\n=01", "foo \001", 0, 0 },
{ "foo \nbar=", "foo\nbar", 1, 0 },
{ "=0A=0D ", "\n\r", 2, 0 },
{ "foo_bar", "foo_bar", 0, 0 },
{ "foo=", "foo", 1, 0 },
{ "foo= ", "foo", 3, 0 },
{ "foo=A", "foo", 2, 0 },
{ "foo=Ax", "foo=Ax", 0, -1 },
{ "foo=Ax=xy", "foo=Ax=xy", 0, -1 }
};
buffer_t *buf;
unsigned int i, start, end, len;
size_t src_pos;
int ret;
test_begin("quoted printable decode");
buf = buffer_create_dynamic(pool_datastack_create(), 128);
for (i = 0; i < N_ELEMENTS(data); i++) {
len = strlen(data[i].input);
ret = quoted_printable_decode((const void *)data[i].input, len,
&src_pos, buf);
test_assert(ret == data[i].ret);
test_assert(src_pos + data[i].end_skip == len);
test_assert(strcmp(data[i].output, str_c(buf)) == 0);
buffer_set_used_size(buf, 0);
for (start = 0, end = 1; end <= len; ) {
quoted_printable_decode(CONST_PTR_OFFSET(data[i].input, start),
end - start, &src_pos, buf);
src_pos += start;
start = src_pos;
if (src_pos <= end)
end++;
else
end = src_pos + 1;
}
test_assert(src_pos + data[i].end_skip == len);
test_assert(strcmp(data[i].output, str_c(buf)) == 0);
buffer_set_used_size(buf, 0);
}
test_end();
}
static bool message_decode_header(struct message_decoder_context *ctx,
struct message_header_line *hdr,
struct message_block *output)
{
size_t value_len;
if (hdr->continues) {
hdr->use_full_value = TRUE;
return FALSE;
}
T_BEGIN {
if (hdr->name_len == 12 &&
strcasecmp(hdr->name, "Content-Type") == 0)
parse_content_type(ctx, hdr);
if (hdr->name_len == 25 &&
strcasecmp(hdr->name, "Content-Transfer-Encoding") == 0)
ctx->message_cte = message_decoder_parse_cte(hdr);
} T_END;
buffer_set_used_size(ctx->buf, 0);
message_header_decode_utf8(hdr->full_value, hdr->full_value_len,
ctx->buf, ctx->normalizer);
value_len = ctx->buf->used;
if (ctx->normalizer != NULL) {
(void)ctx->normalizer(hdr->name, hdr->name_len, ctx->buf);
buffer_append_c(ctx->buf, '\0');
} else {
if (!uni_utf8_get_valid_data((const unsigned char *)hdr->name,
hdr->name_len, ctx->buf))
buffer_append_c(ctx->buf, '\0');
}
ctx->hdr = *hdr;
ctx->hdr.full_value = ctx->buf->data;
ctx->hdr.full_value_len = value_len;
ctx->hdr.value_len = 0;
if (ctx->buf->used != value_len) {
ctx->hdr.name = CONST_PTR_OFFSET(ctx->buf->data,
ctx->hdr.full_value_len);
ctx->hdr.name_len = ctx->buf->used - 1 - value_len;
}
output->hdr = &ctx->hdr;
return TRUE;
}
static int
log_file_track_mailbox_sync_offset_hdr(struct mail_transaction_log_file *file,
const void *data, unsigned int trans_size)
{
const struct mail_transaction_header_update *u = data;
const struct mail_index_header *ihdr;
const unsigned int size = trans_size - sizeof(struct mail_transaction_header);
const unsigned int offset_pos =
offsetof(struct mail_index_header, log_file_tail_offset);
const unsigned int offset_size = sizeof(ihdr->log_file_tail_offset);
uint32_t tail_offset;
i_assert(offset_size == sizeof(tail_offset));
if (size < sizeof(*u) || size < sizeof(*u) + u->size) {
mail_transaction_log_file_set_corrupted(file,
"header update extends beyond record size");
return -1;
}
if (u->offset <= offset_pos &&
u->offset + u->size >= offset_pos + offset_size) {
memcpy(&tail_offset,
CONST_PTR_OFFSET(u + 1, offset_pos - u->offset),
sizeof(tail_offset));
if (tail_offset < file->saved_tail_offset) {
/* ignore shrinking tail offsets */
return 1;
} else if (tail_offset > file->sync_offset + trans_size) {
mail_transaction_log_file_set_corrupted(file,
"log_file_tail_offset %u goes past sync offset %"PRIuUOFF_T,
tail_offset, file->sync_offset + trans_size);
} else {
file->saved_tail_offset = tail_offset;
if (tail_offset > file->max_tail_offset)
file->max_tail_offset = tail_offset;
return 1;
}
}
return 0;
}
static buffer_t *
keywords_get_header_buf(struct mail_index_map *map,
const struct mail_index_ext *ext,
unsigned int new_count, unsigned int *keywords_count_r,
size_t *rec_offset_r, size_t *name_offset_root_r,
size_t *name_offset_r)
{
buffer_t *buf;
const struct mail_index_keyword_header *kw_hdr;
const struct mail_index_keyword_header_rec *kw_rec;
const char *name;
struct mail_index_keyword_header new_kw_hdr;
uint32_t offset;
kw_hdr = CONST_PTR_OFFSET(map->hdr_base, ext->hdr_offset);
kw_rec = (const void *)(kw_hdr + 1);
name = (const char *)(kw_rec + kw_hdr->keywords_count);
if (kw_hdr->keywords_count == 0)
return NULL;
i_assert((size_t)(name - (const char *)kw_hdr) < ext->hdr_size);
new_kw_hdr = *kw_hdr;
new_kw_hdr.keywords_count += new_count;
*keywords_count_r = new_kw_hdr.keywords_count;
offset = kw_rec[kw_hdr->keywords_count-1].name_offset;
offset += strlen(name + offset) + 1;
buf = buffer_create_dynamic(pool_datastack_create(), 512);
buffer_append(buf, &new_kw_hdr, sizeof(new_kw_hdr));
buffer_append(buf, kw_rec, sizeof(*kw_rec) * kw_hdr->keywords_count);
*rec_offset_r = buf->used;
buffer_write(buf, buf->used + sizeof(*kw_rec) * new_count,
name, offset);
*name_offset_root_r = buf->used;
*name_offset_r = offset;
return buf;
}
int write_full(int fd, const void *data, size_t size)
{
ssize_t ret;
while (size > 0) {
ret = write(fd, data, size < SSIZE_T_MAX ? size : SSIZE_T_MAX);
if (unlikely(ret < 0))
return -1;
if (unlikely(ret == 0)) {
/* nothing was written, only reason for this should
be out of disk space */
errno = ENOSPC;
return -1;
}
data = CONST_PTR_OFFSET(data, ret);
size -= ret;
}
return 0;
}
static bool read_from_buffer(struct seekable_istream *sstream, ssize_t *ret_r)
{
struct istream_private *stream = &sstream->istream;
const unsigned char *data;
size_t size, pos, offset;
i_assert(stream->skip == 0);
if (stream->istream.v_offset + stream->pos >= sstream->membuf->used) {
/* need to read more */
if (sstream->membuf->used >= stream->max_buffer_size)
return FALSE;
size = sstream->cur_input == NULL ? 0 :
i_stream_get_data_size(sstream->cur_input);
if (size == 0) {
/* read more to buffer */
*ret_r = read_more(sstream);
if (*ret_r == 0 || *ret_r == -1)
return TRUE;
}
/* we should have more now. */
data = i_stream_get_data(sstream->cur_input, &size);
i_assert(size > 0);
buffer_append(sstream->membuf, data, size);
i_stream_skip(sstream->cur_input, size);
}
offset = stream->istream.v_offset;
stream->buffer = CONST_PTR_OFFSET(sstream->membuf->data, offset);
pos = sstream->membuf->used - offset;
*ret_r = pos - stream->pos;
i_assert(*ret_r > 0);
stream->pos = pos;
return TRUE;
}
static void
setting_export_section_name(string_t *str, const struct setting_define *def,
const void *set, unsigned int idx)
{
const char *const *name;
size_t name_offset;
if (def->type != SET_DEFLIST_UNIQUE) {
/* not unique, use the index */
str_printfa(str, "%u", idx);
return;
}
name_offset = def->list_info->type_offset;
i_assert(name_offset != (size_t)-1);
name = CONST_PTR_OFFSET(set, name_offset);
if (*name == NULL || **name == '\0') {
/* no name, this one isn't unique. use the index. */
str_printfa(str, "%u", idx);
} else {
str_append(str, settings_section_escape(*name));
}
}
static ssize_t i_stream_seekable_read(struct istream_private *stream)
{
struct seekable_istream *sstream = (struct seekable_istream *)stream;
const unsigned char *data;
size_t size, pos;
ssize_t ret;
if (sstream->fd == -1) {
if (read_from_buffer(sstream, &ret))
return ret;
/* copy everything to temp file and use it as the stream */
if (copy_to_temp_file(sstream) < 0) {
stream->max_buffer_size = (size_t)-1;
if (!read_from_buffer(sstream, &ret))
i_unreached();
return ret;
}
i_assert(sstream->fd != -1);
}
stream->buffer = CONST_PTR_OFFSET(stream->buffer, stream->skip);
stream->pos -= stream->skip;
stream->skip = 0;
i_assert(stream->istream.v_offset + stream->pos <= sstream->write_peak);
if (stream->istream.v_offset + stream->pos == sstream->write_peak) {
/* need to read more */
if (sstream->cur_input == NULL ||
i_stream_get_data_size(sstream->cur_input) == 0) {
ret = read_more(sstream);
if (ret == -1 || ret == 0)
return ret;
}
/* save to our file */
data = i_stream_get_data(sstream->cur_input, &size);
ret = write(sstream->fd, data, size);
if (ret <= 0) {
if (ret < 0 && !ENOSPACE(errno)) {
i_error("istream-seekable: write_full(%s) failed: %m",
sstream->temp_path);
}
if (i_stream_seekable_write_failed(sstream) < 0)
return -1;
if (!read_from_buffer(sstream, &ret))
i_unreached();
return ret;
}
i_stream_sync(sstream->fd_input);
i_stream_skip(sstream->cur_input, ret);
sstream->write_peak += ret;
}
i_stream_seek(sstream->fd_input, stream->istream.v_offset);
ret = i_stream_read_memarea(sstream->fd_input);
if (ret <= 0) {
stream->istream.eof = sstream->fd_input->eof;
stream->istream.stream_errno =
sstream->fd_input->stream_errno;
} else {
ret = -2;
}
stream->buffer = i_stream_get_data(sstream->fd_input, &pos);
stream->pos -= stream->skip;
stream->skip = 0;
ret = pos > stream->pos ? (ssize_t)(pos - stream->pos) : ret;
stream->pos = pos;
return ret;
}
int mail_index_sync_keywords(struct mail_index_sync_map_ctx *ctx,
const struct mail_transaction_header *hdr,
const struct mail_transaction_keyword_update *rec)
{
struct mail_index_view *view = ctx->view;
const char *keyword_name;
const struct mail_index_ext *ext;
const uint32_t *uid, *end;
uint32_t seqset_offset, ext_map_idx;
unsigned int keyword_idx;
int ret;
i_assert(rec->name_size > 0);
seqset_offset = sizeof(*rec) + rec->name_size;
if ((seqset_offset % 4) != 0)
seqset_offset += 4 - (seqset_offset % 4);
i_assert(seqset_offset < hdr->size);
uid = CONST_PTR_OFFSET(rec, seqset_offset);
end = CONST_PTR_OFFSET(rec, hdr->size);
keyword_name = t_strndup(rec + 1, rec->name_size);
if (!keyword_lookup(ctx, keyword_name, &keyword_idx))
keywords_header_add(ctx, keyword_name, &keyword_idx);
/* if the keyword wasn't found, the "keywords" extension was created.
if it was found, the record size should already be correct, but
in case it isn't just fix it ourself. */
if (!mail_index_map_lookup_ext(view->map, MAIL_INDEX_EXT_KEYWORDS,
&ext_map_idx))
i_unreached();
ext = array_idx(&view->map->extensions, ext_map_idx);
if (keyword_idx / CHAR_BIT >= ext->record_size) {
if (rec->modify_type == MODIFY_REMOVE) {
/* nothing to do */
return 1;
}
/* grow the record size */
keywords_ext_register(ctx, ext_map_idx, ext->reset_id,
ext->hdr_size,
array_count(&view->map->keyword_idx_map));
if (!mail_index_map_lookup_ext(view->map,
MAIL_INDEX_EXT_KEYWORDS,
&ext_map_idx))
i_unreached();
ext = array_idx(&view->map->extensions, ext_map_idx);
}
while (uid+2 <= end) {
ret = keywords_update_records(ctx, ext, keyword_idx,
rec->modify_type,
uid[0], uid[1]);
if (ret <= 0)
return ret;
uid += 2;
}
return 1;
}
int mail_index_map_parse_keywords(struct mail_index_map *map)
{
struct mail_index *index = map->index;
const struct mail_index_ext *ext;
const struct mail_index_keyword_header *kw_hdr;
const struct mail_index_keyword_header_rec *kw_rec;
const char *name;
unsigned int i, name_area_end_offset, old_count;
uint32_t idx;
if (!mail_index_map_lookup_ext(map, MAIL_INDEX_EXT_KEYWORDS, &idx)) {
if (array_is_created(&map->keyword_idx_map))
array_clear(&map->keyword_idx_map);
return 0;
}
ext = array_idx(&map->extensions, idx);
/* Extension header contains:
- struct mail_index_keyword_header
- struct mail_index_keyword_header_rec * keywords_count
- const char names[] * keywords_count
*/
i_assert(ext->hdr_offset < map->hdr.header_size);
kw_hdr = CONST_PTR_OFFSET(map->hdr_base, ext->hdr_offset);
kw_rec = (const void *)(kw_hdr + 1);
name = (const char *)(kw_rec + kw_hdr->keywords_count);
old_count = !array_is_created(&map->keyword_idx_map) ? 0 :
array_count(&map->keyword_idx_map);
/* Keywords can only be added into same mapping. Removing requires a
new mapping (recreating the index file) */
if (kw_hdr->keywords_count == old_count) {
/* nothing changed */
return 0;
}
/* make sure the header is valid */
if (kw_hdr->keywords_count < old_count) {
mail_index_set_error(index, "Corrupted index file %s: "
"Keywords removed unexpectedly",
index->filepath);
return -1;
}
if ((size_t)(name - (const char *)kw_hdr) > ext->hdr_size) {
mail_index_set_error(index, "Corrupted index file %s: "
"keywords_count larger than header size",
index->filepath);
return -1;
}
name_area_end_offset = (const char *)kw_hdr + ext->hdr_size - name;
for (i = 0; i < kw_hdr->keywords_count; i++) {
if (kw_rec[i].name_offset > name_area_end_offset) {
mail_index_set_error(index, "Corrupted index file %s: "
"name_offset points outside allocated header",
index->filepath);
return -1;
}
}
if (name[name_area_end_offset-1] != '\0') {
mail_index_set_error(index, "Corrupted index file %s: "
"Keyword header doesn't end with NUL",
index->filepath);
return -1;
}
/* create file -> index mapping */
if (!array_is_created(&map->keyword_idx_map))
i_array_init(&map->keyword_idx_map, kw_hdr->keywords_count);
#ifdef DEBUG
/* Check that existing headers are still the same. It's behind DEBUG
since it's pretty useless waste of CPU normally. */
for (i = 0; i < array_count(&map->keyword_idx_map); i++) {
const char *keyword = name + kw_rec[i].name_offset;
const unsigned int *old_idx;
unsigned int kw_idx;
old_idx = array_idx(&map->keyword_idx_map, i);
if (!mail_index_keyword_lookup(index, keyword, &kw_idx) ||
kw_idx != *old_idx) {
mail_index_set_error(index, "Corrupted index file %s: "
"Keywords changed unexpectedly",
index->filepath);
return -1;
}
}
#endif
/* Register the newly seen keywords */
i = array_count(&map->keyword_idx_map);
for (; i < kw_hdr->keywords_count; i++) {
const char *keyword = name + kw_rec[i].name_offset;
unsigned int kw_idx;
if (*keyword == '\0') {
mail_index_set_error(index, "Corrupted index file %s: "
"Empty keyword name in header",
index->filepath);
return -1;
}
mail_index_keyword_lookup_or_create(index, keyword, &kw_idx);
array_append(&map->keyword_idx_map, &kw_idx, 1);
}
return 0;
}
static bool client_exec_script(struct master_service_connection *conn)
{
ARRAY_TYPE(const_string) envs;
const char *const *args;
string_t *input;
void *buf;
size_t prev_size, scanpos;
bool header_complete = FALSE, noreply = FALSE;
ssize_t ret;
int status;
pid_t pid;
net_set_nonblock(conn->fd, FALSE);
input = t_buffer_create(IO_BLOCK_SIZE);
/* Input contains:
VERSION .. <lf>
[alarm=<secs> <lf>]
"noreply" | "-" (or anything really) <lf>
arg 1 <lf>
arg 2 <lf>
...
<lf>
DATA
This is quite a horrible protocol. If alarm is specified, it MUST be
before "noreply". If "noreply" isn't given, something other string
(typically "-") must be given which is eaten away.
*/
alarm(SCRIPT_READ_TIMEOUT_SECS);
scanpos = 1;
while (!header_complete) {
const unsigned char *pos, *end;
prev_size = input->used;
buf = buffer_append_space_unsafe(input, IO_BLOCK_SIZE);
/* peek in socket input buffer */
ret = recv(conn->fd, buf, IO_BLOCK_SIZE, MSG_PEEK);
if (ret <= 0) {
buffer_set_used_size(input, prev_size);
if (strchr(str_c(input), '\n') != NULL)
script_verify_version(t_strcut(str_c(input), '\n'));
if (ret < 0)
i_fatal("recv(MSG_PEEK) failed: %m");
i_fatal("recv(MSG_PEEK) failed: disconnected");
}
/* scan for final \n\n */
pos = CONST_PTR_OFFSET(input->data, scanpos);
end = CONST_PTR_OFFSET(input->data, prev_size + ret);
for (; pos < end; pos++) {
if (pos[-1] == '\n' && pos[0] == '\n') {
header_complete = TRUE;
pos++;
break;
}
}
scanpos = pos - (const unsigned char *)input->data;
/* read data for real (up to and including \n\n) */
ret = recv(conn->fd, buf, scanpos-prev_size, 0);
if (prev_size+ret != scanpos) {
if (ret < 0)
i_fatal("recv() failed: %m");
if (ret == 0)
i_fatal("recv() failed: disconnected");
i_fatal("recv() failed: size of definitive recv() differs from peek");
}
buffer_set_used_size(input, scanpos);
}
alarm(0);
/* drop the last two LFs */
buffer_set_used_size(input, scanpos-2);
args = t_strsplit(str_c(input), "\n");
script_verify_version(*args); args++;
t_array_init(&envs, 16);
if (*args != NULL) {
const char *p;
if (str_begins(*args, "alarm=")) {
unsigned int seconds;
if (str_to_uint(*args + 6, &seconds) < 0)
i_fatal("invalid alarm option");
alarm(seconds);
args++;
}
while (str_begins(*args, "env_")) {
const char *envname, *env;
env = t_str_tabunescape(*args+4);
p = strchr(env, '=');
if (p == NULL)
i_fatal("invalid environment variable");
envname = t_strdup_until(*args+4, p);
if (str_array_find(accepted_envs, envname))
array_append(&envs, &env, 1);
args++;
}
if (strcmp(*args, "noreply") == 0) {
noreply = TRUE;
}
if (**args == '\0')
i_fatal("empty options");
args++;
}
array_append_zero(&envs);
if (noreply) {
/* no need to fork and check exit status */
exec_child(conn, args, array_idx(&envs, 0));
i_unreached();
}
if ((pid = fork()) == (pid_t)-1) {
i_error("fork() failed: %m");
return FALSE;
}
if (pid == 0) {
/* child */
exec_child(conn, args, array_idx(&envs, 0));
i_unreached();
}
/* parent */
/* check script exit status */
if (waitpid(pid, &status, 0) < 0) {
i_error("waitpid() failed: %m");
return FALSE;
} else if (WIFEXITED(status)) {
ret = WEXITSTATUS(status);
if (ret != 0) {
i_error("Script terminated abnormally, exit status %d", (int)ret);
return FALSE;
}
} else if (WIFSIGNALED(status)) {
i_error("Script terminated abnormally, signal %d", WTERMSIG(status));
return FALSE;
} else if (WIFSTOPPED(status)) {
i_fatal("Script stopped, signal %d", WSTOPSIG(status));
return FALSE;
} else {
i_fatal("Script terminated abnormally, return status %d", status);
return FALSE;
}
return TRUE;
}
static void
settings_export(struct config_export_context *ctx,
const struct setting_parser_info *info,
bool parent_unique_deflist,
const void *set, const void *change_set)
{
const struct setting_define *def;
const void *value, *default_value, *change_value;
void *const *children = NULL, *const *change_children = NULL;
unsigned int i, count, count2, prefix_len;
const char *str;
char *key;
bool dump, dump_default = FALSE;
for (def = info->defines; def->key != NULL; def++) {
value = CONST_PTR_OFFSET(set, def->offset);
default_value = info->defaults == NULL ? NULL :
CONST_PTR_OFFSET(info->defaults, def->offset);
change_value = CONST_PTR_OFFSET(change_set, def->offset);
switch (ctx->scope) {
case CONFIG_DUMP_SCOPE_ALL:
dump_default = TRUE;
break;
case CONFIG_DUMP_SCOPE_SET:
dump_default = *((const char *)change_value) != 0;
break;
case CONFIG_DUMP_SCOPE_CHANGED:
dump_default = FALSE;
break;
}
if (!parent_unique_deflist ||
(ctx->flags & CONFIG_DUMP_FLAG_HIDE_LIST_DEFAULTS) == 0) {
/* .. */
} else if (*((const char *)change_value) == 0 &&
def->offset != info->type_offset) {
/* this is mainly for service {} blocks. if value
hasn't changed, it's the default. even if
info->defaults has a different value. */
default_value = value;
} else {
/* value is set explicitly, but we don't know the
default here. assume it's not the default. */
dump_default = TRUE;
}
dump = FALSE;
count = 0;
str_truncate(ctx->value, 0);
switch (def->type) {
case SET_BOOL:
case SET_SIZE:
case SET_UINT:
case SET_UINT_OCT:
case SET_TIME:
case SET_STR_VARS:
case SET_STR:
case SET_ENUM:
if (!config_export_type(ctx->value, value,
default_value, def->type,
dump_default, &dump))
i_unreached();
break;
case SET_DEFLIST:
case SET_DEFLIST_UNIQUE: {
const ARRAY_TYPE(void_array) *val = value;
const ARRAY_TYPE(void_array) *change_val = change_value;
if (!array_is_created(val))
break;
children = array_get(val, &count);
for (i = 0; i < count; i++) {
if (i > 0)
str_append_c(ctx->value, ' ');
setting_export_section_name(ctx->value, def, children[i], i);
}
change_children = array_get(change_val, &count2);
i_assert(count == count2);
break;
}
case SET_STRLIST: {
const ARRAY_TYPE(const_string) *val = value;
const char *const *strings;
if (!array_is_created(val))
break;
key = p_strconcat(ctx->pool, str_c(ctx->prefix),
def->key, NULL);
if (hash_table_lookup(ctx->keys, key) != NULL) {
/* already added all of these */
break;
}
hash_table_insert(ctx->keys, key, key);
/* for doveconf -n to see this KEY_LIST */
ctx->callback(key, "", CONFIG_KEY_LIST, ctx->context);
strings = array_get(val, &count);
i_assert(count % 2 == 0);
for (i = 0; i < count; i += 2) {
str = p_strdup_printf(ctx->pool, "%s%s%c%s",
str_c(ctx->prefix),
def->key,
SETTINGS_SEPARATOR,
strings[i]);
ctx->callback(str, strings[i+1],
CONFIG_KEY_NORMAL, ctx->context);
}
count = 0;
break;
}
case SET_ALIAS:
break;
}
if (str_len(ctx->value) > 0 || dump) {
key = p_strconcat(ctx->pool, str_c(ctx->prefix),
def->key, NULL);
if (hash_table_lookup(ctx->keys, key) == NULL) {
enum config_key_type type;
if (def->offset == info->type_offset &&
parent_unique_deflist)
type = CONFIG_KEY_UNIQUE_KEY;
else if (SETTING_TYPE_IS_DEFLIST(def->type))
type = CONFIG_KEY_LIST;
else
type = CONFIG_KEY_NORMAL;
ctx->callback(key, str_c(ctx->value), type,
ctx->context);
hash_table_insert(ctx->keys, key, key);
}
}
prefix_len = str_len(ctx->prefix);
for (i = 0; i < count; i++) {
str_append(ctx->prefix, def->key);
str_append_c(ctx->prefix, SETTINGS_SEPARATOR);
setting_export_section_name(ctx->prefix, def, children[i], i);
str_append_c(ctx->prefix, SETTINGS_SEPARATOR);
settings_export(ctx, def->list_info,
def->type == SET_DEFLIST_UNIQUE,
children[i], change_children[i]);
str_truncate(ctx->prefix, prefix_len);
}
}
}
int mail_cache_lookup_iter_next(struct mail_cache_lookup_iterate_ctx *ctx,
struct mail_cache_iterate_field *field_r)
{
struct mail_cache *cache = ctx->view->cache;
unsigned int field_idx;
unsigned int data_size;
uint32_t file_field;
int ret;
i_assert(ctx->remap_counter == cache->remap_counter);
if (ctx->pos + sizeof(uint32_t) > ctx->rec_size) {
if (ctx->pos != ctx->rec_size) {
mail_cache_set_corrupted(cache,
"record has invalid size");
return -1;
}
if ((ret = mail_cache_lookup_iter_next_record(ctx)) <= 0)
return ret;
}
/* return the next field */
file_field = *((const uint32_t *)CONST_PTR_OFFSET(ctx->rec, ctx->pos));
ctx->pos += sizeof(uint32_t);
if (file_field >= cache->file_fields_count) {
/* new field, have to re-read fields header to figure
out its size. don't do this if we're compressing. */
if (!cache->locked) {
if (mail_cache_header_fields_read(cache) < 0)
return -1;
}
if (file_field >= cache->file_fields_count) {
mail_cache_set_corrupted(cache,
"field index too large (%u >= %u)",
file_field, cache->file_fields_count);
return -1;
}
/* field reading might have re-mmaped the file and
caused rec pointer to break. need to get it again. */
if (mail_cache_get_record(cache, ctx->offset, &ctx->rec) < 0)
return -1;
ctx->remap_counter = cache->remap_counter;
}
field_idx = cache->file_field_map[file_field];
data_size = cache->fields[field_idx].field.field_size;
if (data_size == UINT_MAX &&
ctx->pos + sizeof(uint32_t) <= ctx->rec->size) {
/* variable size field. get its size from the file. */
data_size = *((const uint32_t *)
CONST_PTR_OFFSET(ctx->rec, ctx->pos));
ctx->pos += sizeof(uint32_t);
}
if (ctx->rec->size - ctx->pos < data_size) {
mail_cache_set_corrupted(cache,
"record continues outside its allocated size");
return -1;
}
field_r->field_idx = field_idx;
field_r->data = CONST_PTR_OFFSET(ctx->rec, ctx->pos);
field_r->size = data_size;
field_r->offset = ctx->offset + ctx->pos;
/* each record begins from 32bit aligned position */
ctx->pos += (data_size + sizeof(uint32_t)-1) & ~(sizeof(uint32_t)-1);
return 1;
}
static void log_record_print(const struct mail_transaction_header *hdr,
const void *data, size_t data_size,
uint64_t *modseq)
{
unsigned int size = hdr->size - sizeof(*hdr);
switch (hdr->type & MAIL_TRANSACTION_TYPE_MASK) {
case MAIL_TRANSACTION_EXPUNGE|MAIL_TRANSACTION_EXPUNGE_PROT: {
const struct mail_transaction_expunge *exp = data;
printf(" - uids=");
for (; size > 0; size -= sizeof(*exp), exp++) {
printf("%u-%u,", exp->uid1, exp->uid2);
}
printf("\n");
break;
}
case MAIL_TRANSACTION_EXPUNGE_GUID|MAIL_TRANSACTION_EXPUNGE_PROT: {
const struct mail_transaction_expunge_guid *exp = data;
for (; size > 0; size -= sizeof(*exp), exp++) {
printf(" - uid=%u (guid ", exp->uid);
print_data(exp->guid_128, sizeof(exp->guid_128));
printf(")\n");
}
break;
}
case MAIL_TRANSACTION_APPEND: {
const struct mail_index_record *rec = data;
printf(" - uids=");
for (; size > 0; size -= sizeof(*rec), rec++) {
printf("%u", rec->uid);
if (rec->flags != 0)
printf(" (flags=%x)", rec->flags);
printf(",");
}
printf("\n");
break;
}
case MAIL_TRANSACTION_FLAG_UPDATE: {
const struct mail_transaction_flag_update *u = data;
for (; size > 0; size -= sizeof(*u), u++) {
printf(" - uids=%u-%u (flags +%x-%x, modseq_inc_flag=%d)\n",
u->uid1, u->uid2, u->add_flags, u->remove_flags, u->modseq_inc_flag);
}
break;
}
case MAIL_TRANSACTION_HEADER_UPDATE: {
const struct mail_transaction_header_update *u = data;
log_header_update(u, data_size);
break;
}
case MAIL_TRANSACTION_EXT_INTRO: {
const struct mail_transaction_ext_intro *intro = data;
prev_intro = *intro;
printf(" - ext_id = %u\n", intro->ext_id);
printf(" - reset_id = %u\n", intro->reset_id);
printf(" - hdr_size = %u\n", intro->hdr_size);
printf(" - record_size = %u\n", intro->record_size);
printf(" - record_align = %u\n", intro->record_align);
printf(" - flags = %u\n", intro->flags);
printf(" - name_size = %u\n", intro->name_size);
if (intro->name_size > 0) {
const char *name = (const char *)(intro+1);
printf(" - name = '%.*s'\n", intro->name_size, name);
if (*modseq == 0 && intro->name_size == 6 &&
memcmp(name, "modseq", 6) == 0)
*modseq = 1;
}
break;
}
case MAIL_TRANSACTION_EXT_RESET: {
const struct mail_transaction_ext_reset *reset = data;
printf(" - new_reset_id = %u\n", reset->new_reset_id);
printf(" - preserve_data = %u\n", reset->preserve_data);
break;
}
case MAIL_TRANSACTION_EXT_HDR_UPDATE: {
const struct mail_transaction_ext_hdr_update *u = data;
printf(" - offset = %u, size = %u", u->offset, u->size);
if (sizeof(*u) + u->size <= data_size) {
printf(": ");
print_data(u + 1, u->size);
} else {
printf(" (too large)");
}
printf("\n");
break;
}
case MAIL_TRANSACTION_EXT_HDR_UPDATE32: {
const struct mail_transaction_ext_hdr_update32 *u = data;
printf(" - offset = %u, size = %u", u->offset, u->size);
if (sizeof(*u) + u->size <= data_size) {
printf(": ");
print_data(u + 1, u->size);
} else {
printf(" (too large)");
}
printf("\n");
break;
}
case MAIL_TRANSACTION_EXT_REC_UPDATE: {
const struct mail_transaction_ext_rec_update *rec = data, *end;
size_t record_size;
end = CONST_PTR_OFFSET(data, size);
record_size = (sizeof(*rec) + prev_intro.record_size + 3) & ~3;
while (rec < end) {
printf(" - uid=%u: ", rec->uid);
if (prev_intro.record_size <= (char*)end - (char *)(rec+1))
print_data(rec + 1, prev_intro.record_size);
else
printf("(record_size too large)");
printf("\n");
rec = CONST_PTR_OFFSET(rec, record_size);
}
break;
}
case MAIL_TRANSACTION_EXT_ATOMIC_INC: {
const struct mail_transaction_ext_atomic_inc *rec = data, *end;
end = CONST_PTR_OFFSET(data, size);
for (; rec < end; rec++) {
printf(" - uid=%u: ", rec->uid);
if (rec->diff > 0)
printf("+%d\n", rec->diff);
else
printf("%d\n", rec->diff);
}
break;
}
case MAIL_TRANSACTION_KEYWORD_UPDATE: {
const struct mail_transaction_keyword_update *u = data;
const uint32_t *uid;
unsigned int uid_offset;
printf(" - modify=%d, name=%.*s, uids=",
u->modify_type, u->name_size, (const char *)(u+1));
uid_offset = sizeof(*u) + u->name_size +
((u->name_size % 4) == 0 ? 0 : 4 - (u->name_size%4));
uid = (const uint32_t *)((const char *)u + uid_offset);
size -= uid_offset;
for (; size > 0; size -= sizeof(*uid)*2, uid += 2) {
printf("%u-%u,", uid[0], uid[1]);
}
printf("\n");
break;
}
case MAIL_TRANSACTION_KEYWORD_RESET: {
const struct mail_transaction_keyword_reset *u = data;
printf(" - uids=");
for (; size > 0; size -= sizeof(*u), u++) {
printf("%u-%u, ", u->uid1, u->uid2);
}
printf("\n");
break;
}
case MAIL_TRANSACTION_MODSEQ_UPDATE: {
const struct mail_transaction_modseq_update *rec, *end;
end = CONST_PTR_OFFSET(data, size);
for (rec = data; rec < end; rec++) {
printf(" - uid=%u modseq=%llu\n", rec->uid,
((unsigned long long)rec->modseq_high32 << 32) |
rec->modseq_low32);
}
break;
}
case MAIL_TRANSACTION_INDEX_DELETED:
case MAIL_TRANSACTION_INDEX_UNDELETED:
break;
case MAIL_TRANSACTION_BOUNDARY: {
const struct mail_transaction_boundary *rec = data;
printf(" - size=%u\n", rec->size);
break;
}
case MAIL_TRANSACTION_ATTRIBUTE_UPDATE: {
const char *keys = data;
const uint32_t *extra;
unsigned int i, extra_pos, extra_count = 0;
for (i = 0; i < size && keys[i] != '\0'; ) {
if (keys[i] == '+')
extra_count++;
extra_count++;
i += strlen(keys+i) + 1;
}
if (i % sizeof(uint32_t) != 0)
i += sizeof(uint32_t) - i%sizeof(uint32_t);
extra = (const void *)(keys+i);
if ((size-i) != extra_count*sizeof(uint32_t)) {
printf(" - broken entry\n");
break;
}
extra_pos = 0;
for (i = 0; i < size && keys[i] != '\0'; ) {
printf(" - %s: %s/%s : timestamp=%s",
keys[i] == '+' ? "add" : keys[i] == '-' ? "remove" : "?",
keys[i+1] == 'p' ? "private" :
keys[i+1] == 's' ? "shared" : "?error?",
keys+i+2, unixdate2str(extra[extra_pos++]));
if (keys[i] == '+')
printf(" value_len=%u", extra[extra_pos++]);
printf("\n");
i += strlen(keys+i) + 1;
}
break;
}
default:
break;
}
}
static int
mail_transaction_log_file_sync(struct mail_transaction_log_file *file)
{
const struct mail_transaction_header *hdr;
const void *data;
struct stat st;
size_t size, avail;
uint32_t trans_size = 0;
int ret;
i_assert(file->sync_offset >= file->buffer_offset);
data = buffer_get_data(file->buffer, &size);
if (file->buffer_offset + size < file->sync_offset) {
mail_transaction_log_file_set_corrupted(file,
"log file shrank (%"PRIuUOFF_T" < %"PRIuUOFF_T")",
file->buffer_offset + (uoff_t)size, file->sync_offset);
return -1;
}
while (file->sync_offset - file->buffer_offset + sizeof(*hdr) <= size) {
hdr = CONST_PTR_OFFSET(data, file->sync_offset -
file->buffer_offset);
trans_size = mail_index_offset_to_uint32(hdr->size);
if (trans_size == 0) {
/* unfinished */
return 1;
}
if (trans_size < sizeof(*hdr)) {
mail_transaction_log_file_set_corrupted(file,
"hdr.size too small (%u)", trans_size);
return -1;
}
if (file->sync_offset - file->buffer_offset + trans_size > size)
break;
/* transaction has been fully written */
if ((ret = log_file_track_sync(file, hdr, trans_size)) <= 0) {
if (ret < 0)
return -1;
break;
}
file->sync_offset += trans_size;
}
if (file->mmap_base != NULL && !file->locked) {
/* Now that all the mmaped pages have page faulted, check if
the file had changed while doing that. Only after the last
page has faulted, the size returned by fstat() can be
trusted. Otherwise it might point to a page boundary while
the next page is still being written.
Without this check we might see partial transactions,
sometimes causing "Extension record updated without intro
prefix" errors. */
if (fstat(file->fd, &st) < 0) {
log_file_set_syscall_error(file, "fstat()");
return -1;
}
if ((uoff_t)st.st_size != file->last_size) {
file->last_size = st.st_size;
return 0;
}
}
avail = file->sync_offset - file->buffer_offset;
if (avail != size) {
/* There's more data than we could sync at the moment. If the
last record's size wasn't valid, we can't know if it will
be updated unless we've locked the log. */
if (file->locked) {
mail_transaction_log_file_set_corrupted(file,
"Unexpected garbage at EOF");
return -1;
}
/* The size field will be updated soon */
mail_index_flush_read_cache(file->log->index, file->filepath,
file->fd, file->locked);
}
if (file->next != NULL &&
file->hdr.file_seq == file->next->hdr.prev_file_seq &&
file->next->hdr.prev_file_offset != file->sync_offset) {
mail_transaction_log_file_set_corrupted(file,
"Invalid transaction log size "
"(%"PRIuUOFF_T" vs %u): %s", file->sync_offset,
file->log->head->hdr.prev_file_offset, file->filepath);
return -1;
}
return 1;
}
static void log_record_print(const struct mail_transaction_header *hdr,
const void *data, uint64_t *modseq)
{
unsigned int size = hdr->size - sizeof(*hdr);
switch (hdr->type & MAIL_TRANSACTION_TYPE_MASK) {
case MAIL_TRANSACTION_EXPUNGE|MAIL_TRANSACTION_EXPUNGE_PROT: {
const struct mail_transaction_expunge *exp = data;
printf(" - uids=");
for (; size > 0; size -= sizeof(*exp), exp++) {
printf("%u-%u,", exp->uid1, exp->uid2);
}
printf("\n");
break;
}
case MAIL_TRANSACTION_EXPUNGE_GUID|MAIL_TRANSACTION_EXPUNGE_PROT: {
const struct mail_transaction_expunge_guid *exp = data;
for (; size > 0; size -= sizeof(*exp), exp++) {
printf(" - uid=%u (guid ", exp->uid);
print_data(exp->guid_128, sizeof(exp->guid_128));
printf(")\n");
}
break;
}
case MAIL_TRANSACTION_APPEND: {
const struct mail_index_record *rec = data;
printf(" - uids=");
for (; size > 0; size -= sizeof(*rec), rec++) {
printf("%u", rec->uid);
if (rec->flags != 0)
printf(" (flags=%x)", rec->flags);
printf(",");
}
printf("\n");
break;
}
case MAIL_TRANSACTION_FLAG_UPDATE: {
const struct mail_transaction_flag_update *u = data;
for (; size > 0; size -= sizeof(*u), u++) {
printf(" - uids=%u-%u (flags +%x-%x)\n",
u->uid1, u->uid2, u->add_flags, u->remove_flags);
}
break;
}
case MAIL_TRANSACTION_HEADER_UPDATE: {
const struct mail_transaction_header_update *u = data;
log_header_update(u);
break;
}
case MAIL_TRANSACTION_EXT_INTRO: {
const struct mail_transaction_ext_intro *intro = data;
prev_intro = *intro;
printf(" - ext_id = %u\n", intro->ext_id);
printf(" - reset_id = %u\n", intro->reset_id);
printf(" - hdr_size = %u\n", intro->hdr_size);
printf(" - record_size = %u\n", intro->record_size);
printf(" - record_align = %u\n", intro->record_align);
printf(" - flags = %u\n", intro->flags);
printf(" - name_size = %u\n", intro->name_size);
if (intro->name_size > 0) {
const char *name = (const char *)(intro+1);
printf(" - name = '%.*s'\n", intro->name_size, name);
if (*modseq == 0 && intro->name_size == 6 &&
memcmp(name, "modseq", 6) == 0)
*modseq = 1;
}
break;
}
case MAIL_TRANSACTION_EXT_RESET: {
const struct mail_transaction_ext_reset *reset = data;
printf(" - new_reset_id = %u\n", reset->new_reset_id);
printf(" - preserve_data = %u\n", reset->preserve_data);
break;
}
case MAIL_TRANSACTION_EXT_HDR_UPDATE: {
const struct mail_transaction_ext_hdr_update *u = data;
printf(" - offset = %u, size = %u: ", u->offset, u->size);
print_data(u + 1, u->size);
printf("\n");
break;
}
case MAIL_TRANSACTION_EXT_HDR_UPDATE32: {
const struct mail_transaction_ext_hdr_update32 *u = data;
printf(" - offset = %u, size = %u: ", u->offset, u->size);
print_data(u + 1, u->size);
printf("\n");
break;
}
case MAIL_TRANSACTION_EXT_REC_UPDATE: {
const struct mail_transaction_ext_rec_update *rec = data, *end;
size_t record_size;
end = CONST_PTR_OFFSET(data, size);
record_size = (sizeof(*rec) + prev_intro.record_size + 3) & ~3;
while (rec < end) {
printf(" - uid=%u: ", rec->uid);
print_data(rec + 1, prev_intro.record_size);
printf("\n");
rec = CONST_PTR_OFFSET(rec, record_size);
}
break;
}
case MAIL_TRANSACTION_EXT_ATOMIC_INC: {
const struct mail_transaction_ext_atomic_inc *rec = data, *end;
end = CONST_PTR_OFFSET(data, size);
for (; rec < end; rec++) {
printf(" - uid=%u: ", rec->uid);
if (rec->diff > 0)
printf("+%d\n", rec->diff);
else
printf("%d\n", rec->diff);
}
break;
}
case MAIL_TRANSACTION_KEYWORD_UPDATE: {
const struct mail_transaction_keyword_update *u = data;
const uint32_t *uid;
unsigned int uid_offset;
printf(" - modify=%d, name=%.*s, uids=",
u->modify_type, u->name_size, (const char *)(u+1));
uid_offset = sizeof(*u) + u->name_size +
((u->name_size % 4) == 0 ? 0 : 4 - (u->name_size%4));
uid = (const uint32_t *)((const char *)u + uid_offset);
size -= uid_offset;
for (; size > 0; size -= sizeof(*uid)*2, uid += 2) {
printf("%u-%u,", uid[0], uid[1]);
}
printf("\n");
break;
}
case MAIL_TRANSACTION_KEYWORD_RESET: {
const struct mail_transaction_keyword_reset *u = data;
printf(" - uids=");
for (; size > 0; size -= sizeof(*u), u++) {
printf("%u-%u, ", u->uid1, u->uid2);
}
printf("\n");
break;
}
case MAIL_TRANSACTION_MODSEQ_UPDATE: {
const struct mail_transaction_modseq_update *rec, *end;
end = CONST_PTR_OFFSET(data, size);
for (rec = data; rec < end; rec++) {
printf(" - uid=%u modseq=%llu\n", rec->uid,
((unsigned long long)rec->modseq_high32 << 32) |
rec->modseq_low32);
}
break;
}
case MAIL_TRANSACTION_INDEX_DELETED:
case MAIL_TRANSACTION_INDEX_UNDELETED:
break;
case MAIL_TRANSACTION_BOUNDARY: {
const struct mail_transaction_boundary *rec = data;
printf(" - size=%u\n", rec->size);
break;
}
default:
break;
}
}
