void array_idx_set_i(struct array *array, unsigned int idx, const void *data)
{
size_t pos;
pos = idx * array->element_size;
if (pos > array->buffer->used) {
/* index doesn't exist yet, initialize with zero */
buffer_append_zero(array->buffer, pos - array->buffer->used);
}
buffer_write(array->buffer, pos, data, array->element_size);
}
int openssl_iostream_generate_params(buffer_t *output)
{
unsigned int i;
for (i = 0; i < N_ELEMENTS(dh_param_bitsizes); i++) {
if (generate_dh_parameters(dh_param_bitsizes[i], output) < 0)
return -1;
}
buffer_append_zero(output, sizeof(int));
return 0;
}
void *array_idx_modifiable_i(struct array *array, unsigned int idx)
{
size_t pos;
pos = idx * array->element_size;
if (pos >= array->buffer->used) {
/* index doesn't exist yet, initialize with zero */
buffer_append_zero(array->buffer, pos + array->element_size -
array->buffer->used);
}
return buffer_get_space_unsafe(array->buffer, pos, array->element_size);
}
void array_idx_clear_i(struct array *array, unsigned int idx)
{
size_t pos;
pos = idx * array->element_size;
if (pos > array->buffer->used) {
/* index doesn't exist yet, initialize with zero */
buffer_append_zero(array->buffer, pos - array->buffer->used +
array->element_size);
} else {
buffer_write_zero(array->buffer, pos, array->element_size);
}
}
static void
mail_cache_compress_field(struct mail_cache_copy_context *ctx,
const struct mail_cache_iterate_field *field)
{
struct mail_cache_field *cache_field;
enum mail_cache_decision_type dec;
uint32_t file_field_idx, size32;
uint8_t *field_seen;
file_field_idx = ctx->field_file_map[field->field_idx];
if (file_field_idx == (uint32_t)-1)
return;
cache_field = &ctx->cache->fields[field->field_idx].field;
field_seen = buffer_get_space_unsafe(ctx->field_seen,
field->field_idx, 1);
if (*field_seen == ctx->field_seen_value) {
/* duplicate */
if (cache_field->type == MAIL_CACHE_FIELD_BITMASK)
mail_cache_merge_bitmask(ctx, field);
return;
}
*field_seen = ctx->field_seen_value;
dec = cache_field->decision & ~MAIL_CACHE_DECISION_FORCED;
if (ctx->new_msg) {
if (dec == MAIL_CACHE_DECISION_NO)
return;
} else {
if (dec != MAIL_CACHE_DECISION_YES)
return;
}
buffer_append(ctx->buffer, &file_field_idx, sizeof(file_field_idx));
if (cache_field->field_size == UINT_MAX) {
size32 = (uint32_t)field->size;
buffer_append(ctx->buffer, &size32, sizeof(size32));
}
if (cache_field->type == MAIL_CACHE_FIELD_BITMASK) {
/* remember the position in case we need to update it */
unsigned int pos = ctx->buffer->used;
array_idx_set(&ctx->bitmask_pos, field->field_idx, &pos);
}
buffer_append(ctx->buffer, field->data, field->size);
if ((field->size & 3) != 0)
buffer_append_zero(ctx->buffer, 4 - (field->size & 3));
}
static void
log_append_ext_hdr_update(struct mail_index_export_context *ctx,
const struct mail_index_transaction_ext_hdr_update *hdr)
{
buffer_t *buf;
const unsigned char *data, *mask;
struct mail_transaction_ext_hdr_update u;
struct mail_transaction_ext_hdr_update32 u32;
size_t offset;
bool started = FALSE, use_32 = hdr->alloc_size >= 65536;
memset(&u, 0, sizeof(u));
memset(&u32, 0, sizeof(u32));
data = hdr->data;
mask = hdr->mask;
buf = buffer_create_dynamic(pool_datastack_create(), 256);
for (offset = 0; offset <= hdr->alloc_size; offset++) {
if (offset < hdr->alloc_size && mask[offset] != 0) {
if (!started) {
u32.offset = offset;
started = TRUE;
}
} else {
if (started) {
u32.size = offset - u32.offset;
if (use_32)
buffer_append(buf, &u32, sizeof(u32));
else {
u.offset = u32.offset;
u.size = u32.size;
buffer_append(buf, &u, sizeof(u));
}
buffer_append(buf, data + u32.offset, u32.size);
started = FALSE;
}
}
}
if (buf->used % 4 != 0)
buffer_append_zero(buf, 4 - buf->used % 4);
log_append_buffer(ctx, buf, use_32 ? MAIL_TRANSACTION_EXT_HDR_UPDATE32 :
MAIL_TRANSACTION_EXT_HDR_UPDATE);
}
static void log_write_ext_hdr_init_data(struct mail_index *index, buffer_t *buf)
{
const struct mail_index_registered_ext *rext;
struct mail_transaction_header *hdr;
struct mail_transaction_ext_intro *intro;
struct mail_transaction_ext_hdr_update *ext_hdr;
unsigned int hdr_offset;
rext = array_idx(&index->extensions, index->ext_hdr_init_id);
/* introduce the extension */
hdr_offset = buf->used;
hdr = buffer_append_space_unsafe(buf, sizeof(*hdr));
hdr->type = MAIL_TRANSACTION_EXT_INTRO;
intro = buffer_append_space_unsafe(buf, sizeof(*intro));
intro->ext_id = (uint32_t)-1;
intro->hdr_size = rext->hdr_size;
intro->record_size = rext->record_size;
intro->record_align = rext->record_align;
intro->name_size = strlen(rext->name);
buffer_append(buf, rext->name, intro->name_size);
if (buf->used % 4 != 0)
buffer_append_zero(buf, 4 - buf->used % 4);
hdr = buffer_get_space_unsafe(buf, hdr_offset, sizeof(*hdr));
hdr->size = mail_index_uint32_to_offset(buf->used - hdr_offset);
/* add the extension header data */
hdr_offset = buf->used;
hdr = buffer_append_space_unsafe(buf, sizeof(*hdr));
hdr->type = MAIL_TRANSACTION_EXT_HDR_UPDATE;
ext_hdr = buffer_append_space_unsafe(buf, sizeof(*ext_hdr));
ext_hdr->size = rext->hdr_size;
buffer_append(buf, index->ext_hdr_init_data, rext->hdr_size);
hdr = buffer_get_space_unsafe(buf, hdr_offset, sizeof(*hdr));
hdr->size = mail_index_uint32_to_offset(buf->used - hdr_offset);
}
static void index_mail_parse_header_finish(struct index_mail *mail)
{
struct mail *_mail = &mail->mail.mail;
const struct index_mail_line *lines;
const unsigned char *header, *data;
const uint8_t *match;
buffer_t *buf;
size_t data_size;
unsigned int i, j, count, match_idx, match_count;
bool noncontiguous;
/* sort it first so fields are grouped together and ordered by
line number */
array_sort(&mail->header_lines, header_line_cmp);
lines = array_get(&mail->header_lines, &count);
match = array_get(&mail->header_match, &match_count);
header = buffer_get_data(mail->header_data, NULL);
buf = buffer_create_dynamic(pool_datastack_create(), 256);
/* go through all the header lines we found */
for (i = match_idx = 0; i < count; i = j) {
/* matches and header lines are both sorted, all matches
until lines[i] weren't found */
while (match_idx < lines[i].field_idx &&
match_idx < match_count) {
if (HEADER_MATCH_USABLE(mail, match[match_idx]) &&
mail_cache_field_can_add(_mail->transaction->cache_trans,
_mail->seq, match_idx)) {
/* this header doesn't exist. remember that. */
i_assert((match[match_idx] &
HEADER_MATCH_FLAG_FOUND) == 0);
index_mail_cache_add_idx(mail, match_idx,
"", 0);
}
match_idx++;
}
if (match_idx < match_count) {
/* save index to first header line */
j = i + 1;
array_idx_set(&mail->header_match_lines, match_idx, &j);
match_idx++;
}
if (!mail_cache_field_can_add(_mail->transaction->cache_trans,
_mail->seq, lines[i].field_idx)) {
/* header is already cached */
j = i + 1;
continue;
}
/* buffer contains: { uint32_t line_num[], 0, header texts }
noncontiguous is just a small optimization.. */
buffer_set_used_size(buf, 0);
buffer_append(buf, &lines[i].line_num,
sizeof(lines[i].line_num));
noncontiguous = FALSE;
for (j = i+1; j < count; j++) {
if (lines[j].field_idx != lines[i].field_idx)
break;
if (lines[j].start_pos != lines[j-1].end_pos)
noncontiguous = TRUE;
buffer_append(buf, &lines[j].line_num,
sizeof(lines[j].line_num));
}
buffer_append_zero(buf, sizeof(uint32_t));
if (noncontiguous) {
for (; i < j; i++) {
buffer_append(buf, header + lines[i].start_pos,
lines[i].end_pos -
lines[i].start_pos);
}
i--;
} else {
buffer_append(buf, header + lines[i].start_pos,
lines[j-1].end_pos - lines[i].start_pos);
}
data = buffer_get_data(buf, &data_size);
index_mail_cache_add_idx(mail, lines[i].field_idx,
data, data_size);
}
for (; match_idx < match_count; match_idx++) {
if (HEADER_MATCH_USABLE(mail, match[match_idx]) &&
mail_cache_field_can_add(_mail->transaction->cache_trans,
_mail->seq, match_idx)) {
/* this header doesn't exist. remember that. */
i_assert((match[match_idx] &
HEADER_MATCH_FLAG_FOUND) == 0);
index_mail_cache_add_idx(mail, match_idx, "", 0);
}
}
mail->data.dont_cache_field_idx = UINT_MAX;
}
static void
keywords_header_add(struct mail_index_sync_map_ctx *ctx,
const char *keyword_name, unsigned int *keyword_idx_r)
{
struct mail_index_map *map;
const struct mail_index_ext *ext = NULL;
struct mail_index_keyword_header *kw_hdr;
struct mail_index_keyword_header_rec kw_rec;
uint32_t ext_map_idx;
buffer_t *buf = NULL;
size_t keyword_len, rec_offset, name_offset, name_offset_root;
unsigned int keywords_count;
/* if we crash in the middle of writing the header, the
keywords are more or less corrupted. avoid that by
making sure the header is updated atomically. */
map = mail_index_sync_get_atomic_map(ctx);
if (!mail_index_map_lookup_ext(map, MAIL_INDEX_EXT_KEYWORDS,
&ext_map_idx))
ext_map_idx = (uint32_t)-1;
else {
/* update existing header */
ext = array_idx(&map->extensions, ext_map_idx);
buf = keywords_get_header_buf(map, ext, 1, &keywords_count,
&rec_offset, &name_offset_root,
&name_offset);
}
if (buf == NULL) {
/* create new / replace broken header */
const unsigned int initial_keywords_count = 1;
buf = buffer_create_dynamic(pool_datastack_create(), 512);
kw_hdr = buffer_append_space_unsafe(buf, sizeof(*kw_hdr));
kw_hdr->keywords_count = initial_keywords_count;
keywords_count = kw_hdr->keywords_count;
rec_offset = buf->used;
name_offset_root = rec_offset +
initial_keywords_count * sizeof(kw_rec);
name_offset = 0;
}
/* add the keyword */
memset(&kw_rec, 0, sizeof(kw_rec));
kw_rec.name_offset = name_offset;
keyword_len = strlen(keyword_name) + 1;
buffer_write(buf, rec_offset, &kw_rec, sizeof(kw_rec));
buffer_write(buf, name_offset_root, keyword_name, keyword_len);
rec_offset += sizeof(kw_rec);
kw_rec.name_offset += keyword_len;
name_offset_root += keyword_len;
if ((buf->used % 4) != 0)
buffer_append_zero(buf, 4 - (buf->used % 4));
if (ext == NULL || buf->used > ext->hdr_size ||
(uint32_t)ext->record_size * CHAR_BIT < keywords_count) {
/* if we need to grow the buffer, add some padding */
buffer_append_zero(buf, 128);
keywords_ext_register(ctx, ext_map_idx,
ext == NULL ? 0 : ext->reset_id,
buf->used, keywords_count);
/* map may have changed */
map = ctx->view->map;
if (!mail_index_map_lookup_ext(map, MAIL_INDEX_EXT_KEYWORDS,
&ext_map_idx))
i_unreached();
ext = array_idx(&map->extensions, ext_map_idx);
i_assert(ext->hdr_size == buf->used);
}
buffer_copy(map->hdr_copy_buf, ext->hdr_offset, buf, 0, buf->used);
map->hdr_base = map->hdr_copy_buf->data;
i_assert(map->hdr_copy_buf->used == map->hdr.header_size);
if (mail_index_map_parse_keywords(map) < 0)
i_panic("Keyword update corrupted keywords header");
*keyword_idx_r = keywords_count - 1;
i_assert(*keyword_idx_r / CHAR_BIT < ext->record_size);
}
void test_buffer(void)
{
#define BUF_TEST_SIZE (1024*2)
#define BUF_TEST_COUNT 1000
buffer_t *buf;
unsigned char *p, testdata[BUF_TEST_SIZE], shadowbuf[BUF_TEST_SIZE];
unsigned int i, shadowbuf_size;
size_t pos, pos2, size;
int test = -1;
bool zero;
buf = buffer_create_dynamic(default_pool, 1);
for (i = 0; i < BUF_TEST_SIZE; i++)
testdata[i] = random();
memset(shadowbuf, 0, sizeof(shadowbuf));
srand(1);
shadowbuf_size = 0;
for (i = 0; i < BUF_TEST_COUNT; i++) {
if (buf->used == BUF_TEST_SIZE) {
size = shadowbuf_size = rand() % (buf->used - 1);
buffer_set_used_size(buf, size);
memset(shadowbuf + shadowbuf_size, 0,
BUF_TEST_SIZE - shadowbuf_size);
i_assert(buf->used < BUF_TEST_SIZE);
}
test = rand() % 6;
zero = rand() % 10 == 0;
switch (test) {
case 0:
pos = rand() % (BUF_TEST_SIZE-1);
size = rand() % (BUF_TEST_SIZE - pos);
if (!zero) {
buffer_write(buf, pos, testdata, size);
memcpy(shadowbuf + pos, testdata, size);
} else {
buffer_write_zero(buf, pos, size);
memset(shadowbuf + pos, 0, size);
}
if (pos + size > shadowbuf_size)
shadowbuf_size = pos + size;
break;
case 1:
size = rand() % (BUF_TEST_SIZE - buf->used);
if (!zero) {
buffer_append(buf, testdata, size);
memcpy(shadowbuf + shadowbuf_size,
testdata, size);
} else {
buffer_append_zero(buf, size);
memset(shadowbuf + shadowbuf_size, 0, size);
}
shadowbuf_size += size;
break;
case 2:
pos = rand() % (BUF_TEST_SIZE-1);
size = rand() % (BUF_TEST_SIZE - I_MAX(buf->used, pos));
if (!zero) {
buffer_insert(buf, pos, testdata, size);
memmove(shadowbuf + pos + size,
shadowbuf + pos,
BUF_TEST_SIZE - (pos + size));
memcpy(shadowbuf + pos, testdata, size);
} else {
buffer_insert_zero(buf, pos, size);
memmove(shadowbuf + pos + size,
shadowbuf + pos,
BUF_TEST_SIZE - (pos + size));
memset(shadowbuf + pos, 0, size);
}
if (pos < shadowbuf_size)
shadowbuf_size += size;
else
shadowbuf_size = pos + size;
break;
case 3:
pos = rand() % (BUF_TEST_SIZE-1);
size = rand() % (BUF_TEST_SIZE - pos);
buffer_delete(buf, pos, size);
if (pos < shadowbuf_size) {
if (pos + size > shadowbuf_size)
size = shadowbuf_size - pos;
memmove(shadowbuf + pos,
shadowbuf + pos + size,
BUF_TEST_SIZE - (pos + size));
shadowbuf_size -= size;
memset(shadowbuf + shadowbuf_size, 0,
BUF_TEST_SIZE - shadowbuf_size);
}
break;
case 4:
if (shadowbuf_size == 0)
break;
pos = rand() % (shadowbuf_size-1); /* dest */
pos2 = rand() % (shadowbuf_size-1); /* source */
size = rand() % (shadowbuf_size - I_MAX(pos, pos2));
buffer_copy(buf, pos, buf, pos2, size);
memmove(shadowbuf + pos,
shadowbuf + pos2, size);
if (pos > pos2 && pos + size > shadowbuf_size)
shadowbuf_size = pos + size;
break;
case 5:
pos = rand() % (BUF_TEST_SIZE-1);
size = rand() % (BUF_TEST_SIZE - pos);
p = buffer_get_space_unsafe(buf, pos, size);
memcpy(p, testdata, size);
memcpy(shadowbuf + pos, testdata, size);
if (pos + size > shadowbuf_size)
shadowbuf_size = pos + size;
break;
}
i_assert(shadowbuf_size <= BUF_TEST_SIZE);
if (buf->used != shadowbuf_size ||
memcmp(buf->data, shadowbuf, buf->used) != 0)
break;
}
if (i == BUF_TEST_COUNT)
test_out("buffer", TRUE);
else {
test_out_reason("buffer", FALSE,
t_strdup_printf("round %u test %d failed", i, test));
}
buffer_free(&buf);
}
static void log_append_ext_intro(struct mail_index_export_context *ctx,
uint32_t ext_id, uint32_t reset_id)
{
struct mail_index_transaction *t = ctx->trans;
const struct mail_index_registered_ext *rext;
struct mail_transaction_ext_intro *intro, *resizes;
buffer_t *buf;
uint32_t idx;
unsigned int count;
i_assert(ext_id != (uint32_t)-1);
if (t->reset ||
!mail_index_map_get_ext_idx(t->view->index->map, ext_id, &idx)) {
/* new extension */
idx = (uint32_t)-1;
}
rext = array_idx(&t->view->index->extensions, ext_id);
if (!array_is_created(&t->ext_resizes)) {
resizes = NULL;
count = 0;
} else {
resizes = array_get_modifiable(&t->ext_resizes, &count);
}
buf = buffer_create_dynamic(pool_datastack_create(), 128);
if (ext_id < count && resizes[ext_id].name_size != 0) {
/* we're resizing the extension. use the resize struct. */
intro = &resizes[ext_id];
i_assert(intro->ext_id == idx);
intro->name_size = idx != (uint32_t)-1 ? 0 :
strlen(rext->name);
buffer_append(buf, intro, sizeof(*intro));
} else {
/* generate a new intro structure */
intro = buffer_append_space_unsafe(buf, sizeof(*intro));
intro->ext_id = idx;
intro->hdr_size = rext->hdr_size;
intro->record_size = rext->record_size;
intro->record_align = rext->record_align;
intro->flags = MAIL_TRANSACTION_EXT_INTRO_FLAG_NO_SHRINK;
intro->name_size = idx != (uint32_t)-1 ? 0 :
strlen(rext->name);
}
if (reset_id != 0) {
/* we're going to reset this extension in this transaction */
intro->reset_id = reset_id;
} else if (idx != (uint32_t)-1) {
/* use the existing reset_id */
const struct mail_index_ext *map_ext =
array_idx(&t->view->index->map->extensions, idx);
intro->reset_id = map_ext->reset_id;
} else {
/* new extension, reset_id defaults to 0 */
}
buffer_append(buf, rext->name, intro->name_size);
if ((buf->used % 4) != 0)
buffer_append_zero(buf, 4 - (buf->used % 4));
if (ctx->append_ctx->new_highest_modseq == 0 &&
strcmp(rext->name, MAIL_INDEX_MODSEQ_EXT_NAME) == 0) {
/* modseq tracking started */
ctx->append_ctx->new_highest_modseq = 1;
}
log_append_buffer(ctx, buf, MAIL_TRANSACTION_EXT_INTRO);
}
