void fts_icu_utf8_to_utf16(buffer_t *dest_utf16, const char *src_utf8)
{
UErrorCode err = U_ZERO_ERROR;
unsigned int src_bytes = strlen(src_utf8);
int32_t utf16_len;
UChar *dest_data, *retp = NULL;
int32_t avail_uchars = 0;
/* try to encode with the current buffer size */
avail_uchars = buffer_get_writable_size(dest_utf16) / sizeof(UChar);
dest_data = buffer_get_space_unsafe(dest_utf16, 0,
buffer_get_writable_size(dest_utf16));
retp = u_strFromUTF8Lenient(dest_data, avail_uchars,
&utf16_len, src_utf8, src_bytes, &err);
if (err == U_BUFFER_OVERFLOW_ERROR) {
/* try again with a larger buffer */
dest_data = buffer_get_space_unsafe(dest_utf16, 0,
utf16_len * sizeof(UChar));
err = U_ZERO_ERROR;
retp = u_strFromUTF8Lenient(dest_data, utf16_len,
&utf16_len, src_utf8,
src_bytes, &err);
}
if (U_FAILURE(err)) {
i_panic("LibICU u_strFromUTF8Lenient() failed: %s",
u_errorName(err));
}
buffer_set_used_size(dest_utf16, utf16_len * sizeof(UChar));
i_assert(retp == dest_data);
}
void fts_icu_utf16_to_utf8(string_t *dest_utf8, const UChar *src_utf16,
unsigned int src_len)
{
int32_t dest_len = 0;
int32_t sub_num = 0;
char *dest_data, *retp = NULL;
UErrorCode err = U_ZERO_ERROR;
/* try to encode with the current buffer size */
dest_data = buffer_get_space_unsafe(dest_utf8, 0,
buffer_get_writable_size(dest_utf8));
retp = u_strToUTF8WithSub(dest_data, buffer_get_writable_size(dest_utf8),
&dest_len, src_utf16, src_len,
UNICODE_REPLACEMENT_CHAR, &sub_num, &err);
if (err == U_BUFFER_OVERFLOW_ERROR) {
/* try again with a larger buffer */
dest_data = buffer_get_space_unsafe(dest_utf8, 0, dest_len);
err = U_ZERO_ERROR;
retp = u_strToUTF8WithSub(dest_data, buffer_get_writable_size(dest_utf8), &dest_len,
src_utf16, src_len,
UNICODE_REPLACEMENT_CHAR,
&sub_num, &err);
}
if (U_FAILURE(err)) {
i_panic("LibICU u_strToUTF8WithSub() failed: %s",
u_errorName(err));
}
buffer_set_used_size(dest_utf8, dest_len);
i_assert(retp == dest_data);
}
void fts_icu_lcase(string_t *dest_utf8, const char *src_utf8)
{
struct UCaseMap *csm = fts_icu_csm();
size_t avail_bytes, dest_pos = dest_utf8->used;
char *dest_data;
int dest_full_len;
UErrorCode err = U_ZERO_ERROR;
avail_bytes = buffer_get_writable_size(dest_utf8) - dest_pos;
dest_data = buffer_get_space_unsafe(dest_utf8, dest_pos, avail_bytes);
dest_full_len = ucasemap_utf8ToLower(csm, dest_data, avail_bytes,
src_utf8, -1, &err);
if (err == U_BUFFER_OVERFLOW_ERROR) {
err = U_ZERO_ERROR;
dest_data = buffer_get_space_unsafe(dest_utf8, dest_pos, dest_full_len);
dest_full_len = ucasemap_utf8ToLower(csm, dest_data, dest_full_len,
src_utf8, -1, &err);
i_assert(err != U_BUFFER_OVERFLOW_ERROR);
}
if (U_FAILURE(err)) {
i_fatal("LibICU ucasemap_utf8ToLower() failed: %s",
u_errorName(err));
}
buffer_set_used_size(dest_utf8, dest_full_len);
}
static void test_buffer_set_used_size(void)
{
buffer_t *buf;
test_begin("buffer_set_used_size");
buf = t_buffer_create(8);
memset(buffer_append_space_unsafe(buf, 7), 'a', 7);
buffer_set_used_size(buf, 4);
test_assert(memcmp(buffer_get_space_unsafe(buf, 0, 7), "aaaa\0\0\0", 7) == 0);
memset(buffer_get_space_unsafe(buf, 4, 7), 'b', 7);
buffer_set_used_size(buf, 10);
test_assert(memcmp(buffer_append_space_unsafe(buf, 1), "\0", 1) == 0);
buffer_set_used_size(buf, 11);
test_assert(memcmp(buffer_get_space_unsafe(buf, 0, 11), "aaaabbbbbb\0", 11) == 0);
test_end();
}
static int
mail_transaction_log_file_insert_read(struct mail_transaction_log_file *file,
uoff_t offset)
{
void *data;
size_t size;
ssize_t ret;
size = file->buffer_offset - offset;
buffer_copy(file->buffer, size, file->buffer, 0, (size_t)-1);
data = buffer_get_space_unsafe(file->buffer, 0, size);
ret = pread_full(file->fd, data, size, offset);
if (ret > 0) {
/* success */
file->buffer_offset -= size;
return 1;
}
/* failure. don't leave ourself to inconsistent state */
buffer_copy(file->buffer, 0, file->buffer, size, (size_t)-1);
buffer_set_used_size(file->buffer, file->buffer->used - size);
if (ret == 0) {
mail_transaction_log_file_set_corrupted(file, "file shrank");
return 0;
} else if (errno == ESTALE) {
/* log file was deleted in NFS server, fail silently */
return 0;
} else {
log_file_set_syscall_error(file, "pread()");
return -1;
}
}
static int
mail_cache_lookup_bitmask(struct mail_cache_lookup_iterate_ctx *iter,
unsigned int field_idx, unsigned int field_size,
buffer_t *dest_buf)
{
struct mail_cache_iterate_field field;
const unsigned char *src;
unsigned char *dest;
unsigned int i;
bool found = FALSE;
int ret;
/* make sure all bits are cleared first */
buffer_write_zero(dest_buf, 0, field_size);
while ((ret = mail_cache_lookup_iter_next(iter, &field)) > 0) {
if (field.field_idx != field_idx)
continue;
/* merge all bits */
src = field.data;
dest = buffer_get_space_unsafe(dest_buf, 0, field.size);
for (i = 0; i < field.size; i++)
dest[i] |= src[i];
found = TRUE;
}
return ret < 0 ? -1 : (found ? 1 : 0);
}
void mbox_sync_headers_add_space(struct mbox_sync_mail_context *ctx,
size_t size)
{
size_t data_size, pos, start_pos;
const unsigned char *data;
void *p;
i_assert(size < SSIZE_T_MAX);
if (ctx->mail.pseudo)
start_pos = ctx->hdr_pos[MBOX_HDR_X_IMAPBASE];
else if (ctx->mail.space > 0) {
/* update the header using the existing offset.
otherwise we might chose wrong header and just decrease
the available space */
start_pos = ctx->mail.offset - ctx->hdr_offset;
} else {
/* Append at the end of X-Keywords header,
or X-UID if it doesn't exist */
start_pos = ctx->hdr_pos[MBOX_HDR_X_KEYWORDS] != (size_t)-1 ?
ctx->hdr_pos[MBOX_HDR_X_KEYWORDS] :
ctx->hdr_pos[MBOX_HDR_X_UID];
}
data = str_data(ctx->header);
data_size = str_len(ctx->header);
i_assert(start_pos < data_size);
for (pos = start_pos; pos < data_size; pos++) {
if (data[pos] == '\n') {
/* possibly continues in next line */
if (pos+1 == data_size || !IS_LWSP(data[pos+1]))
break;
start_pos = pos+1;
} else if (!IS_LWSP(data[pos]) && data[pos] != '\r') {
start_pos = pos+1;
}
}
/* pos points to end of header now, and start_pos to beginning
of whitespace. */
mbox_sync_move_buffer(ctx, pos, size, 0);
p = buffer_get_space_unsafe(ctx->header, pos, size);
memset(p, ' ', size);
if (ctx->header_first_change > pos)
ctx->header_first_change = pos;
ctx->header_last_change = (size_t)-1;
ctx->mail.space = (pos - start_pos) + size;
ctx->mail.offset = ctx->hdr_offset;
if (ctx->mail.space > 0)
ctx->mail.offset += start_pos;
}
int fts_icu_translate(buffer_t *dest_utf16, const UChar *src_utf16,
unsigned int src_len, UTransliterator *transliterator,
const char **error_r)
{
UErrorCode err = U_ZERO_ERROR;
int32_t utf16_len = src_len;
UChar *dest_data;
int32_t avail_uchars, limit = src_len;
size_t dest_pos = dest_utf16->used;
/* translation is done in-place in the buffer. try first with the
current buffer size. */
buffer_append(dest_utf16, src_utf16, src_len*sizeof(UChar));
avail_uchars = (buffer_get_writable_size(dest_utf16)-dest_pos) / sizeof(UChar);
dest_data = buffer_get_space_unsafe(dest_utf16, dest_pos,
buffer_get_writable_size(dest_utf16)-dest_pos);
utrans_transUChars(transliterator, dest_data, &utf16_len,
avail_uchars, 0, &limit, &err);
if (err == U_BUFFER_OVERFLOW_ERROR) {
/* try again with a larger buffer */
err = U_ZERO_ERROR;
avail_uchars = utf16_len;
limit = utf16_len = src_len;
buffer_write(dest_utf16, dest_pos,
src_utf16, src_len*sizeof(UChar));
dest_data = buffer_get_space_unsafe(dest_utf16, dest_pos,
avail_uchars * sizeof(UChar));
utrans_transUChars(transliterator, dest_data, &utf16_len,
avail_uchars, 0, &limit, &err);
i_assert(err != U_BUFFER_OVERFLOW_ERROR);
}
if (U_FAILURE(err)) {
*error_r = t_strdup_printf("LibICU utrans_transUChars() failed: %s",
u_errorName(err));
buffer_set_used_size(dest_utf16, dest_pos);
return -1;
}
buffer_set_used_size(dest_utf16, utf16_len * sizeof(UChar));
return 0;
}
void *array_idx_modifiable_i(struct array *array, unsigned int idx)
{
size_t pos;
pos = idx * array->element_size;
if (pos >= array->buffer->used) {
/* index doesn't exist yet, initialize with zero */
buffer_append_zero(array->buffer, pos + array->element_size -
array->buffer->used);
}
return buffer_get_space_unsafe(array->buffer, pos, array->element_size);
}
static buffer_t *_file_lazy_load_buffer
(struct sieve_binary_file *file, off_t *offset, size_t size)
{
buffer_t *buffer = buffer_create_dynamic(file->pool, size);
if ( _file_lazy_read
(file, offset, buffer_get_space_unsafe(buffer, 0, size), size) ) {
return buffer;
}
return NULL;
}
static void log_write_ext_hdr_init_data(struct mail_index *index, buffer_t *buf)
{
const struct mail_index_registered_ext *rext;
struct mail_transaction_header *hdr;
struct mail_transaction_ext_intro *intro;
struct mail_transaction_ext_hdr_update *ext_hdr;
unsigned int hdr_offset;
rext = array_idx(&index->extensions, index->ext_hdr_init_id);
/* introduce the extension */
hdr_offset = buf->used;
hdr = buffer_append_space_unsafe(buf, sizeof(*hdr));
hdr->type = MAIL_TRANSACTION_EXT_INTRO;
intro = buffer_append_space_unsafe(buf, sizeof(*intro));
intro->ext_id = (uint32_t)-1;
intro->hdr_size = rext->hdr_size;
intro->record_size = rext->record_size;
intro->record_align = rext->record_align;
intro->name_size = strlen(rext->name);
buffer_append(buf, rext->name, intro->name_size);
if (buf->used % 4 != 0)
buffer_append_zero(buf, 4 - buf->used % 4);
hdr = buffer_get_space_unsafe(buf, hdr_offset, sizeof(*hdr));
hdr->size = mail_index_uint32_to_offset(buf->used - hdr_offset);
/* add the extension header data */
hdr_offset = buf->used;
hdr = buffer_append_space_unsafe(buf, sizeof(*hdr));
hdr->type = MAIL_TRANSACTION_EXT_HDR_UPDATE;
ext_hdr = buffer_append_space_unsafe(buf, sizeof(*ext_hdr));
ext_hdr->size = rext->hdr_size;
buffer_append(buf, index->ext_hdr_init_data, rext->hdr_size);
hdr = buffer_get_space_unsafe(buf, hdr_offset, sizeof(*hdr));
hdr->size = mail_index_uint32_to_offset(buf->used - hdr_offset);
}
void str_vprintfa(string_t *str, const char *fmt, va_list args)
{
#define SNPRINTF_INITIAL_EXTRA_SIZE 128
va_list args2;
char *tmp;
unsigned int init_size;
size_t pos = str->used;
int ret, ret2;
VA_COPY(args2, args);
/* the format string is modified only if %m exists in it. it happens
only in error conditions, so don't try to t_push() here since it'll
just slow down the normal code path. */
fmt = printf_format_fix_get_len(fmt, &init_size);
init_size += SNPRINTF_INITIAL_EXTRA_SIZE;
/* @UNSAFE */
if (pos+init_size > buffer_get_size(str) &&
pos < buffer_get_size(str)) {
/* avoid growing buffer larger if possible. this is also
required if buffer isn't dynamically growing. */
init_size = buffer_get_size(str)-pos;
}
tmp = buffer_get_space_unsafe(str, pos, init_size);
ret = vsnprintf(tmp, init_size, fmt, args);
i_assert(ret >= 0);
if ((unsigned int)ret >= init_size) {
/* didn't fit with the first guess. now we know the size,
so try again. */
tmp = buffer_get_space_unsafe(str, pos, ret + 1);
ret2 = vsnprintf(tmp, ret + 1, fmt, args2);
i_assert(ret2 == ret);
}
/* drop the unused data, including terminating NUL */
buffer_set_used_size(str, pos + ret);
}
void *array_insert_space_i(struct array *array, unsigned int idx)
{
void *data;
size_t pos;
pos = idx * array->element_size;
buffer_copy(array->buffer, pos + array->element_size,
array->buffer, pos, (size_t)-1);
data = buffer_get_space_unsafe(array->buffer, pos, array->element_size);
memset(data, 0, array->element_size);
return data;
}
static void
mail_cache_compress_field(struct mail_cache_copy_context *ctx,
const struct mail_cache_iterate_field *field)
{
struct mail_cache_field *cache_field;
enum mail_cache_decision_type dec;
uint32_t file_field_idx, size32;
uint8_t *field_seen;
file_field_idx = ctx->field_file_map[field->field_idx];
if (file_field_idx == (uint32_t)-1)
return;
cache_field = &ctx->cache->fields[field->field_idx].field;
field_seen = buffer_get_space_unsafe(ctx->field_seen,
field->field_idx, 1);
if (*field_seen == ctx->field_seen_value) {
/* duplicate */
if (cache_field->type == MAIL_CACHE_FIELD_BITMASK)
mail_cache_merge_bitmask(ctx, field);
return;
}
*field_seen = ctx->field_seen_value;
dec = cache_field->decision & ~MAIL_CACHE_DECISION_FORCED;
if (ctx->new_msg) {
if (dec == MAIL_CACHE_DECISION_NO)
return;
} else {
if (dec != MAIL_CACHE_DECISION_YES)
return;
}
buffer_append(ctx->buffer, &file_field_idx, sizeof(file_field_idx));
if (cache_field->field_size == UINT_MAX) {
size32 = (uint32_t)field->size;
buffer_append(ctx->buffer, &size32, sizeof(size32));
}
if (cache_field->type == MAIL_CACHE_FIELD_BITMASK) {
/* remember the position in case we need to update it */
unsigned int pos = ctx->buffer->used;
array_idx_set(&ctx->bitmask_pos, field->field_idx, &pos);
}
buffer_append(ctx->buffer, field->data, field->size);
if ((field->size & 3) != 0)
buffer_append_zero(ctx->buffer, 4 - (field->size & 3));
}
static void status_flags_replace(struct mbox_sync_mail_context *ctx, size_t pos,
const struct mbox_flag_type *flags_list)
{
unsigned char *data;
size_t size;
int i, need, have;
ctx->mail.flags ^= MBOX_NONRECENT_KLUDGE;
if (ctx->header_first_change > pos)
ctx->header_first_change = pos;
/* how many bytes do we need? */
for (i = 0, need = 0; flags_list[i].chr != 0; i++) {
if ((ctx->mail.flags & flags_list[i].flag) != 0)
need++;
}
/* how many bytes do we have now? */
data = buffer_get_modifiable_data(ctx->header, &size);
for (have = 0; pos < size; pos++) {
if (data[pos] == '\n' || data[pos] == '\r')
break;
/* see if this is unknown flag for us */
for (i = 0; flags_list[i].chr != 0; i++) {
if (flags_list[i].chr == (char)data[pos])
break;
}
if (flags_list[i].chr != 0)
have++;
else {
/* save this one */
data[pos-have] = data[pos];
}
}
pos -= have;
mbox_sync_move_buffer(ctx, pos, need, have);
/* @UNSAFE */
data = buffer_get_space_unsafe(ctx->header, pos, need);
for (i = 0; flags_list[i].chr != 0; i++) {
if ((ctx->mail.flags & flags_list[i].flag) != 0)
*data++ = flags_list[i].chr;
}
ctx->mail.flags ^= MBOX_NONRECENT_KLUDGE;
}
buffer_t *t_hmac_data(const struct hash_method *meth,
const unsigned char *key, size_t key_len,
const void *data, size_t data_len)
{
struct hmac_context ctx;
i_assert(meth != NULL);
i_assert(key != NULL && key_len > 0);
i_assert(data != NULL || data_len == 0);
buffer_t *res = buffer_create_dynamic(pool_datastack_create(), meth->digest_size);
hmac_init(&ctx, key, key_len, meth);
if (data_len > 0)
hmac_update(&ctx, data, data_len);
unsigned char *buf = buffer_get_space_unsafe(res, 0, meth->digest_size);
hmac_final(&ctx, buf);
return res;
}
static void
mail_cache_merge_bitmask(struct mail_cache_copy_context *ctx,
const struct mail_cache_iterate_field *field)
{
unsigned char *dest;
unsigned int i, *pos;
pos = array_idx_modifiable(&ctx->bitmask_pos, field->field_idx);
if (*pos == 0) {
/* we decided to drop this field */
return;
}
dest = buffer_get_space_unsafe(ctx->buffer, *pos, field->size);
for (i = 0; i < field->size; i++)
dest[i] |= ((const unsigned char*)field->data)[i];
}
void test_buffer(void)
{
#define BUF_TEST_SIZE (1024*2)
#define BUF_TEST_COUNT 1000
buffer_t *buf;
unsigned char *p, testdata[BUF_TEST_SIZE], shadowbuf[BUF_TEST_SIZE];
unsigned int i, shadowbuf_size;
size_t pos, pos2, size;
int test = -1;
bool zero;
buf = buffer_create_dynamic(default_pool, 1);
for (i = 0; i < BUF_TEST_SIZE; i++)
testdata[i] = random();
memset(shadowbuf, 0, sizeof(shadowbuf));
srand(1);
shadowbuf_size = 0;
for (i = 0; i < BUF_TEST_COUNT; i++) {
if (buf->used == BUF_TEST_SIZE) {
size = shadowbuf_size = rand() % (buf->used - 1);
buffer_set_used_size(buf, size);
memset(shadowbuf + shadowbuf_size, 0,
BUF_TEST_SIZE - shadowbuf_size);
i_assert(buf->used < BUF_TEST_SIZE);
}
test = rand() % 6;
zero = rand() % 10 == 0;
switch (test) {
case 0:
pos = rand() % (BUF_TEST_SIZE-1);
size = rand() % (BUF_TEST_SIZE - pos);
if (!zero) {
buffer_write(buf, pos, testdata, size);
memcpy(shadowbuf + pos, testdata, size);
} else {
buffer_write_zero(buf, pos, size);
memset(shadowbuf + pos, 0, size);
}
if (pos + size > shadowbuf_size)
shadowbuf_size = pos + size;
break;
case 1:
size = rand() % (BUF_TEST_SIZE - buf->used);
if (!zero) {
buffer_append(buf, testdata, size);
memcpy(shadowbuf + shadowbuf_size,
testdata, size);
} else {
buffer_append_zero(buf, size);
memset(shadowbuf + shadowbuf_size, 0, size);
}
shadowbuf_size += size;
break;
case 2:
pos = rand() % (BUF_TEST_SIZE-1);
size = rand() % (BUF_TEST_SIZE - I_MAX(buf->used, pos));
if (!zero) {
buffer_insert(buf, pos, testdata, size);
memmove(shadowbuf + pos + size,
shadowbuf + pos,
BUF_TEST_SIZE - (pos + size));
memcpy(shadowbuf + pos, testdata, size);
} else {
buffer_insert_zero(buf, pos, size);
memmove(shadowbuf + pos + size,
shadowbuf + pos,
BUF_TEST_SIZE - (pos + size));
memset(shadowbuf + pos, 0, size);
}
if (pos < shadowbuf_size)
shadowbuf_size += size;
else
shadowbuf_size = pos + size;
break;
case 3:
pos = rand() % (BUF_TEST_SIZE-1);
size = rand() % (BUF_TEST_SIZE - pos);
buffer_delete(buf, pos, size);
if (pos < shadowbuf_size) {
if (pos + size > shadowbuf_size)
size = shadowbuf_size - pos;
memmove(shadowbuf + pos,
shadowbuf + pos + size,
BUF_TEST_SIZE - (pos + size));
shadowbuf_size -= size;
memset(shadowbuf + shadowbuf_size, 0,
BUF_TEST_SIZE - shadowbuf_size);
}
break;
case 4:
if (shadowbuf_size == 0)
break;
pos = rand() % (shadowbuf_size-1); /* dest */
pos2 = rand() % (shadowbuf_size-1); /* source */
size = rand() % (shadowbuf_size - I_MAX(pos, pos2));
buffer_copy(buf, pos, buf, pos2, size);
memmove(shadowbuf + pos,
shadowbuf + pos2, size);
if (pos > pos2 && pos + size > shadowbuf_size)
shadowbuf_size = pos + size;
break;
case 5:
pos = rand() % (BUF_TEST_SIZE-1);
size = rand() % (BUF_TEST_SIZE - pos);
p = buffer_get_space_unsafe(buf, pos, size);
memcpy(p, testdata, size);
memcpy(shadowbuf + pos, testdata, size);
if (pos + size > shadowbuf_size)
shadowbuf_size = pos + size;
break;
}
i_assert(shadowbuf_size <= BUF_TEST_SIZE);
if (buf->used != shadowbuf_size ||
memcmp(buf->data, shadowbuf, buf->used) != 0)
break;
}
if (i == BUF_TEST_COUNT)
test_out("buffer", TRUE);
else {
test_out_reason("buffer", FALSE,
t_strdup_printf("round %u test %d failed", i, test));
}
buffer_free(&buf);
}
void *buffer_append_space_unsafe(buffer_t *buf, size_t size)
{
return buffer_get_space_unsafe(buf, buf->used, size);
}
