void fts_icu_utf8_to_utf16(buffer_t *dest_utf16, const char *src_utf8) { UErrorCode err = U_ZERO_ERROR; unsigned int src_bytes = strlen(src_utf8); int32_t utf16_len; UChar *dest_data, *retp = NULL; int32_t avail_uchars = 0; /* try to encode with the current buffer size */ avail_uchars = buffer_get_writable_size(dest_utf16) / sizeof(UChar); dest_data = buffer_get_space_unsafe(dest_utf16, 0, buffer_get_writable_size(dest_utf16)); retp = u_strFromUTF8Lenient(dest_data, avail_uchars, &utf16_len, src_utf8, src_bytes, &err); if (err == U_BUFFER_OVERFLOW_ERROR) { /* try again with a larger buffer */ dest_data = buffer_get_space_unsafe(dest_utf16, 0, utf16_len * sizeof(UChar)); err = U_ZERO_ERROR; retp = u_strFromUTF8Lenient(dest_data, utf16_len, &utf16_len, src_utf8, src_bytes, &err); } if (U_FAILURE(err)) { i_panic("LibICU u_strFromUTF8Lenient() failed: %s", u_errorName(err)); } buffer_set_used_size(dest_utf16, utf16_len * sizeof(UChar)); i_assert(retp == dest_data); }
void fts_icu_utf16_to_utf8(string_t *dest_utf8, const UChar *src_utf16, unsigned int src_len) { int32_t dest_len = 0; int32_t sub_num = 0; char *dest_data, *retp = NULL; UErrorCode err = U_ZERO_ERROR; /* try to encode with the current buffer size */ dest_data = buffer_get_space_unsafe(dest_utf8, 0, buffer_get_writable_size(dest_utf8)); retp = u_strToUTF8WithSub(dest_data, buffer_get_writable_size(dest_utf8), &dest_len, src_utf16, src_len, UNICODE_REPLACEMENT_CHAR, &sub_num, &err); if (err == U_BUFFER_OVERFLOW_ERROR) { /* try again with a larger buffer */ dest_data = buffer_get_space_unsafe(dest_utf8, 0, dest_len); err = U_ZERO_ERROR; retp = u_strToUTF8WithSub(dest_data, buffer_get_writable_size(dest_utf8), &dest_len, src_utf16, src_len, UNICODE_REPLACEMENT_CHAR, &sub_num, &err); } if (U_FAILURE(err)) { i_panic("LibICU u_strToUTF8WithSub() failed: %s", u_errorName(err)); } buffer_set_used_size(dest_utf8, dest_len); i_assert(retp == dest_data); }
void fts_icu_lcase(string_t *dest_utf8, const char *src_utf8) { struct UCaseMap *csm = fts_icu_csm(); size_t avail_bytes, dest_pos = dest_utf8->used; char *dest_data; int dest_full_len; UErrorCode err = U_ZERO_ERROR; avail_bytes = buffer_get_writable_size(dest_utf8) - dest_pos; dest_data = buffer_get_space_unsafe(dest_utf8, dest_pos, avail_bytes); dest_full_len = ucasemap_utf8ToLower(csm, dest_data, avail_bytes, src_utf8, -1, &err); if (err == U_BUFFER_OVERFLOW_ERROR) { err = U_ZERO_ERROR; dest_data = buffer_get_space_unsafe(dest_utf8, dest_pos, dest_full_len); dest_full_len = ucasemap_utf8ToLower(csm, dest_data, dest_full_len, src_utf8, -1, &err); i_assert(err != U_BUFFER_OVERFLOW_ERROR); } if (U_FAILURE(err)) { i_fatal("LibICU ucasemap_utf8ToLower() failed: %s", u_errorName(err)); } buffer_set_used_size(dest_utf8, dest_full_len); }
static void test_buffer_set_used_size(void) { buffer_t *buf; test_begin("buffer_set_used_size"); buf = t_buffer_create(8); memset(buffer_append_space_unsafe(buf, 7), 'a', 7); buffer_set_used_size(buf, 4); test_assert(memcmp(buffer_get_space_unsafe(buf, 0, 7), "aaaa\0\0\0", 7) == 0); memset(buffer_get_space_unsafe(buf, 4, 7), 'b', 7); buffer_set_used_size(buf, 10); test_assert(memcmp(buffer_append_space_unsafe(buf, 1), "\0", 1) == 0); buffer_set_used_size(buf, 11); test_assert(memcmp(buffer_get_space_unsafe(buf, 0, 11), "aaaabbbbbb\0", 11) == 0); test_end(); }
static int mail_transaction_log_file_insert_read(struct mail_transaction_log_file *file, uoff_t offset) { void *data; size_t size; ssize_t ret; size = file->buffer_offset - offset; buffer_copy(file->buffer, size, file->buffer, 0, (size_t)-1); data = buffer_get_space_unsafe(file->buffer, 0, size); ret = pread_full(file->fd, data, size, offset); if (ret > 0) { /* success */ file->buffer_offset -= size; return 1; } /* failure. don't leave ourself to inconsistent state */ buffer_copy(file->buffer, 0, file->buffer, size, (size_t)-1); buffer_set_used_size(file->buffer, file->buffer->used - size); if (ret == 0) { mail_transaction_log_file_set_corrupted(file, "file shrank"); return 0; } else if (errno == ESTALE) { /* log file was deleted in NFS server, fail silently */ return 0; } else { log_file_set_syscall_error(file, "pread()"); return -1; } }
static int mail_cache_lookup_bitmask(struct mail_cache_lookup_iterate_ctx *iter, unsigned int field_idx, unsigned int field_size, buffer_t *dest_buf) { struct mail_cache_iterate_field field; const unsigned char *src; unsigned char *dest; unsigned int i; bool found = FALSE; int ret; /* make sure all bits are cleared first */ buffer_write_zero(dest_buf, 0, field_size); while ((ret = mail_cache_lookup_iter_next(iter, &field)) > 0) { if (field.field_idx != field_idx) continue; /* merge all bits */ src = field.data; dest = buffer_get_space_unsafe(dest_buf, 0, field.size); for (i = 0; i < field.size; i++) dest[i] |= src[i]; found = TRUE; } return ret < 0 ? -1 : (found ? 1 : 0); }
void mbox_sync_headers_add_space(struct mbox_sync_mail_context *ctx, size_t size) { size_t data_size, pos, start_pos; const unsigned char *data; void *p; i_assert(size < SSIZE_T_MAX); if (ctx->mail.pseudo) start_pos = ctx->hdr_pos[MBOX_HDR_X_IMAPBASE]; else if (ctx->mail.space > 0) { /* update the header using the existing offset. otherwise we might chose wrong header and just decrease the available space */ start_pos = ctx->mail.offset - ctx->hdr_offset; } else { /* Append at the end of X-Keywords header, or X-UID if it doesn't exist */ start_pos = ctx->hdr_pos[MBOX_HDR_X_KEYWORDS] != (size_t)-1 ? ctx->hdr_pos[MBOX_HDR_X_KEYWORDS] : ctx->hdr_pos[MBOX_HDR_X_UID]; } data = str_data(ctx->header); data_size = str_len(ctx->header); i_assert(start_pos < data_size); for (pos = start_pos; pos < data_size; pos++) { if (data[pos] == '\n') { /* possibly continues in next line */ if (pos+1 == data_size || !IS_LWSP(data[pos+1])) break; start_pos = pos+1; } else if (!IS_LWSP(data[pos]) && data[pos] != '\r') { start_pos = pos+1; } } /* pos points to end of header now, and start_pos to beginning of whitespace. */ mbox_sync_move_buffer(ctx, pos, size, 0); p = buffer_get_space_unsafe(ctx->header, pos, size); memset(p, ' ', size); if (ctx->header_first_change > pos) ctx->header_first_change = pos; ctx->header_last_change = (size_t)-1; ctx->mail.space = (pos - start_pos) + size; ctx->mail.offset = ctx->hdr_offset; if (ctx->mail.space > 0) ctx->mail.offset += start_pos; }
int fts_icu_translate(buffer_t *dest_utf16, const UChar *src_utf16, unsigned int src_len, UTransliterator *transliterator, const char **error_r) { UErrorCode err = U_ZERO_ERROR; int32_t utf16_len = src_len; UChar *dest_data; int32_t avail_uchars, limit = src_len; size_t dest_pos = dest_utf16->used; /* translation is done in-place in the buffer. try first with the current buffer size. */ buffer_append(dest_utf16, src_utf16, src_len*sizeof(UChar)); avail_uchars = (buffer_get_writable_size(dest_utf16)-dest_pos) / sizeof(UChar); dest_data = buffer_get_space_unsafe(dest_utf16, dest_pos, buffer_get_writable_size(dest_utf16)-dest_pos); utrans_transUChars(transliterator, dest_data, &utf16_len, avail_uchars, 0, &limit, &err); if (err == U_BUFFER_OVERFLOW_ERROR) { /* try again with a larger buffer */ err = U_ZERO_ERROR; avail_uchars = utf16_len; limit = utf16_len = src_len; buffer_write(dest_utf16, dest_pos, src_utf16, src_len*sizeof(UChar)); dest_data = buffer_get_space_unsafe(dest_utf16, dest_pos, avail_uchars * sizeof(UChar)); utrans_transUChars(transliterator, dest_data, &utf16_len, avail_uchars, 0, &limit, &err); i_assert(err != U_BUFFER_OVERFLOW_ERROR); } if (U_FAILURE(err)) { *error_r = t_strdup_printf("LibICU utrans_transUChars() failed: %s", u_errorName(err)); buffer_set_used_size(dest_utf16, dest_pos); return -1; } buffer_set_used_size(dest_utf16, utf16_len * sizeof(UChar)); return 0; }
void *array_idx_modifiable_i(struct array *array, unsigned int idx) { size_t pos; pos = idx * array->element_size; if (pos >= array->buffer->used) { /* index doesn't exist yet, initialize with zero */ buffer_append_zero(array->buffer, pos + array->element_size - array->buffer->used); } return buffer_get_space_unsafe(array->buffer, pos, array->element_size); }
static buffer_t *_file_lazy_load_buffer (struct sieve_binary_file *file, off_t *offset, size_t size) { buffer_t *buffer = buffer_create_dynamic(file->pool, size); if ( _file_lazy_read (file, offset, buffer_get_space_unsafe(buffer, 0, size), size) ) { return buffer; } return NULL; }
static void log_write_ext_hdr_init_data(struct mail_index *index, buffer_t *buf) { const struct mail_index_registered_ext *rext; struct mail_transaction_header *hdr; struct mail_transaction_ext_intro *intro; struct mail_transaction_ext_hdr_update *ext_hdr; unsigned int hdr_offset; rext = array_idx(&index->extensions, index->ext_hdr_init_id); /* introduce the extension */ hdr_offset = buf->used; hdr = buffer_append_space_unsafe(buf, sizeof(*hdr)); hdr->type = MAIL_TRANSACTION_EXT_INTRO; intro = buffer_append_space_unsafe(buf, sizeof(*intro)); intro->ext_id = (uint32_t)-1; intro->hdr_size = rext->hdr_size; intro->record_size = rext->record_size; intro->record_align = rext->record_align; intro->name_size = strlen(rext->name); buffer_append(buf, rext->name, intro->name_size); if (buf->used % 4 != 0) buffer_append_zero(buf, 4 - buf->used % 4); hdr = buffer_get_space_unsafe(buf, hdr_offset, sizeof(*hdr)); hdr->size = mail_index_uint32_to_offset(buf->used - hdr_offset); /* add the extension header data */ hdr_offset = buf->used; hdr = buffer_append_space_unsafe(buf, sizeof(*hdr)); hdr->type = MAIL_TRANSACTION_EXT_HDR_UPDATE; ext_hdr = buffer_append_space_unsafe(buf, sizeof(*ext_hdr)); ext_hdr->size = rext->hdr_size; buffer_append(buf, index->ext_hdr_init_data, rext->hdr_size); hdr = buffer_get_space_unsafe(buf, hdr_offset, sizeof(*hdr)); hdr->size = mail_index_uint32_to_offset(buf->used - hdr_offset); }
void str_vprintfa(string_t *str, const char *fmt, va_list args) { #define SNPRINTF_INITIAL_EXTRA_SIZE 128 va_list args2; char *tmp; unsigned int init_size; size_t pos = str->used; int ret, ret2; VA_COPY(args2, args); /* the format string is modified only if %m exists in it. it happens only in error conditions, so don't try to t_push() here since it'll just slow down the normal code path. */ fmt = printf_format_fix_get_len(fmt, &init_size); init_size += SNPRINTF_INITIAL_EXTRA_SIZE; /* @UNSAFE */ if (pos+init_size > buffer_get_size(str) && pos < buffer_get_size(str)) { /* avoid growing buffer larger if possible. this is also required if buffer isn't dynamically growing. */ init_size = buffer_get_size(str)-pos; } tmp = buffer_get_space_unsafe(str, pos, init_size); ret = vsnprintf(tmp, init_size, fmt, args); i_assert(ret >= 0); if ((unsigned int)ret >= init_size) { /* didn't fit with the first guess. now we know the size, so try again. */ tmp = buffer_get_space_unsafe(str, pos, ret + 1); ret2 = vsnprintf(tmp, ret + 1, fmt, args2); i_assert(ret2 == ret); } /* drop the unused data, including terminating NUL */ buffer_set_used_size(str, pos + ret); }
void *array_insert_space_i(struct array *array, unsigned int idx) { void *data; size_t pos; pos = idx * array->element_size; buffer_copy(array->buffer, pos + array->element_size, array->buffer, pos, (size_t)-1); data = buffer_get_space_unsafe(array->buffer, pos, array->element_size); memset(data, 0, array->element_size); return data; }
static void mail_cache_compress_field(struct mail_cache_copy_context *ctx, const struct mail_cache_iterate_field *field) { struct mail_cache_field *cache_field; enum mail_cache_decision_type dec; uint32_t file_field_idx, size32; uint8_t *field_seen; file_field_idx = ctx->field_file_map[field->field_idx]; if (file_field_idx == (uint32_t)-1) return; cache_field = &ctx->cache->fields[field->field_idx].field; field_seen = buffer_get_space_unsafe(ctx->field_seen, field->field_idx, 1); if (*field_seen == ctx->field_seen_value) { /* duplicate */ if (cache_field->type == MAIL_CACHE_FIELD_BITMASK) mail_cache_merge_bitmask(ctx, field); return; } *field_seen = ctx->field_seen_value; dec = cache_field->decision & ~MAIL_CACHE_DECISION_FORCED; if (ctx->new_msg) { if (dec == MAIL_CACHE_DECISION_NO) return; } else { if (dec != MAIL_CACHE_DECISION_YES) return; } buffer_append(ctx->buffer, &file_field_idx, sizeof(file_field_idx)); if (cache_field->field_size == UINT_MAX) { size32 = (uint32_t)field->size; buffer_append(ctx->buffer, &size32, sizeof(size32)); } if (cache_field->type == MAIL_CACHE_FIELD_BITMASK) { /* remember the position in case we need to update it */ unsigned int pos = ctx->buffer->used; array_idx_set(&ctx->bitmask_pos, field->field_idx, &pos); } buffer_append(ctx->buffer, field->data, field->size); if ((field->size & 3) != 0) buffer_append_zero(ctx->buffer, 4 - (field->size & 3)); }
static void status_flags_replace(struct mbox_sync_mail_context *ctx, size_t pos, const struct mbox_flag_type *flags_list) { unsigned char *data; size_t size; int i, need, have; ctx->mail.flags ^= MBOX_NONRECENT_KLUDGE; if (ctx->header_first_change > pos) ctx->header_first_change = pos; /* how many bytes do we need? */ for (i = 0, need = 0; flags_list[i].chr != 0; i++) { if ((ctx->mail.flags & flags_list[i].flag) != 0) need++; } /* how many bytes do we have now? */ data = buffer_get_modifiable_data(ctx->header, &size); for (have = 0; pos < size; pos++) { if (data[pos] == '\n' || data[pos] == '\r') break; /* see if this is unknown flag for us */ for (i = 0; flags_list[i].chr != 0; i++) { if (flags_list[i].chr == (char)data[pos]) break; } if (flags_list[i].chr != 0) have++; else { /* save this one */ data[pos-have] = data[pos]; } } pos -= have; mbox_sync_move_buffer(ctx, pos, need, have); /* @UNSAFE */ data = buffer_get_space_unsafe(ctx->header, pos, need); for (i = 0; flags_list[i].chr != 0; i++) { if ((ctx->mail.flags & flags_list[i].flag) != 0) *data++ = flags_list[i].chr; } ctx->mail.flags ^= MBOX_NONRECENT_KLUDGE; }
buffer_t *t_hmac_data(const struct hash_method *meth, const unsigned char *key, size_t key_len, const void *data, size_t data_len) { struct hmac_context ctx; i_assert(meth != NULL); i_assert(key != NULL && key_len > 0); i_assert(data != NULL || data_len == 0); buffer_t *res = buffer_create_dynamic(pool_datastack_create(), meth->digest_size); hmac_init(&ctx, key, key_len, meth); if (data_len > 0) hmac_update(&ctx, data, data_len); unsigned char *buf = buffer_get_space_unsafe(res, 0, meth->digest_size); hmac_final(&ctx, buf); return res; }
static void mail_cache_merge_bitmask(struct mail_cache_copy_context *ctx, const struct mail_cache_iterate_field *field) { unsigned char *dest; unsigned int i, *pos; pos = array_idx_modifiable(&ctx->bitmask_pos, field->field_idx); if (*pos == 0) { /* we decided to drop this field */ return; } dest = buffer_get_space_unsafe(ctx->buffer, *pos, field->size); for (i = 0; i < field->size; i++) dest[i] |= ((const unsigned char*)field->data)[i]; }
void test_buffer(void) { #define BUF_TEST_SIZE (1024*2) #define BUF_TEST_COUNT 1000 buffer_t *buf; unsigned char *p, testdata[BUF_TEST_SIZE], shadowbuf[BUF_TEST_SIZE]; unsigned int i, shadowbuf_size; size_t pos, pos2, size; int test = -1; bool zero; buf = buffer_create_dynamic(default_pool, 1); for (i = 0; i < BUF_TEST_SIZE; i++) testdata[i] = random(); memset(shadowbuf, 0, sizeof(shadowbuf)); srand(1); shadowbuf_size = 0; for (i = 0; i < BUF_TEST_COUNT; i++) { if (buf->used == BUF_TEST_SIZE) { size = shadowbuf_size = rand() % (buf->used - 1); buffer_set_used_size(buf, size); memset(shadowbuf + shadowbuf_size, 0, BUF_TEST_SIZE - shadowbuf_size); i_assert(buf->used < BUF_TEST_SIZE); } test = rand() % 6; zero = rand() % 10 == 0; switch (test) { case 0: pos = rand() % (BUF_TEST_SIZE-1); size = rand() % (BUF_TEST_SIZE - pos); if (!zero) { buffer_write(buf, pos, testdata, size); memcpy(shadowbuf + pos, testdata, size); } else { buffer_write_zero(buf, pos, size); memset(shadowbuf + pos, 0, size); } if (pos + size > shadowbuf_size) shadowbuf_size = pos + size; break; case 1: size = rand() % (BUF_TEST_SIZE - buf->used); if (!zero) { buffer_append(buf, testdata, size); memcpy(shadowbuf + shadowbuf_size, testdata, size); } else { buffer_append_zero(buf, size); memset(shadowbuf + shadowbuf_size, 0, size); } shadowbuf_size += size; break; case 2: pos = rand() % (BUF_TEST_SIZE-1); size = rand() % (BUF_TEST_SIZE - I_MAX(buf->used, pos)); if (!zero) { buffer_insert(buf, pos, testdata, size); memmove(shadowbuf + pos + size, shadowbuf + pos, BUF_TEST_SIZE - (pos + size)); memcpy(shadowbuf + pos, testdata, size); } else { buffer_insert_zero(buf, pos, size); memmove(shadowbuf + pos + size, shadowbuf + pos, BUF_TEST_SIZE - (pos + size)); memset(shadowbuf + pos, 0, size); } if (pos < shadowbuf_size) shadowbuf_size += size; else shadowbuf_size = pos + size; break; case 3: pos = rand() % (BUF_TEST_SIZE-1); size = rand() % (BUF_TEST_SIZE - pos); buffer_delete(buf, pos, size); if (pos < shadowbuf_size) { if (pos + size > shadowbuf_size) size = shadowbuf_size - pos; memmove(shadowbuf + pos, shadowbuf + pos + size, BUF_TEST_SIZE - (pos + size)); shadowbuf_size -= size; memset(shadowbuf + shadowbuf_size, 0, BUF_TEST_SIZE - shadowbuf_size); } break; case 4: if (shadowbuf_size == 0) break; pos = rand() % (shadowbuf_size-1); /* dest */ pos2 = rand() % (shadowbuf_size-1); /* source */ size = rand() % (shadowbuf_size - I_MAX(pos, pos2)); buffer_copy(buf, pos, buf, pos2, size); memmove(shadowbuf + pos, shadowbuf + pos2, size); if (pos > pos2 && pos + size > shadowbuf_size) shadowbuf_size = pos + size; break; case 5: pos = rand() % (BUF_TEST_SIZE-1); size = rand() % (BUF_TEST_SIZE - pos); p = buffer_get_space_unsafe(buf, pos, size); memcpy(p, testdata, size); memcpy(shadowbuf + pos, testdata, size); if (pos + size > shadowbuf_size) shadowbuf_size = pos + size; break; } i_assert(shadowbuf_size <= BUF_TEST_SIZE); if (buf->used != shadowbuf_size || memcmp(buf->data, shadowbuf, buf->used) != 0) break; } if (i == BUF_TEST_COUNT) test_out("buffer", TRUE); else { test_out_reason("buffer", FALSE, t_strdup_printf("round %u test %d failed", i, test)); } buffer_free(&buf); }
void *buffer_append_space_unsafe(buffer_t *buf, size_t size) { return buffer_get_space_unsafe(buf, buf->used, size); }