static void S_init_arena(MemoryPool *self, size_t amount) { ByteBuf *bb; // Indicate which arena we're using at present. self->tick++; if (self->tick < (int32_t)VA_Get_Size(self->arenas)) { // In recycle mode, use previously acquired memory. bb = (ByteBuf*)VA_Fetch(self->arenas, self->tick); if (amount >= BB_Get_Size(bb)) { BB_Grow(bb, amount); BB_Set_Size(bb, amount); } } else { // In add mode, get more mem from system. size_t buf_size = (amount + 1) > self->arena_size ? (amount + 1) : self->arena_size; char *ptr = (char*)MALLOCATE(buf_size); bb = BB_new_steal_bytes(ptr, buf_size - 1, buf_size); VA_Push(self->arenas, (Obj*)bb); } // Recalculate consumption to take into account blocked off space. self->consumed = 0; for (int32_t i = 0; i < self->tick; i++) { ByteBuf *bb = (ByteBuf*)VA_Fetch(self->arenas, i); self->consumed += BB_Get_Size(bb); } self->buf = BB_Get_Buf(bb); self->limit = self->buf + BB_Get_Size(bb); }
static void test_Grow(TestBatchRunner *runner) { ByteBuf *bb = BB_new(1); TEST_INT_EQ(runner, BB_Get_Capacity(bb), 8, "Allocate in 8-byte increments"); BB_Grow(bb, 9); TEST_INT_EQ(runner, BB_Get_Capacity(bb), 16, "Grow in 8-byte increments"); DECREF(bb); }
void DefDocReader_Read_Record_IMP(DefaultDocReader *self, ByteBuf *buffer, int32_t doc_id) { DefaultDocReaderIVARS *const ivars = DefDocReader_IVARS(self); // Find start and length of variable length record. InStream_Seek(ivars->ix_in, (int64_t)doc_id * 8); int64_t start = InStream_Read_I64(ivars->ix_in); int64_t end = InStream_Read_I64(ivars->ix_in); size_t size = (size_t)(end - start); // Read in the record. char *buf = BB_Grow(buffer, size); InStream_Seek(ivars->dat_in, start); InStream_Read_Bytes(ivars->dat_in, buf, size); BB_Set_Size(buffer, size); }
void DefHLReader_read_record(DefaultHighlightReader *self, int32_t doc_id, ByteBuf *target) { InStream *dat_in = self->dat_in; InStream *ix_in = self->ix_in; InStream_Seek(ix_in, doc_id * 8); // Copy the whole record. int64_t filepos = InStream_Read_I64(ix_in); int64_t end = InStream_Read_I64(ix_in); size_t size = (size_t)(end - filepos); char *buf = BB_Grow(target, size); InStream_Seek(dat_in, filepos); InStream_Read_Bytes(dat_in, buf, size); BB_Set_Size(target, size); }
void DefDocReader_read_record(DefaultDocReader *self, ByteBuf *buffer, i32_t doc_id) { i64_t start; i64_t end; i32_t size; /* Find start and length of variable length record. */ InStream_Seek(self->ix_in, (i64_t)doc_id * 8); start = InStream_Read_U64(self->ix_in); end = InStream_Read_U64(self->ix_in); size = end - start; /* Read in the record. */ BB_Grow(buffer, size); InStream_Seek(self->dat_in, start); InStream_Read_Bytes(self->dat_in, buffer->ptr, size); BB_Set_Size(buffer, size); }
static void S_init_arena(MemoryPool *self, size_t amount) { ByteBuf *bb; i32_t i; /* Indicate which arena we're using at present. */ self->tick++; if (self->tick < (i32_t)VA_Get_Size(self->arenas)) { /* In recycle mode, use previously acquired memory. */ bb = (ByteBuf*)VA_Fetch(self->arenas, self->tick); if (amount >= BB_Get_Size(bb)) { BB_Grow(bb, amount); BB_Set_Size(bb, amount); } } else { /* In add mode, get more mem from system. */ size_t buf_size = (amount + 1) > self->arena_size ? (amount + 1) : self->arena_size; char *ptr = MALLOCATE(buf_size, char); if (ptr == NULL) THROW("Failed to allocate memory"); bb = BB_new_steal_str(ptr, buf_size - 1, buf_size); VA_Push(self->arenas, (Obj*)bb); } /* Recalculate consumption to take into account blocked off space. */ self->consumed = 0; for (i = 0; i < self->tick; i++) { ByteBuf *bb = (ByteBuf*)VA_Fetch(self->arenas, i); self->consumed += BB_Get_Size(bb); } self->buf = bb->ptr; self->limit = BBEND(bb); }
ByteBuf* HLWriter_TV_Buf_IMP(HighlightWriter *self, Inversion *inversion) { const char *last_text = ""; size_t last_len = 0; ByteBuf *tv_buf = BB_new(20 + Inversion_Get_Size(inversion) * 8); uint32_t num_postings = 0; Token **tokens; uint32_t freq; UNUSED_VAR(self); // Leave space for a c32 indicating the number of postings. BB_Set_Size(tv_buf, C32_MAX_BYTES); Inversion_Reset(inversion); while ((tokens = Inversion_Next_Cluster(inversion, &freq)) != NULL) { Token *token = *tokens; char *const token_text = Token_Get_Text(token); const int32_t token_len = Token_Get_Len(token); int32_t overlap = StrHelp_overlap(last_text, token_text, last_len, token_len); char *ptr; char *orig; size_t old_size = BB_Get_Size(tv_buf); size_t new_size = old_size + C32_MAX_BYTES // overlap + C32_MAX_BYTES // length of string diff + (token_len - overlap) // diff char data + C32_MAX_BYTES // num prox + (C32_MAX_BYTES * freq * 3); // pos data // Allocate for worst-case scenario. ptr = BB_Grow(tv_buf, new_size); orig = ptr; ptr += old_size; // Track number of postings. num_postings += 1; // Append the string diff to the tv_buf. NumUtil_encode_c32(overlap, &ptr); NumUtil_encode_c32((token_len - overlap), &ptr); memcpy(ptr, (token_text + overlap), (token_len - overlap)); ptr += token_len - overlap; // Save text and text_len for comparison next loop. last_text = token_text; last_len = token_len; // Append the number of positions for this term. NumUtil_encode_c32(freq, &ptr); do { // Add position, start_offset, and end_offset to tv_buf. NumUtil_encode_c32(Token_Get_Pos(token), &ptr); NumUtil_encode_c32(Token_Get_Start_Offset(token), &ptr); NumUtil_encode_c32(Token_Get_End_Offset(token), &ptr); } while (--freq && (token = *++tokens)); // Set new byte length. BB_Set_Size(tv_buf, ptr - orig); } // Go back and start the term vector string with the posting count. char *dest = BB_Get_Buf(tv_buf); NumUtil_encode_padded_c32(num_postings, &dest); return tv_buf; }
ByteBuf* HLWriter_tv_buf(HighlightWriter *self, Inversion *inversion) { char *last_text = ""; size_t last_len = 0; ByteBuf *tv_buf = BB_new(20 + inversion->size * 8); /* generous */ u32_t num_postings = 0; char *dest; Token **tokens; u32_t freq; UNUSED_VAR(self); /* heh. */ /* Leave space for a c32 indicating the number of postings. */ BB_Set_Size(tv_buf, C32_MAX_BYTES); Inversion_Reset(inversion); while ( (tokens = Inversion_Next_Cluster(inversion, &freq)) != NULL ) { Token *token = *tokens; i32_t overlap = StrHelp_string_diff(last_text, token->text, last_len, token->len); char *ptr; size_t new_size = BB_Get_Size(tv_buf) + C32_MAX_BYTES /* overlap */ + C32_MAX_BYTES /* length of string diff */ + (token->len - overlap) /* diff char data */ + C32_MAX_BYTES /* num prox */ + (C32_MAX_BYTES * freq * 3); /* pos data */ /* Allocate for worst-case scenario. */ BB_Grow(tv_buf, new_size); ptr = BBEND(tv_buf); /* Track number of postings. */ num_postings += 1; /* Append the string diff to the tv_buf. */ Math_encode_c32(overlap, &ptr); Math_encode_c32( (token->len - overlap), &ptr); memcpy(ptr, (token->text + overlap), (token->len - overlap)); ptr += token->len - overlap; /* Save text and text_len for comparison next loop. */ last_text = token->text; last_len = token->len; /* Append the number of positions for this term. */ Math_encode_c32(freq, &ptr); do { /* Add position, start_offset, and end_offset to tv_buf. */ Math_encode_c32(token->pos, &ptr); Math_encode_c32(token->start_offset, &ptr); Math_encode_c32(token->end_offset, &ptr); } while (--freq && (token = *++tokens)); /* Set new byte length. */ BB_Set_Size(tv_buf, ptr - tv_buf->ptr); } /* Go back and start the term vector string with the number of postings. */ dest = tv_buf->ptr; Math_encode_padded_c32(num_postings, &dest); return tv_buf; }