void
TestMemPool_run_tests() {
TestBatch *batch = TestBatch_new(4);
MemoryPool *mem_pool = MemPool_new(0);
MemoryPool *other = MemPool_new(0);
char *ptr_a, *ptr_b;
TestBatch_Plan(batch);
ptr_a = (char*)MemPool_Grab(mem_pool, 10);
strcpy(ptr_a, "foo");
MemPool_Release_All(mem_pool);
ptr_b = (char*)MemPool_Grab(mem_pool, 10);
TEST_STR_EQ(batch, ptr_b, "foo", "Recycle RAM on Release_All");
ptr_a = mem_pool->buf;
MemPool_Resize(mem_pool, ptr_b, 6);
TEST_TRUE(batch, mem_pool->buf < ptr_a, "Resize");
ptr_a = (char*)MemPool_Grab(other, 20);
MemPool_Release_All(other);
MemPool_Eat(other, mem_pool);
TEST_TRUE(batch, other->buf == mem_pool->buf, "Eat");
TEST_TRUE(batch, other->buf != NULL, "Eat");
DECREF(mem_pool);
DECREF(other);
DECREF(batch);
}
void
MatchPost_add_inversion_to_pool(MatchPosting *self, PostingPool *post_pool,
Inversion *inversion, FieldType *type,
i32_t doc_id, float doc_boost,
float length_norm)
{
MemoryPool *mem_pool = post_pool->mem_pool;
Token **tokens;
u32_t freq;
UNUSED_VAR(self);
UNUSED_VAR(type);
UNUSED_VAR(doc_boost);
UNUSED_VAR(length_norm);
Inversion_Reset(inversion);
while ( (tokens = Inversion_Next_Cluster(inversion, &freq)) != NULL ) {
Token *token = *tokens;
u32_t raw_post_bytes = MAX_RAW_POSTING_LEN(token->len);
RawPosting *raw_posting = RawPost_new(
MemPool_Grab(mem_pool, raw_post_bytes), doc_id, freq,
token->text, token->len
);
PostPool_Add_Elem(post_pool, (Obj*)raw_posting);
}
}
void
MatchPost_Add_Inversion_To_Pool_IMP(MatchPosting *self,
PostingPool *post_pool,
Inversion *inversion, FieldType *type,
int32_t doc_id, float doc_boost,
float length_norm) {
MemoryPool *mem_pool = PostPool_Get_Mem_Pool(post_pool);
const size_t base_size = VTable_Get_Obj_Alloc_Size(RAWPOSTING);
Token **tokens;
uint32_t freq;
UNUSED_VAR(self);
UNUSED_VAR(type);
UNUSED_VAR(doc_boost);
UNUSED_VAR(length_norm);
Inversion_Reset(inversion);
while ((tokens = Inversion_Next_Cluster(inversion, &freq)) != NULL) {
TokenIVARS *const token_ivars = Token_IVARS(*tokens);
uint32_t raw_post_bytes
= MAX_RAW_POSTING_LEN(base_size, token_ivars->len);
RawPosting *raw_posting
= RawPost_new(MemPool_Grab(mem_pool, raw_post_bytes), doc_id,
freq, token_ivars->text, token_ivars->len);
PostPool_Feed(post_pool, &raw_posting);
}
}
RawPosting*
RichPost_read_raw(RichPosting *self, InStream *instream, int32_t last_doc_id,
CharBuf *term_text, MemoryPool *mem_pool)
{
char *const text_buf = (char*)CB_Get_Ptr8(term_text);
const size_t text_size = CB_Get_Size(term_text);
const uint32_t doc_code = InStream_Read_C32(instream);
const uint32_t delta_doc = doc_code >> 1;
const int32_t doc_id = last_doc_id + delta_doc;
const uint32_t freq = (doc_code & 1)
? 1
: InStream_Read_C32(instream);
size_t raw_post_bytes = MAX_RAW_POSTING_LEN(text_size, freq);
void *const allocation = MemPool_Grab(mem_pool, raw_post_bytes);
RawPosting *const raw_posting = RawPost_new(allocation, doc_id, freq,
text_buf, text_size);
uint32_t num_prox = freq;
char *const start = raw_posting->blob + text_size;
char * dest = start;
UNUSED_VAR(self);
// Read positions and per-position boosts.
while (num_prox--) {
dest += InStream_Read_Raw_C64(instream, dest);
*((uint8_t*)dest) = InStream_Read_U8(instream);
dest++;
}
// Resize raw posting memory allocation.
raw_posting->aux_len = dest - start;
raw_post_bytes = dest - (char*)raw_posting;
MemPool_Resize(mem_pool, raw_posting, raw_post_bytes);
return raw_posting;
}
void
ScorePost_Add_Inversion_To_Pool_IMP(ScorePosting *self,
PostingPool *post_pool,
Inversion *inversion, FieldType *type,
int32_t doc_id, float doc_boost,
float length_norm) {
ScorePostingIVARS *const ivars = ScorePost_IVARS(self);
MemoryPool *mem_pool = PostPool_Get_Mem_Pool(post_pool);
Similarity *sim = ivars->sim;
float field_boost = doc_boost * FType_Get_Boost(type) * length_norm;
const uint8_t field_boost_byte = Sim_Encode_Norm(sim, field_boost);
const size_t base_size = Class_Get_Obj_Alloc_Size(RAWPOSTING);
Token **tokens;
uint32_t freq;
Inversion_Reset(inversion);
while ((tokens = Inversion_Next_Cluster(inversion, &freq)) != NULL) {
TokenIVARS *const token_ivars = Token_IVARS(*tokens);
uint32_t raw_post_bytes
= MAX_RAW_POSTING_LEN(base_size, token_ivars->len, freq);
RawPosting *raw_posting
= RawPost_new(MemPool_Grab(mem_pool, raw_post_bytes), doc_id,
freq, token_ivars->text, token_ivars->len);
RawPostingIVARS *const raw_post_ivars = RawPost_IVARS(raw_posting);
char *const start = raw_post_ivars->blob + token_ivars->len;
char *dest = start;
uint32_t last_prox = 0;
// Field_boost.
*((uint8_t*)dest) = field_boost_byte;
dest++;
// Positions.
for (uint32_t i = 0; i < freq; i++) {
TokenIVARS *const t_ivars = Token_IVARS(tokens[i]);
const uint32_t prox_delta = t_ivars->pos - last_prox;
NumUtil_encode_c32(prox_delta, &dest);
last_prox = t_ivars->pos;
}
// Resize raw posting memory allocation.
raw_post_ivars->aux_len = dest - start;
raw_post_bytes = dest - (char*)raw_posting;
MemPool_Resize(mem_pool, raw_posting, raw_post_bytes);
PostPool_Feed(post_pool, (Obj*)raw_posting);
}
}
RawPosting*
MatchPost_read_raw(MatchPosting *self, InStream *instream, i32_t last_doc_id,
CharBuf *term_text, MemoryPool *mem_pool)
{
const size_t text_size = CB_Get_Size(term_text);
const u32_t doc_code = InStream_Read_C32(instream);
const u32_t delta_doc = doc_code >> 1;
const i32_t doc_id = last_doc_id + delta_doc;
const u32_t freq = (doc_code & 1)
? 1
: InStream_Read_C32(instream);
size_t raw_post_bytes = MAX_RAW_POSTING_LEN(text_size);
void *const allocation = MemPool_Grab(mem_pool, raw_post_bytes);
UNUSED_VAR(self);
return RawPost_new(allocation, doc_id, freq, term_text->ptr, text_size);
}
void
RichPost_add_inversion_to_pool(RichPosting *self, PostingPool *post_pool,
Inversion *inversion, FieldType *type,
int32_t doc_id, float doc_boost,
float length_norm)
{
MemoryPool *mem_pool = PostPool_Get_Mem_Pool(post_pool);
Similarity *sim = self->sim;
float field_boost = doc_boost * FType_Get_Boost(type) * length_norm;
Token **tokens;
uint32_t freq;
Inversion_Reset(inversion);
while ( (tokens = Inversion_Next_Cluster(inversion, &freq)) != NULL ) {
Token *token = *tokens;
uint32_t raw_post_bytes = MAX_RAW_POSTING_LEN(token->len, freq);
RawPosting *raw_posting = RawPost_new(
MemPool_Grab(mem_pool, raw_post_bytes), doc_id, freq,
token->text, token->len
);
char *const start = raw_posting->blob + token->len;
char *dest = start;
uint32_t last_prox = 0;
uint32_t i;
// Positions and boosts.
for (i = 0; i < freq; i++) {
Token *const t = tokens[i];
const uint32_t prox_delta = t->pos - last_prox;
const float boost = field_boost * t->boost;
NumUtil_encode_c32(prox_delta, &dest);
last_prox = t->pos;
*((uint8_t*)dest) = Sim_Encode_Norm(sim, boost);
dest++;
}
// Resize raw posting memory allocation.
raw_posting->aux_len = dest - start;
raw_post_bytes = dest - (char*)raw_posting;
MemPool_Resize(mem_pool, raw_posting, raw_post_bytes);
PostPool_Feed(post_pool, &raw_posting);
}
}
RawPosting*
MatchPost_Read_Raw_IMP(MatchPosting *self, InStream *instream,
int32_t last_doc_id, String *term_text,
MemoryPool *mem_pool) {
const char *const text_buf = Str_Get_Ptr8(term_text);
const size_t text_size = Str_Get_Size(term_text);
const uint32_t doc_code = InStream_Read_C32(instream);
const uint32_t delta_doc = doc_code >> 1;
const int32_t doc_id = last_doc_id + delta_doc;
const uint32_t freq = (doc_code & 1)
? 1
: InStream_Read_C32(instream);
const size_t base_size = VTable_Get_Obj_Alloc_Size(RAWPOSTING);
size_t raw_post_bytes = MAX_RAW_POSTING_LEN(base_size, text_size);
void *const allocation = MemPool_Grab(mem_pool, raw_post_bytes);
UNUSED_VAR(self);
return RawPost_new(allocation, doc_id, freq, text_buf, text_size);
}
RawPosting*
ScorePost_Read_Raw_IMP(ScorePosting *self, InStream *instream,
int32_t last_doc_id, String *term_text,
MemoryPool *mem_pool) {
const char *const text_buf = Str_Get_Ptr8(term_text);
const size_t text_size = Str_Get_Size(term_text);
const uint32_t doc_code = InStream_Read_C32(instream);
const uint32_t delta_doc = doc_code >> 1;
const int32_t doc_id = last_doc_id + delta_doc;
const uint32_t freq = (doc_code & 1)
? 1
: InStream_Read_C32(instream);
const size_t base_size = Class_Get_Obj_Alloc_Size(RAWPOSTING);
size_t raw_post_bytes = MAX_RAW_POSTING_LEN(base_size, text_size, freq);
void *const allocation = MemPool_Grab(mem_pool, raw_post_bytes);
RawPosting *const raw_posting
= RawPost_new(allocation, doc_id, freq, text_buf, text_size);
RawPostingIVARS *const raw_post_ivars = RawPost_IVARS(raw_posting);
uint32_t num_prox = freq;
char *const start = raw_post_ivars->blob + text_size;
char *dest = start;
UNUSED_VAR(self);
// Field_boost.
*((uint8_t*)dest) = InStream_Read_U8(instream);
dest++;
// Read positions.
while (num_prox--) {
dest += InStream_Read_Raw_C64(instream, dest);
}
// Resize raw posting memory allocation.
raw_post_ivars->aux_len = dest - start;
raw_post_bytes = dest - (char*)raw_posting;
MemPool_Resize(mem_pool, raw_posting, raw_post_bytes);
return raw_posting;
}