void
TestMemPool_run_tests() {
TestBatch *batch = TestBatch_new(4);
MemoryPool *mem_pool = MemPool_new(0);
MemoryPool *other = MemPool_new(0);
char *ptr_a, *ptr_b;
TestBatch_Plan(batch);
ptr_a = (char*)MemPool_Grab(mem_pool, 10);
strcpy(ptr_a, "foo");
MemPool_Release_All(mem_pool);
ptr_b = (char*)MemPool_Grab(mem_pool, 10);
TEST_STR_EQ(batch, ptr_b, "foo", "Recycle RAM on Release_All");
ptr_a = mem_pool->buf;
MemPool_Resize(mem_pool, ptr_b, 6);
TEST_TRUE(batch, mem_pool->buf < ptr_a, "Resize");
ptr_a = (char*)MemPool_Grab(other, 20);
MemPool_Release_All(other);
MemPool_Eat(other, mem_pool);
TEST_TRUE(batch, other->buf == mem_pool->buf, "Eat");
TEST_TRUE(batch, other->buf != NULL, "Eat");
DECREF(mem_pool);
DECREF(other);
DECREF(batch);
}
int main(int argc, char** argv) {
char* file = argv[1];
char* buf = 0;
long buflen = 0;
FILE* fd = fopen(file, "r");
TokenType token = 0;
MemPool* pool = MemPool_new(1 << 30);
Lexer* lexer = Lexer_new(pool);
fseek(fd, 0, SEEK_END);
buflen = ftell(fd);
fseek(fd, 0, SEEK_SET);
buf = malloc(buflen);
fread(buf, 1, buflen, fd);
Lexer_setpos(lexer, buf);
while (token != TT_EOF) {
token = Lexer_debugtoken(lexer);
printf("\n");
}
return 0;
}
SortWriter*
SortWriter_init(SortWriter *self, Schema *schema, Snapshot *snapshot,
Segment *segment, PolyReader *polyreader) {
uint32_t field_max = Schema_Num_Fields(schema) + 1;
DataWriter_init((DataWriter*)self, schema, snapshot, segment, polyreader);
SortWriterIVARS *const ivars = SortWriter_IVARS(self);
// Init.
ivars->field_writers = VA_new(field_max);
ivars->counts = Hash_new(0);
ivars->null_ords = Hash_new(0);
ivars->ord_widths = Hash_new(0);
ivars->temp_ord_out = NULL;
ivars->temp_ix_out = NULL;
ivars->temp_dat_out = NULL;
ivars->mem_pool = MemPool_new(0);
ivars->mem_thresh = default_mem_thresh;
ivars->flush_at_finish = false;
return self;
}
uint32_t
PostPool_Refill_IMP(PostingPool *self) {
PostingPoolIVARS *const ivars = PostPool_IVARS(self);
Lexicon *const lexicon = ivars->lexicon;
PostingList *const plist = ivars->plist;
I32Array *const doc_map = ivars->doc_map;
const uint32_t mem_thresh = ivars->mem_thresh;
const int32_t doc_base = ivars->doc_base;
uint32_t num_elems = 0; // number of items recovered
String *term_text = NULL;
if (ivars->lexicon == NULL) { return 0; }
else { term_text = (String*)Lex_Get_Term(lexicon); }
// Make sure buffer is empty.
if (ivars->buf_max - ivars->buf_tick > 0) {
THROW(ERR, "Refill called but buffer contains %u32 items",
ivars->buf_max - ivars->buf_tick);
}
ivars->buf_max = 0;
ivars->buf_tick = 0;
// Ditch old MemoryPool and get another.
DECREF(ivars->mem_pool);
ivars->mem_pool = MemPool_new(0);
MemoryPool *const mem_pool = ivars->mem_pool;
MemoryPoolIVARS *const mem_pool_ivars = MemPool_IVARS(mem_pool);
while (1) {
if (ivars->post_count == 0) {
// Read a term.
if (Lex_Next(lexicon)) {
ivars->post_count = Lex_Doc_Freq(lexicon);
term_text = (String*)Lex_Get_Term(lexicon);
if (term_text && !Obj_Is_A((Obj*)term_text, STRING)) {
THROW(ERR, "Only String terms are supported for now");
}
Posting *posting = PList_Get_Posting(plist);
Post_Set_Doc_ID(posting, doc_base);
ivars->last_doc_id = doc_base;
}
// Bail if we've read everything in this run.
else {
break;
}
}
// Bail if we've hit the ceiling for this run's buffer.
if (mem_pool_ivars->consumed >= mem_thresh && num_elems > 0) {
break;
}
// Read a posting from the input stream.
RawPosting *rawpost
= PList_Read_Raw(plist, ivars->last_doc_id, term_text, mem_pool);
RawPostingIVARS *const rawpost_ivars = RawPost_IVARS(rawpost);
ivars->last_doc_id = rawpost_ivars->doc_id;
ivars->post_count--;
// Skip deletions.
if (doc_map != NULL) {
const int32_t remapped
= I32Arr_Get(doc_map, rawpost_ivars->doc_id - doc_base);
if (!remapped) {
continue;
}
rawpost_ivars->doc_id = remapped;
}
// Add to the run's buffer.
if (num_elems >= ivars->buf_cap) {
size_t new_cap = Memory_oversize(num_elems + 1, sizeof(Obj*));
PostPool_Grow_Buffer(self, new_cap);
}
ivars->buffer[num_elems] = (Obj*)rawpost;
num_elems++;
}
// Reset the buffer array position and length; remember file pos.
ivars->buf_max = num_elems;
ivars->buf_tick = 0;
return num_elems;
}
