Exemple #1
0
void
TestMemPool_run_tests() {
    TestBatch  *batch     = TestBatch_new(4);
    MemoryPool *mem_pool  = MemPool_new(0);
    MemoryPool *other     = MemPool_new(0);
    char *ptr_a, *ptr_b;

    TestBatch_Plan(batch);

    ptr_a = (char*)MemPool_Grab(mem_pool, 10);
    strcpy(ptr_a, "foo");
    MemPool_Release_All(mem_pool);

    ptr_b = (char*)MemPool_Grab(mem_pool, 10);
    TEST_STR_EQ(batch, ptr_b, "foo", "Recycle RAM on Release_All");

    ptr_a = mem_pool->buf;
    MemPool_Resize(mem_pool, ptr_b, 6);
    TEST_TRUE(batch, mem_pool->buf < ptr_a, "Resize");

    ptr_a = (char*)MemPool_Grab(other, 20);
    MemPool_Release_All(other);
    MemPool_Eat(other, mem_pool);
    TEST_TRUE(batch, other->buf == mem_pool->buf, "Eat");
    TEST_TRUE(batch, other->buf != NULL, "Eat");

    DECREF(mem_pool);
    DECREF(other);
    DECREF(batch);
}
Exemple #2
0
int main(int argc, char** argv) {
  char* file = argv[1];
  char* buf = 0;
  long buflen = 0;
  FILE* fd = fopen(file, "r");
  TokenType token = 0;
  MemPool* pool = MemPool_new(1 << 30);
  Lexer* lexer = Lexer_new(pool);

  fseek(fd, 0, SEEK_END);
  buflen = ftell(fd);
  fseek(fd, 0, SEEK_SET);
  buf = malloc(buflen);
  fread(buf, 1, buflen, fd);

  Lexer_setpos(lexer, buf);
  
  while (token != TT_EOF) {
    token = Lexer_debugtoken(lexer);
    printf("\n");
  }


  return 0;
}
Exemple #3
0
SortWriter*
SortWriter_init(SortWriter *self, Schema *schema, Snapshot *snapshot,
                Segment *segment, PolyReader *polyreader) {
    uint32_t field_max = Schema_Num_Fields(schema) + 1;
    DataWriter_init((DataWriter*)self, schema, snapshot, segment, polyreader);
    SortWriterIVARS *const ivars = SortWriter_IVARS(self);

    // Init.
    ivars->field_writers   = VA_new(field_max);
    ivars->counts          = Hash_new(0);
    ivars->null_ords       = Hash_new(0);
    ivars->ord_widths      = Hash_new(0);
    ivars->temp_ord_out    = NULL;
    ivars->temp_ix_out     = NULL;
    ivars->temp_dat_out    = NULL;
    ivars->mem_pool        = MemPool_new(0);
    ivars->mem_thresh      = default_mem_thresh;
    ivars->flush_at_finish = false;

    return self;
}
Exemple #4
0
uint32_t
PostPool_Refill_IMP(PostingPool *self) {
    PostingPoolIVARS *const ivars = PostPool_IVARS(self);
    Lexicon *const     lexicon     = ivars->lexicon;
    PostingList *const plist       = ivars->plist;
    I32Array    *const doc_map     = ivars->doc_map;
    const uint32_t     mem_thresh  = ivars->mem_thresh;
    const int32_t      doc_base    = ivars->doc_base;
    uint32_t           num_elems   = 0; // number of items recovered
    String            *term_text   = NULL;

    if (ivars->lexicon == NULL) { return 0; }
    else { term_text = (String*)Lex_Get_Term(lexicon); }

    // Make sure buffer is empty.
    if (ivars->buf_max - ivars->buf_tick > 0) {
        THROW(ERR, "Refill called but buffer contains %u32 items",
              ivars->buf_max - ivars->buf_tick);
    }
    ivars->buf_max  = 0;
    ivars->buf_tick = 0;

    // Ditch old MemoryPool and get another.
    DECREF(ivars->mem_pool);
    ivars->mem_pool = MemPool_new(0);
    MemoryPool *const mem_pool = ivars->mem_pool;
    MemoryPoolIVARS *const mem_pool_ivars = MemPool_IVARS(mem_pool);


    while (1) {
        if (ivars->post_count == 0) {
            // Read a term.
            if (Lex_Next(lexicon)) {
                ivars->post_count = Lex_Doc_Freq(lexicon);
                term_text = (String*)Lex_Get_Term(lexicon);
                if (term_text && !Obj_Is_A((Obj*)term_text, STRING)) {
                    THROW(ERR, "Only String terms are supported for now");
                }
                Posting *posting = PList_Get_Posting(plist);
                Post_Set_Doc_ID(posting, doc_base);
                ivars->last_doc_id = doc_base;
            }
            // Bail if we've read everything in this run.
            else {
                break;
            }
        }

        // Bail if we've hit the ceiling for this run's buffer.
        if (mem_pool_ivars->consumed >= mem_thresh && num_elems > 0) {
            break;
        }

        // Read a posting from the input stream.
        RawPosting *rawpost
            = PList_Read_Raw(plist, ivars->last_doc_id, term_text, mem_pool);
        RawPostingIVARS *const rawpost_ivars = RawPost_IVARS(rawpost);
        ivars->last_doc_id = rawpost_ivars->doc_id;
        ivars->post_count--;

        // Skip deletions.
        if (doc_map != NULL) {
            const int32_t remapped
                = I32Arr_Get(doc_map, rawpost_ivars->doc_id - doc_base);
            if (!remapped) {
                continue;
            }
            rawpost_ivars->doc_id = remapped;
        }

        // Add to the run's buffer.
        if (num_elems >= ivars->buf_cap) {
            size_t new_cap = Memory_oversize(num_elems + 1, sizeof(Obj*));
            PostPool_Grow_Buffer(self, new_cap);
        }
        ivars->buffer[num_elems] = (Obj*)rawpost;
        num_elems++;
    }

    // Reset the buffer array position and length; remember file pos.
    ivars->buf_max   = num_elems;
    ivars->buf_tick  = 0;

    return num_elems;
}