void DefDelWriter_finish(DefaultDeletionsWriter *self) { DefaultDeletionsWriterIVARS *const ivars = DefDelWriter_IVARS(self); Folder *const folder = ivars->folder; for (uint32_t i = 0, max = VA_Get_Size(ivars->seg_readers); i < max; i++) { SegReader *seg_reader = (SegReader*)VA_Fetch(ivars->seg_readers, i); if (ivars->updated[i]) { BitVector *deldocs = (BitVector*)VA_Fetch(ivars->bit_vecs, i); int32_t doc_max = SegReader_Doc_Max(seg_reader); double used = (doc_max + 1) / 8.0; uint32_t byte_size = (uint32_t)ceil(used); uint32_t new_max = byte_size * 8 - 1; CharBuf *filename = S_del_filename(self, seg_reader); OutStream *outstream = Folder_Open_Out(folder, filename); if (!outstream) { RETHROW(INCREF(Err_get_error())); } // Ensure that we have 1 bit for each doc in segment. BitVec_Grow(deldocs, new_max); // Write deletions data and clean up. OutStream_Write_Bytes(outstream, (char*)BitVec_Get_Raw_Bits(deldocs), byte_size); OutStream_Close(outstream); DECREF(outstream); DECREF(filename); } } Seg_Store_Metadata_Str(ivars->segment, "deletions", 9, (Obj*)DefDelWriter_Metadata(self)); }
static void test_Write_File_and_Read_File(TestBatch *batch) { RAMFolder *folder = RAMFolder_new(NULL); Segment *segment = Seg_new(100); Segment *got = Seg_new(100); CharBuf *meta; CharBuf *flotsam = (CharBuf*)ZCB_WRAP_STR("flotsam", 7); CharBuf *jetsam = (CharBuf*)ZCB_WRAP_STR("jetsam", 6); Seg_Set_Count(segment, 111); Seg_Store_Metadata_Str(segment, "foo", 3, (Obj*)CB_newf("bar")); Seg_Add_Field(segment, flotsam); Seg_Add_Field(segment, jetsam); RAMFolder_MkDir(folder, Seg_Get_Name(segment)); Seg_Write_File(segment, (Folder*)folder); Seg_Read_File(got, (Folder*)folder); TEST_TRUE(batch, Seg_Get_Count(got) == Seg_Get_Count(segment), "Round-trip count through file"); TEST_TRUE(batch, Seg_Field_Num(got, jetsam) == Seg_Field_Num(segment, jetsam), "Round trip field names through file"); meta = (CharBuf*)Seg_Fetch_Metadata_Str(got, "foo", 3); TEST_TRUE(batch, meta && CB_Is_A(meta, CHARBUF) && CB_Equals_Str(meta, "bar", 3), "Round trip metadata through file"); DECREF(got); DECREF(segment); DECREF(folder); }
static void test_metadata_storage(TestBatch *batch) { Segment *segment = Seg_new(1); CharBuf *got; Seg_Store_Metadata_Str(segment, "foo", 3, (Obj*)CB_newf("bar")); got = (CharBuf*)Seg_Fetch_Metadata_Str(segment, "foo", 3); TEST_TRUE(batch, got && CB_Is_A(got, CHARBUF) && CB_Equals_Str(got, "bar", 3), "metadata round trip"); DECREF(segment); }
void DocWriter_finish(DocWriter *self) { if (self->dat_out) { // Write one final file pointer, so that we can derive the length of // the last record. int64_t end = OutStream_Tell(self->dat_out); OutStream_Write_I64(self->ix_out, end); // Close down output streams. OutStream_Close(self->dat_out); OutStream_Close(self->ix_out); Seg_Store_Metadata_Str(self->segment, "documents", 9, (Obj*)DocWriter_Metadata(self)); } }
void HLWriter_finish(HighlightWriter *self) { HighlightWriterIVARS *const ivars = HLWriter_IVARS(self); if (ivars->dat_out) { // Write one final file pointer, so that we can derive the length of // the last record. int64_t end = OutStream_Tell(ivars->dat_out); OutStream_Write_I64(ivars->ix_out, end); // Close down the output streams. OutStream_Close(ivars->dat_out); OutStream_Close(ivars->ix_out); Seg_Store_Metadata_Str(ivars->segment, "highlight", 9, (Obj*)HLWriter_Metadata(self)); } }
void HLWriter_finish(HighlightWriter *self) { if (self->dat_out) { /* Write one final file pointer, so that we can derive the length of * the last record. */ i64_t end = OutStream_Tell(self->dat_out); OutStream_Write_U64(self->ix_out, end); /* Close down the output streams. */ OutStream_Close(self->dat_out); OutStream_Close(self->ix_out); Seg_Store_Metadata_Str(self->segment, "highlight", 9, (Obj*)HLWriter_Metadata(self)); } }
void LexWriter_finish(LexiconWriter *self) { /* Ensure that streams were closed (by calling Finish_Field or * Leave_Temp_Mode). */ if (self->dat_out != NULL) { THROW("File '%o' never closed", self->dat_file); } else if (self->ix_out != NULL) { THROW("File '%o' never closed", self->ix_file); } else if (self->ix_out != NULL) { THROW("File '%o' never closed", self->ix_file); } /* Store metadata. */ Seg_Store_Metadata_Str(self->segment, "lexicon", 7, (Obj*)LexWriter_Metadata(self)); }
void SortWriter_finish(SortWriter *self) { SortWriterIVARS *const ivars = SortWriter_IVARS(self); VArray *const field_writers = ivars->field_writers; // If we have no data, bail out. if (!ivars->temp_ord_out) { return; } // If we've either flushed or added segments, flush everything so that any // one field can use the entire margin up to mem_thresh. if (ivars->flush_at_finish) { for (uint32_t i = 1, max = VA_Get_Size(field_writers); i < max; i++) { SortFieldWriter *field_writer = (SortFieldWriter*)VA_Fetch(field_writers, i); if (field_writer) { SortFieldWriter_Flush(field_writer); } } } // Close down temp streams. OutStream_Close(ivars->temp_ord_out); OutStream_Close(ivars->temp_ix_out); OutStream_Close(ivars->temp_dat_out); for (uint32_t i = 1, max = VA_Get_Size(field_writers); i < max; i++) { SortFieldWriter *field_writer = (SortFieldWriter*)VA_Delete(field_writers, i); if (field_writer) { CharBuf *field = Seg_Field_Name(ivars->segment, i); SortFieldWriter_Flip(field_writer); int32_t count = SortFieldWriter_Finish(field_writer); Hash_Store(ivars->counts, (Obj*)field, (Obj*)CB_newf("%i32", count)); int32_t null_ord = SortFieldWriter_Get_Null_Ord(field_writer); if (null_ord != -1) { Hash_Store(ivars->null_ords, (Obj*)field, (Obj*)CB_newf("%i32", null_ord)); } int32_t ord_width = SortFieldWriter_Get_Ord_Width(field_writer); Hash_Store(ivars->ord_widths, (Obj*)field, (Obj*)CB_newf("%i32", ord_width)); } DECREF(field_writer); } VA_Clear(field_writers); // Store metadata. Seg_Store_Metadata_Str(ivars->segment, "sort", 4, (Obj*)SortWriter_Metadata(self)); // Clean up. Folder *folder = ivars->folder; CharBuf *seg_name = Seg_Get_Name(ivars->segment); CharBuf *path = CB_newf("%o/sort_ord_temp", seg_name); Folder_Delete(folder, path); CB_setf(path, "%o/sort_ix_temp", seg_name); Folder_Delete(folder, path); CB_setf(path, "%o/sort_dat_temp", seg_name); Folder_Delete(folder, path); DECREF(path); }