void HLWriter_delete_segment(HighlightWriter *self, SegReader *reader) { CharBuf *merged_seg_name = Seg_Get_Name(SegReader_Get_Segment(reader)); Snapshot *snapshot = HLWriter_Get_Snapshot(self); CharBuf *ix_file = CB_newf("%o/highlight.ix", merged_seg_name); CharBuf *dat_file = CB_newf("%o/highlight.dat", merged_seg_name); Snapshot_Delete_Entry(snapshot, ix_file); Snapshot_Delete_Entry(snapshot, dat_file); DECREF(ix_file); DECREF(dat_file); }
void Indexer_prepare_commit(Indexer *self) { VArray *seg_readers = PolyReader_Get_Seg_Readers(self->polyreader); uint32_t num_seg_readers = VA_Get_Size(seg_readers); bool_t merge_happened = false; if ( !self->write_lock || self->prepared ) { THROW(ERR, "Can't call Prepare_Commit() more than once"); } // Merge existing index data. if (num_seg_readers) { merge_happened = S_maybe_merge(self, seg_readers); } // Add a new segment and write a new snapshot file if... if ( Seg_Get_Count(self->segment) // Docs/segs added. || merge_happened // Some segs merged. || !Snapshot_Num_Entries(self->snapshot) // Initializing index. || DelWriter_Updated(self->del_writer) ) { Folder *folder = self->folder; Schema *schema = self->schema; Snapshot *snapshot = self->snapshot; CharBuf *old_schema_name = S_find_schema_file(snapshot); uint64_t schema_gen = old_schema_name ? IxFileNames_extract_gen(old_schema_name) + 1 : 1; char base36[StrHelp_MAX_BASE36_BYTES]; CharBuf *new_schema_name; StrHelp_to_base36(schema_gen, &base36); new_schema_name = CB_newf("schema_%s.json", base36); // Finish the segment, write schema file. SegWriter_Finish(self->seg_writer); Schema_Write(schema, folder, new_schema_name); if (old_schema_name) { Snapshot_Delete_Entry(snapshot, old_schema_name); } Snapshot_Add_Entry(snapshot, new_schema_name); DECREF(new_schema_name); // Write temporary snapshot file. DECREF(self->snapfile); self->snapfile = IxManager_Make_Snapshot_Filename(self->manager); CB_Cat_Trusted_Str(self->snapfile, ".temp", 5); Folder_Delete(folder, self->snapfile); Snapshot_Write_File(snapshot, folder, self->snapfile); self->needs_commit = true; } // Close reader, so that we can delete its files if appropriate. PolyReader_Close(self->polyreader); self->prepared = true; }
void Indexer_Prepare_Commit_IMP(Indexer *self) { IndexerIVARS *const ivars = Indexer_IVARS(self); Vector *seg_readers = PolyReader_Get_Seg_Readers(ivars->polyreader); size_t num_seg_readers = Vec_Get_Size(seg_readers); bool merge_happened = false; if (!ivars->write_lock || ivars->prepared) { THROW(ERR, "Can't call Prepare_Commit() more than once"); } // Merge existing index data. if (num_seg_readers) { merge_happened = S_maybe_merge(self, seg_readers); } // Add a new segment and write a new snapshot file if... if (Seg_Get_Count(ivars->segment) // Docs/segs added. || merge_happened // Some segs merged. || !Snapshot_Num_Entries(ivars->snapshot) // Initializing index. || DelWriter_Updated(ivars->del_writer) ) { Folder *folder = ivars->folder; Schema *schema = ivars->schema; Snapshot *snapshot = ivars->snapshot; // Derive snapshot and schema file names. DECREF(ivars->snapfile); String *snapfile = IxManager_Make_Snapshot_Filename(ivars->manager); ivars->snapfile = Str_Cat_Trusted_Utf8(snapfile, ".temp", 5); DECREF(snapfile); uint64_t schema_gen = IxFileNames_extract_gen(ivars->snapfile); char base36[StrHelp_MAX_BASE36_BYTES]; StrHelp_to_base36(schema_gen, &base36); String *new_schema_name = Str_newf("schema_%s.json", base36); // Finish the segment, write schema file. SegWriter_Finish(ivars->seg_writer); Schema_Write(schema, folder, new_schema_name); String *old_schema_name = S_find_schema_file(snapshot); if (old_schema_name) { Snapshot_Delete_Entry(snapshot, old_schema_name); } Snapshot_Add_Entry(snapshot, new_schema_name); DECREF(new_schema_name); // Write temporary snapshot file. Folder_Delete(folder, ivars->snapfile); Snapshot_Write_File(snapshot, folder, ivars->snapfile); ivars->needs_commit = true; } // Close reader, so that we can delete its files if appropriate. PolyReader_Close(ivars->polyreader); ivars->prepared = true; }
void SegWriter_Delete_Segment_IMP(SegWriter *self, SegReader *reader) { SegWriterIVARS *const ivars = SegWriter_IVARS(self); Snapshot *snapshot = SegWriter_Get_Snapshot(self); String *seg_name = Seg_Get_Name(SegReader_Get_Segment(reader)); // Have all the sub-writers delete the segment. for (size_t i = 0, max = Vec_Get_Size(ivars->writers); i < max; i++) { DataWriter *writer = (DataWriter*)Vec_Fetch(ivars->writers, i); DataWriter_Delete_Segment(writer, reader); } DelWriter_Delete_Segment(ivars->del_writer, reader); // Remove seg directory from snapshot. Snapshot_Delete_Entry(snapshot, seg_name); }
void LexWriter_delete_segment(LexiconWriter *self, SegReader *reader) { Snapshot *snapshot = LexWriter_Get_Snapshot(self); CharBuf *merged_seg_name = Seg_Get_Name(SegReader_Get_Segment(reader)); CharBuf *pattern = CB_newf("%o/lexicon", merged_seg_name); VArray *files = Snapshot_List(snapshot); VArray *my_old_files = VA_Grep(files, S_my_file, pattern); u32_t i, max; for (i = 0, max = VA_Get_Size(my_old_files); i < max; i++) { CharBuf *entry = (CharBuf*)VA_Fetch(my_old_files, i); Snapshot_Delete_Entry(snapshot, entry); } DECREF(my_old_files); DECREF(files); DECREF(pattern); }
void SegWriter_Merge_Segment_IMP(SegWriter *self, SegReader *reader, I32Array *doc_map) { SegWriterIVARS *const ivars = SegWriter_IVARS(self); Snapshot *snapshot = SegWriter_Get_Snapshot(self); String *seg_name = Seg_Get_Name(SegReader_Get_Segment(reader)); // Have all the sub-writers merge the segment. for (uint32_t i = 0, max = Vec_Get_Size(ivars->writers); i < max; i++) { DataWriter *writer = (DataWriter*)Vec_Fetch(ivars->writers, i); DataWriter_Merge_Segment(writer, reader, doc_map); } DelWriter_Merge_Segment(ivars->del_writer, reader, doc_map); // Remove seg directory from snapshot. Snapshot_Delete_Entry(snapshot, seg_name); // Adust the document id. S_adjust_doc_id(self, reader, doc_map); }
static void test_Add_and_Delete(TestBatchRunner *runner) { Snapshot *snapshot = Snapshot_new(); String *foo = (String*)SSTR_WRAP_UTF8("foo", 3); String *bar = (String*)SSTR_WRAP_UTF8("bar", 3); Snapshot_Add_Entry(snapshot, foo); Snapshot_Add_Entry(snapshot, foo); // redundant Vector *entries = Snapshot_List(snapshot); TEST_INT_EQ(runner, Snapshot_Num_Entries(snapshot), 1, "One entry added"); TEST_TRUE(runner, Str_Equals(foo, Vec_Fetch(entries, 0)), "correct entry"); DECREF(entries); Snapshot_Add_Entry(snapshot, bar); TEST_INT_EQ(runner, Snapshot_Num_Entries(snapshot), 2, "second entry added"); Snapshot_Delete_Entry(snapshot, foo); TEST_INT_EQ(runner, Snapshot_Num_Entries(snapshot), 1, "Delete_Entry"); DECREF(snapshot); }