void
HLWriter_delete_segment(HighlightWriter *self, SegReader *reader)
{
CharBuf *merged_seg_name = Seg_Get_Name(SegReader_Get_Segment(reader));
Snapshot *snapshot = HLWriter_Get_Snapshot(self);
CharBuf *ix_file = CB_newf("%o/highlight.ix", merged_seg_name);
CharBuf *dat_file = CB_newf("%o/highlight.dat", merged_seg_name);
Snapshot_Delete_Entry(snapshot, ix_file);
Snapshot_Delete_Entry(snapshot, dat_file);
DECREF(ix_file);
DECREF(dat_file);
}
void
Indexer_prepare_commit(Indexer *self)
{
VArray *seg_readers = PolyReader_Get_Seg_Readers(self->polyreader);
uint32_t num_seg_readers = VA_Get_Size(seg_readers);
bool_t merge_happened = false;
if ( !self->write_lock || self->prepared ) {
THROW(ERR, "Can't call Prepare_Commit() more than once");
}
// Merge existing index data.
if (num_seg_readers) {
merge_happened = S_maybe_merge(self, seg_readers);
}
// Add a new segment and write a new snapshot file if...
if ( Seg_Get_Count(self->segment) // Docs/segs added.
|| merge_happened // Some segs merged.
|| !Snapshot_Num_Entries(self->snapshot) // Initializing index.
|| DelWriter_Updated(self->del_writer)
) {
Folder *folder = self->folder;
Schema *schema = self->schema;
Snapshot *snapshot = self->snapshot;
CharBuf *old_schema_name = S_find_schema_file(snapshot);
uint64_t schema_gen = old_schema_name
? IxFileNames_extract_gen(old_schema_name) + 1
: 1;
char base36[StrHelp_MAX_BASE36_BYTES];
CharBuf *new_schema_name;
StrHelp_to_base36(schema_gen, &base36);
new_schema_name = CB_newf("schema_%s.json", base36);
// Finish the segment, write schema file.
SegWriter_Finish(self->seg_writer);
Schema_Write(schema, folder, new_schema_name);
if (old_schema_name) {
Snapshot_Delete_Entry(snapshot, old_schema_name);
}
Snapshot_Add_Entry(snapshot, new_schema_name);
DECREF(new_schema_name);
// Write temporary snapshot file.
DECREF(self->snapfile);
self->snapfile = IxManager_Make_Snapshot_Filename(self->manager);
CB_Cat_Trusted_Str(self->snapfile, ".temp", 5);
Folder_Delete(folder, self->snapfile);
Snapshot_Write_File(snapshot, folder, self->snapfile);
self->needs_commit = true;
}
// Close reader, so that we can delete its files if appropriate.
PolyReader_Close(self->polyreader);
self->prepared = true;
}
void
Indexer_Prepare_Commit_IMP(Indexer *self) {
IndexerIVARS *const ivars = Indexer_IVARS(self);
Vector *seg_readers = PolyReader_Get_Seg_Readers(ivars->polyreader);
size_t num_seg_readers = Vec_Get_Size(seg_readers);
bool merge_happened = false;
if (!ivars->write_lock || ivars->prepared) {
THROW(ERR, "Can't call Prepare_Commit() more than once");
}
// Merge existing index data.
if (num_seg_readers) {
merge_happened = S_maybe_merge(self, seg_readers);
}
// Add a new segment and write a new snapshot file if...
if (Seg_Get_Count(ivars->segment) // Docs/segs added.
|| merge_happened // Some segs merged.
|| !Snapshot_Num_Entries(ivars->snapshot) // Initializing index.
|| DelWriter_Updated(ivars->del_writer)
) {
Folder *folder = ivars->folder;
Schema *schema = ivars->schema;
Snapshot *snapshot = ivars->snapshot;
// Derive snapshot and schema file names.
DECREF(ivars->snapfile);
String *snapfile = IxManager_Make_Snapshot_Filename(ivars->manager);
ivars->snapfile = Str_Cat_Trusted_Utf8(snapfile, ".temp", 5);
DECREF(snapfile);
uint64_t schema_gen = IxFileNames_extract_gen(ivars->snapfile);
char base36[StrHelp_MAX_BASE36_BYTES];
StrHelp_to_base36(schema_gen, &base36);
String *new_schema_name = Str_newf("schema_%s.json", base36);
// Finish the segment, write schema file.
SegWriter_Finish(ivars->seg_writer);
Schema_Write(schema, folder, new_schema_name);
String *old_schema_name = S_find_schema_file(snapshot);
if (old_schema_name) {
Snapshot_Delete_Entry(snapshot, old_schema_name);
}
Snapshot_Add_Entry(snapshot, new_schema_name);
DECREF(new_schema_name);
// Write temporary snapshot file.
Folder_Delete(folder, ivars->snapfile);
Snapshot_Write_File(snapshot, folder, ivars->snapfile);
ivars->needs_commit = true;
}
// Close reader, so that we can delete its files if appropriate.
PolyReader_Close(ivars->polyreader);
ivars->prepared = true;
}
void
SegWriter_Delete_Segment_IMP(SegWriter *self, SegReader *reader) {
SegWriterIVARS *const ivars = SegWriter_IVARS(self);
Snapshot *snapshot = SegWriter_Get_Snapshot(self);
String *seg_name = Seg_Get_Name(SegReader_Get_Segment(reader));
// Have all the sub-writers delete the segment.
for (size_t i = 0, max = Vec_Get_Size(ivars->writers); i < max; i++) {
DataWriter *writer = (DataWriter*)Vec_Fetch(ivars->writers, i);
DataWriter_Delete_Segment(writer, reader);
}
DelWriter_Delete_Segment(ivars->del_writer, reader);
// Remove seg directory from snapshot.
Snapshot_Delete_Entry(snapshot, seg_name);
}
void
LexWriter_delete_segment(LexiconWriter *self, SegReader *reader)
{
Snapshot *snapshot = LexWriter_Get_Snapshot(self);
CharBuf *merged_seg_name = Seg_Get_Name(SegReader_Get_Segment(reader));
CharBuf *pattern = CB_newf("%o/lexicon", merged_seg_name);
VArray *files = Snapshot_List(snapshot);
VArray *my_old_files = VA_Grep(files, S_my_file, pattern);
u32_t i, max;
for (i = 0, max = VA_Get_Size(my_old_files); i < max; i++) {
CharBuf *entry = (CharBuf*)VA_Fetch(my_old_files, i);
Snapshot_Delete_Entry(snapshot, entry);
}
DECREF(my_old_files);
DECREF(files);
DECREF(pattern);
}
void
SegWriter_Merge_Segment_IMP(SegWriter *self, SegReader *reader,
I32Array *doc_map) {
SegWriterIVARS *const ivars = SegWriter_IVARS(self);
Snapshot *snapshot = SegWriter_Get_Snapshot(self);
String *seg_name = Seg_Get_Name(SegReader_Get_Segment(reader));
// Have all the sub-writers merge the segment.
for (uint32_t i = 0, max = Vec_Get_Size(ivars->writers); i < max; i++) {
DataWriter *writer = (DataWriter*)Vec_Fetch(ivars->writers, i);
DataWriter_Merge_Segment(writer, reader, doc_map);
}
DelWriter_Merge_Segment(ivars->del_writer, reader, doc_map);
// Remove seg directory from snapshot.
Snapshot_Delete_Entry(snapshot, seg_name);
// Adust the document id.
S_adjust_doc_id(self, reader, doc_map);
}
static void
test_Add_and_Delete(TestBatchRunner *runner) {
Snapshot *snapshot = Snapshot_new();
String *foo = (String*)SSTR_WRAP_UTF8("foo", 3);
String *bar = (String*)SSTR_WRAP_UTF8("bar", 3);
Snapshot_Add_Entry(snapshot, foo);
Snapshot_Add_Entry(snapshot, foo); // redundant
Vector *entries = Snapshot_List(snapshot);
TEST_INT_EQ(runner, Snapshot_Num_Entries(snapshot), 1,
"One entry added");
TEST_TRUE(runner, Str_Equals(foo, Vec_Fetch(entries, 0)), "correct entry");
DECREF(entries);
Snapshot_Add_Entry(snapshot, bar);
TEST_INT_EQ(runner, Snapshot_Num_Entries(snapshot), 2,
"second entry added");
Snapshot_Delete_Entry(snapshot, foo);
TEST_INT_EQ(runner, Snapshot_Num_Entries(snapshot), 1, "Delete_Entry");
DECREF(snapshot);
}