void
LexWriter_start_field(LexiconWriter *self, i32_t field_num)
{
CharBuf *const seg_name = Seg_Get_Name(self->segment);
Folder *const folder = self->folder;
Snapshot *const snapshot = LexWriter_Get_Snapshot(self);
/* Open outstreams. */
CB_setf(self->dat_file, "%o/lexicon-%i32.dat", seg_name, field_num);
CB_setf(self->ix_file, "%o/lexicon-%i32.ix", seg_name, field_num);
CB_setf(self->ixix_file, "%o/lexicon-%i32.ixix", seg_name, field_num);
Snapshot_Add_Entry(snapshot, self->dat_file);
Snapshot_Add_Entry(snapshot, self->ix_file);
Snapshot_Add_Entry(snapshot, self->ixix_file);
self->dat_out = Folder_Open_Out(folder, self->dat_file);
self->ix_out = Folder_Open_Out(folder, self->ix_file);
self->ixix_out = Folder_Open_Out(folder, self->ixix_file);
if (!self->dat_out) { THROW("Can't open %o", self->dat_file); }
if (!self->ix_out) { THROW("Can't open %o", self->ix_file); }
if (!self->ixix_out) { THROW("Can't open %o", self->ixix_file); }
/* Initialize count and ix_count, TermInfo and last term text. */
self->count = 0;
self->ix_count = 0;
TInfo_Reset(self->last_tinfo);
CB_Set_Size(self->last_text, 0);
}
static OutStream*
S_lazy_init(HighlightWriter *self)
{
if (!self->dat_out) {
Segment *segment = self->segment;
Folder *folder = self->folder;
Snapshot *snapshot = HLWriter_Get_Snapshot(self);
CharBuf *seg_name = Seg_Get_Name(segment);
CharBuf *ix_file = CB_newf("%o/highlight.ix", seg_name);
CharBuf *dat_file = CB_newf("%o/highlight.dat", seg_name);
/* Open outstreams. */
Snapshot_Add_Entry(snapshot, ix_file);
Snapshot_Add_Entry(snapshot, dat_file);
self->ix_out = Folder_Open_Out(folder, ix_file);
self->dat_out = Folder_Open_Out(folder, dat_file);
if (!self->ix_out) { THROW("Can't open %o", ix_file); }
if (!self->dat_out) { THROW("Can't open %o", dat_file); }
DECREF(ix_file);
DECREF(dat_file);
/* Go past invalid doc 0. */
OutStream_Write_U64(self->ix_out, 0);
}
return self->dat_out;
}
static void
test_Read_File_and_Write_File(TestBatchRunner *runner) {
Snapshot *snapshot = Snapshot_new();
Folder *folder = (Folder*)RAMFolder_new(NULL);
String *snap = (String*)SSTR_WRAP_UTF8("snap", 4);
String *foo = (String*)SSTR_WRAP_UTF8("foo", 3);
Snapshot_Add_Entry(snapshot, foo);
Snapshot_Write_File(snapshot, folder, snap);
Snapshot *dupe = Snapshot_new();
Snapshot *read_retval = Snapshot_Read_File(dupe, folder, snap);
TEST_TRUE(runner, dupe == read_retval, "Read_File() returns the object");
Vector *orig_list = Snapshot_List(snapshot);
Vector *dupe_list = Snapshot_List(dupe);
TEST_TRUE(runner, Vec_Equals(orig_list, (Obj*)dupe_list),
"Round trip through Write_File() and Read_File()");
DECREF(orig_list);
DECREF(dupe_list);
DECREF(dupe);
DECREF(snapshot);
DECREF(folder);
}
void
Indexer_prepare_commit(Indexer *self)
{
VArray *seg_readers = PolyReader_Get_Seg_Readers(self->polyreader);
uint32_t num_seg_readers = VA_Get_Size(seg_readers);
bool_t merge_happened = false;
if ( !self->write_lock || self->prepared ) {
THROW(ERR, "Can't call Prepare_Commit() more than once");
}
// Merge existing index data.
if (num_seg_readers) {
merge_happened = S_maybe_merge(self, seg_readers);
}
// Add a new segment and write a new snapshot file if...
if ( Seg_Get_Count(self->segment) // Docs/segs added.
|| merge_happened // Some segs merged.
|| !Snapshot_Num_Entries(self->snapshot) // Initializing index.
|| DelWriter_Updated(self->del_writer)
) {
Folder *folder = self->folder;
Schema *schema = self->schema;
Snapshot *snapshot = self->snapshot;
CharBuf *old_schema_name = S_find_schema_file(snapshot);
uint64_t schema_gen = old_schema_name
? IxFileNames_extract_gen(old_schema_name) + 1
: 1;
char base36[StrHelp_MAX_BASE36_BYTES];
CharBuf *new_schema_name;
StrHelp_to_base36(schema_gen, &base36);
new_schema_name = CB_newf("schema_%s.json", base36);
// Finish the segment, write schema file.
SegWriter_Finish(self->seg_writer);
Schema_Write(schema, folder, new_schema_name);
if (old_schema_name) {
Snapshot_Delete_Entry(snapshot, old_schema_name);
}
Snapshot_Add_Entry(snapshot, new_schema_name);
DECREF(new_schema_name);
// Write temporary snapshot file.
DECREF(self->snapfile);
self->snapfile = IxManager_Make_Snapshot_Filename(self->manager);
CB_Cat_Trusted_Str(self->snapfile, ".temp", 5);
Folder_Delete(folder, self->snapfile);
Snapshot_Write_File(snapshot, folder, self->snapfile);
self->needs_commit = true;
}
// Close reader, so that we can delete its files if appropriate.
PolyReader_Close(self->polyreader);
self->prepared = true;
}
void
Indexer_Prepare_Commit_IMP(Indexer *self) {
IndexerIVARS *const ivars = Indexer_IVARS(self);
Vector *seg_readers = PolyReader_Get_Seg_Readers(ivars->polyreader);
size_t num_seg_readers = Vec_Get_Size(seg_readers);
bool merge_happened = false;
if (!ivars->write_lock || ivars->prepared) {
THROW(ERR, "Can't call Prepare_Commit() more than once");
}
// Merge existing index data.
if (num_seg_readers) {
merge_happened = S_maybe_merge(self, seg_readers);
}
// Add a new segment and write a new snapshot file if...
if (Seg_Get_Count(ivars->segment) // Docs/segs added.
|| merge_happened // Some segs merged.
|| !Snapshot_Num_Entries(ivars->snapshot) // Initializing index.
|| DelWriter_Updated(ivars->del_writer)
) {
Folder *folder = ivars->folder;
Schema *schema = ivars->schema;
Snapshot *snapshot = ivars->snapshot;
// Derive snapshot and schema file names.
DECREF(ivars->snapfile);
String *snapfile = IxManager_Make_Snapshot_Filename(ivars->manager);
ivars->snapfile = Str_Cat_Trusted_Utf8(snapfile, ".temp", 5);
DECREF(snapfile);
uint64_t schema_gen = IxFileNames_extract_gen(ivars->snapfile);
char base36[StrHelp_MAX_BASE36_BYTES];
StrHelp_to_base36(schema_gen, &base36);
String *new_schema_name = Str_newf("schema_%s.json", base36);
// Finish the segment, write schema file.
SegWriter_Finish(ivars->seg_writer);
Schema_Write(schema, folder, new_schema_name);
String *old_schema_name = S_find_schema_file(snapshot);
if (old_schema_name) {
Snapshot_Delete_Entry(snapshot, old_schema_name);
}
Snapshot_Add_Entry(snapshot, new_schema_name);
DECREF(new_schema_name);
// Write temporary snapshot file.
Folder_Delete(folder, ivars->snapfile);
Snapshot_Write_File(snapshot, folder, ivars->snapfile);
ivars->needs_commit = true;
}
// Close reader, so that we can delete its files if appropriate.
PolyReader_Close(ivars->polyreader);
ivars->prepared = true;
}
static void
test_Add_and_Delete(TestBatchRunner *runner) {
Snapshot *snapshot = Snapshot_new();
String *foo = (String*)SSTR_WRAP_UTF8("foo", 3);
String *bar = (String*)SSTR_WRAP_UTF8("bar", 3);
Snapshot_Add_Entry(snapshot, foo);
Snapshot_Add_Entry(snapshot, foo); // redundant
Vector *entries = Snapshot_List(snapshot);
TEST_INT_EQ(runner, Snapshot_Num_Entries(snapshot), 1,
"One entry added");
TEST_TRUE(runner, Str_Equals(foo, Vec_Fetch(entries, 0)), "correct entry");
DECREF(entries);
Snapshot_Add_Entry(snapshot, bar);
TEST_INT_EQ(runner, Snapshot_Num_Entries(snapshot), 2,
"second entry added");
Snapshot_Delete_Entry(snapshot, foo);
TEST_INT_EQ(runner, Snapshot_Num_Entries(snapshot), 1, "Delete_Entry");
DECREF(snapshot);
}
void
SegWriter_Finish_IMP(SegWriter *self) {
SegWriterIVARS *const ivars = SegWriter_IVARS(self);
String *seg_name = Seg_Get_Name(ivars->segment);
// Finish off children.
for (size_t i = 0, max = Vec_Get_Size(ivars->writers); i < max; i++) {
DataWriter *writer = (DataWriter*)Vec_Fetch(ivars->writers, i);
DataWriter_Finish(writer);
}
// Write segment metadata and add the segment directory to the snapshot.
Snapshot *snapshot = SegWriter_Get_Snapshot(self);
String *segmeta_filename = Str_newf("%o/segmeta.json", seg_name);
Seg_Write_File(ivars->segment, ivars->folder);
Snapshot_Add_Entry(snapshot, seg_name);
DECREF(segmeta_filename);
// Collapse segment files into compound file.
Folder_Consolidate(ivars->folder, seg_name);
}
void
BGMerger_Prepare_Commit_IMP(BackgroundMerger *self) {
BackgroundMergerIVARS *const ivars = BGMerger_IVARS(self);
Vector *seg_readers = PolyReader_Get_Seg_Readers(ivars->polyreader);
uint32_t num_seg_readers = Vec_Get_Size(seg_readers);
uint32_t segs_merged = 0;
if (ivars->prepared) {
THROW(ERR, "Can't call Prepare_Commit() more than once");
}
// Maybe merge existing index data.
if (num_seg_readers) {
segs_merged = S_maybe_merge(self);
}
if (!segs_merged) {
// Nothing merged. Leave `needs_commit` false and bail out.
ivars->prepared = true;
return;
}
// Finish the segment and write a new snapshot file.
else {
Folder *folder = ivars->folder;
Snapshot *snapshot = ivars->snapshot;
// Write out new deletions.
if (DelWriter_Updated(ivars->del_writer)) {
// Only write out if they haven't all been applied.
if (segs_merged != num_seg_readers) {
DelWriter_Finish(ivars->del_writer);
}
}
// Finish the segment.
SegWriter_Finish(ivars->seg_writer);
// Grab the write lock.
S_obtain_write_lock(self);
if (!ivars->write_lock) {
RETHROW(INCREF(Err_get_error()));
}
// Write temporary snapshot file.
DECREF(ivars->snapfile);
String *snapfile = IxManager_Make_Snapshot_Filename(ivars->manager);
ivars->snapfile = Str_Cat_Trusted_Utf8(snapfile, ".temp", 5);
DECREF(snapfile);
Folder_Delete(folder, ivars->snapfile);
Snapshot_Write_File(snapshot, folder, ivars->snapfile);
// Determine whether the index has been updated while this background
// merge process was running.
String *start_snapfile
= Snapshot_Get_Path(PolyReader_Get_Snapshot(ivars->polyreader));
Snapshot *latest_snapshot
= Snapshot_Read_File(Snapshot_new(), ivars->folder, NULL);
String *latest_snapfile = Snapshot_Get_Path(latest_snapshot);
bool index_updated
= !Str_Equals(start_snapfile, (Obj*)latest_snapfile);
if (index_updated) {
/* See if new deletions have been applied since this
* background merge process started against any of the
* segments we just merged away. If that's true, we need to
* write another segment which applies the deletions against
* the new composite segment.
*/
S_merge_updated_deletions(self);
// Add the fresh content to our snapshot. (It's important to
// run this AFTER S_merge_updated_deletions, because otherwise
// we couldn't tell whether the deletion counts changed.)
Vector *files = Snapshot_List(latest_snapshot);
for (uint32_t i = 0, max = Vec_Get_Size(files); i < max; i++) {
String *file = (String*)Vec_Fetch(files, i);
if (Str_Starts_With_Utf8(file, "seg_", 4)) {
int64_t gen = (int64_t)IxFileNames_extract_gen(file);
if (gen > ivars->cutoff) {
Snapshot_Add_Entry(ivars->snapshot, file);
}
}
}
DECREF(files);
// Since the snapshot content has changed, we need to rewrite it.
Folder_Delete(folder, ivars->snapfile);
Snapshot_Write_File(snapshot, folder, ivars->snapfile);
}
DECREF(latest_snapshot);
ivars->needs_commit = true;
}
// Close reader, so that we can delete its files if appropriate.
PolyReader_Close(ivars->polyreader);
ivars->prepared = true;
}