static void
test_Read_File_and_Write_File(TestBatchRunner *runner) {
Snapshot *snapshot = Snapshot_new();
Folder *folder = (Folder*)RAMFolder_new(NULL);
String *snap = (String*)SSTR_WRAP_UTF8("snap", 4);
String *foo = (String*)SSTR_WRAP_UTF8("foo", 3);
Snapshot_Add_Entry(snapshot, foo);
Snapshot_Write_File(snapshot, folder, snap);
Snapshot *dupe = Snapshot_new();
Snapshot *read_retval = Snapshot_Read_File(dupe, folder, snap);
TEST_TRUE(runner, dupe == read_retval, "Read_File() returns the object");
Vector *orig_list = Snapshot_List(snapshot);
Vector *dupe_list = Snapshot_List(dupe);
TEST_TRUE(runner, Vec_Equals(orig_list, (Obj*)dupe_list),
"Round trip through Write_File() and Read_File()");
DECREF(orig_list);
DECREF(dupe_list);
DECREF(dupe);
DECREF(snapshot);
DECREF(folder);
}
Segment*
IxManager_make_new_segment(IndexManager *self, Snapshot *snapshot)
{
VArray *files = Snapshot_List(snapshot);
u32_t i, max;
i32_t highest_seg_num = 0;
CharBuf *seg_name = CB_new(20);
Segment *segment;
/* Find highest seg num. */
for (i = 0, max = VA_Get_Size(files); i < max; i++) {
CharBuf *file = (CharBuf*)VA_Fetch(files, i);
if (CB_Starts_With_Str(file, "seg_", 4)) {
i32_t seg_num = IxFileNames_extract_gen(file);
if (seg_num > highest_seg_num) {
highest_seg_num = seg_num;
}
}
}
/* Create segment with num one greater than current max. */
S_cat_seg_name(seg_name, highest_seg_num + 1);
segment = Seg_new(seg_name, self->folder);
DECREF(seg_name);
DECREF(files);
return segment;
}
int64_t
IxManager_Highest_Seg_Num_IMP(IndexManager *self, Snapshot *snapshot) {
Vector *files = Snapshot_List(snapshot);
uint64_t highest_seg_num = 0;
UNUSED_VAR(self);
for (size_t i = 0, max = Vec_Get_Size(files); i < max; i++) {
String *file = (String*)Vec_Fetch(files, i);
if (Seg_valid_seg_name(file)) {
uint64_t seg_num = IxFileNames_extract_gen(file);
if (seg_num > highest_seg_num) { highest_seg_num = seg_num; }
}
}
DECREF(files);
return (int64_t)highest_seg_num;
}
static String*
S_find_schema_file(Snapshot *snapshot) {
Vector *files = Snapshot_List(snapshot);
String *retval = NULL;
for (size_t i = 0, max = Vec_Get_Size(files); i < max; i++) {
String *file = (String*)Vec_Fetch(files, i);
if (Str_Starts_With_Utf8(file, "schema_", 7)
&& Str_Ends_With_Utf8(file, ".json", 5)
) {
retval = file;
break;
}
}
DECREF(files);
return retval;
}
int64_t
IxManager_highest_seg_num(IndexManager *self, Snapshot *snapshot)
{
VArray *files = Snapshot_List(snapshot);
uint32_t i, max;
uint64_t highest_seg_num = 0;
UNUSED_VAR(self);
for (i = 0, max = VA_Get_Size(files); i < max; i++) {
CharBuf *file = (CharBuf*)VA_Fetch(files, i);
if (Seg_valid_seg_name(file)) {
uint64_t seg_num = IxFileNames_extract_gen(file);
if (seg_num > highest_seg_num) { highest_seg_num = seg_num; }
}
}
DECREF(files);
return (int64_t)highest_seg_num;
}
void
LexWriter_delete_segment(LexiconWriter *self, SegReader *reader)
{
Snapshot *snapshot = LexWriter_Get_Snapshot(self);
CharBuf *merged_seg_name = Seg_Get_Name(SegReader_Get_Segment(reader));
CharBuf *pattern = CB_newf("%o/lexicon", merged_seg_name);
VArray *files = Snapshot_List(snapshot);
VArray *my_old_files = VA_Grep(files, S_my_file, pattern);
u32_t i, max;
for (i = 0, max = VA_Get_Size(my_old_files); i < max; i++) {
CharBuf *entry = (CharBuf*)VA_Fetch(my_old_files, i);
Snapshot_Delete_Entry(snapshot, entry);
}
DECREF(my_old_files);
DECREF(files);
DECREF(pattern);
}
static CharBuf*
S_find_schema_file(Snapshot *snapshot)
{
VArray *files = Snapshot_List(snapshot);
uint32_t i, max;
CharBuf *retval = NULL;
for (i = 0, max = VA_Get_Size(files); i < max; i++) {
CharBuf *file = (CharBuf*)VA_Fetch(files, i);
if ( CB_Starts_With_Str(file, "schema_", 7)
&& CB_Ends_With_Str(file, ".json", 5)
) {
retval = file;
break;
}
}
DECREF(files);
return retval;
}
void
Snapshot_Write_File_IMP(Snapshot *self, Folder *folder, String *path) {
SnapshotIVARS *const ivars = Snapshot_IVARS(self);
Hash *all_data = Hash_new(0);
Vector *list = Snapshot_List(self);
// Update path.
DECREF(ivars->path);
if (path != NULL && Str_Get_Size(path) != 0) {
ivars->path = Str_Clone(path);
}
else {
String *latest = IxFileNames_latest_snapshot(folder);
uint64_t gen = latest ? IxFileNames_extract_gen(latest) + 1 : 1;
char base36[StrHelp_MAX_BASE36_BYTES];
StrHelp_to_base36(gen, &base36);
ivars->path = Str_newf("snapshot_%s.json", &base36);
DECREF(latest);
}
// Don't overwrite.
if (Folder_Exists(folder, ivars->path)) {
THROW(ERR, "Snapshot file '%o' already exists", ivars->path);
}
// Sort, then store file names.
Vec_Sort(list);
Hash_Store_Utf8(all_data, "entries", 7, (Obj*)list);
// Create a JSON-izable data structure.
Hash_Store_Utf8(all_data, "format", 6,
(Obj*)Str_newf("%i32", (int32_t)Snapshot_current_file_format));
Hash_Store_Utf8(all_data, "subformat", 9,
(Obj*)Str_newf("%i32", (int32_t)Snapshot_current_file_subformat));
// Write out JSON-ized data to the new file.
Json_spew_json((Obj*)all_data, folder, ivars->path);
DECREF(all_data);
}
void
Snapshot_write_file(Snapshot *self, Folder *folder, const CharBuf *path)
{
Hash *all_data = Hash_new(0);
VArray *list = Snapshot_List(self);
// Update path.
DECREF(self->path);
if (path) {
self->path = CB_Clone(path);
}
else {
CharBuf *latest = IxFileNames_latest_snapshot(folder);
uint64_t gen = latest ? IxFileNames_extract_gen(latest) + 1 : 1;
char base36[StrHelp_MAX_BASE36_BYTES];
StrHelp_to_base36(gen, &base36);
self->path = CB_newf("snapshot_%s.json", &base36);
DECREF(latest);
}
// Don't overwrite.
if (Folder_Exists(folder, self->path)) {
THROW(ERR, "Snapshot file '%o' already exists", self->path);
}
// Sort, then store file names.
VA_Sort(list, NULL, NULL);
Hash_Store_Str(all_data, "entries", 7, (Obj*)list);
// Create a JSON-izable data structure.
Hash_Store_Str(all_data, "format", 6,
(Obj*)CB_newf("%i32", (int32_t)Snapshot_current_file_format) );
Hash_Store_Str(all_data, "subformat", 9,
(Obj*)CB_newf("%i32", (int32_t)Snapshot_current_file_subformat) );
// Write out JSON-ized data to the new file.
Json_spew_json((Obj*)all_data, folder, self->path);
DECREF(all_data);
}
static void
test_Add_and_Delete(TestBatchRunner *runner) {
Snapshot *snapshot = Snapshot_new();
String *foo = (String*)SSTR_WRAP_UTF8("foo", 3);
String *bar = (String*)SSTR_WRAP_UTF8("bar", 3);
Snapshot_Add_Entry(snapshot, foo);
Snapshot_Add_Entry(snapshot, foo); // redundant
Vector *entries = Snapshot_List(snapshot);
TEST_INT_EQ(runner, Snapshot_Num_Entries(snapshot), 1,
"One entry added");
TEST_TRUE(runner, Str_Equals(foo, Vec_Fetch(entries, 0)), "correct entry");
DECREF(entries);
Snapshot_Add_Entry(snapshot, bar);
TEST_INT_EQ(runner, Snapshot_Num_Entries(snapshot), 2,
"second entry added");
Snapshot_Delete_Entry(snapshot, foo);
TEST_INT_EQ(runner, Snapshot_Num_Entries(snapshot), 1, "Delete_Entry");
DECREF(snapshot);
}
void
Snapshot_write_file(Snapshot *self, Folder *folder, const CharBuf *filename)
{
Hash *all_data = Hash_new(0);
VArray *list = Snapshot_List(self);
/* Update filename. */
DECREF(self->filename);
if (filename) {
self->filename = CB_Clone(filename);
}
else {
CharBuf *latest = IxFileNames_latest_snapshot(folder);
i32_t gen = latest ? IxFileNames_extract_gen(latest) + 1 : 1;
CharBuf *base_36 = StrHelp_to_base36(gen);
self->filename = CB_newf("snapshot_%o.json", base_36);
DECREF(latest);
DECREF(base_36);
}
/* Don't overwrite. */
if (Folder_Exists(folder, self->filename)) {
THROW("Snapshot file '%o' already exists", self->filename);
}
/* Sort, then store file names. */
VA_Sort(list, NULL);
Hash_Store_Str(all_data, "entries", 7, (Obj*)list);
/* Create a JSON-izable data structure. */
Hash_Store_Str(all_data, "format", 6,
(Obj*)CB_newf("%i32", (i32_t)Snapshot_current_file_format) );
/* Write out JSON-ized data to the new file. */
Json_spew_json((Obj*)all_data, folder, self->filename);
DECREF(all_data);
}
void
S_try_open_elements(void *context) {
struct try_open_elements_context *args
= (struct try_open_elements_context*)context;
PolyReader *self = args->self;
PolyReaderIVARS *const ivars = PolyReader_IVARS(self);
VArray *files = Snapshot_List(ivars->snapshot);
Folder *folder = PolyReader_Get_Folder(self);
uint32_t num_segs = 0;
uint64_t latest_schema_gen = 0;
CharBuf *schema_file = NULL;
// Find schema file, count segments.
for (uint32_t i = 0, max = VA_Get_Size(files); i < max; i++) {
CharBuf *entry = (CharBuf*)VA_Fetch(files, i);
if (Seg_valid_seg_name(entry)) {
num_segs++;
}
else if (CB_Starts_With_Str(entry, "schema_", 7)
&& CB_Ends_With_Str(entry, ".json", 5)
) {
uint64_t gen = IxFileNames_extract_gen(entry);
if (gen > latest_schema_gen) {
latest_schema_gen = gen;
if (!schema_file) { schema_file = CB_Clone(entry); }
else { CB_Mimic(schema_file, (Obj*)entry); }
}
}
}
// Read Schema.
if (!schema_file) {
DECREF(files);
THROW(ERR, "Can't find a schema file.");
}
else {
Hash *dump = (Hash*)Json_slurp_json(folder, schema_file);
if (dump) { // read file successfully
DECREF(ivars->schema);
ivars->schema = (Schema*)CERTIFY(
VTable_Load_Obj(SCHEMA, (Obj*)dump), SCHEMA);
DECREF(dump);
DECREF(schema_file);
schema_file = NULL;
}
else {
CharBuf *mess = MAKE_MESS("Failed to parse %o", schema_file);
DECREF(schema_file);
DECREF(files);
Err_throw_mess(ERR, mess);
}
}
VArray *segments = VA_new(num_segs);
for (uint32_t i = 0, max = VA_Get_Size(files); i < max; i++) {
CharBuf *entry = (CharBuf*)VA_Fetch(files, i);
// Create a Segment for each segmeta.
if (Seg_valid_seg_name(entry)) {
int64_t seg_num = IxFileNames_extract_gen(entry);
Segment *segment = Seg_new(seg_num);
// Bail if reading the file fails (probably because it's been
// deleted and a new snapshot file has been written so we need to
// retry).
if (Seg_Read_File(segment, folder)) {
VA_Push(segments, (Obj*)segment);
}
else {
CharBuf *mess = MAKE_MESS("Failed to read %o", entry);
DECREF(segment);
DECREF(segments);
DECREF(files);
Err_throw_mess(ERR, mess);
}
}
}
// Sort the segments by age.
VA_Sort(segments, NULL, NULL);
// Open individual SegReaders.
struct try_open_segreader_context seg_context;
seg_context.schema = PolyReader_Get_Schema(self);
seg_context.folder = folder;
seg_context.snapshot = PolyReader_Get_Snapshot(self);
seg_context.segments = segments;
seg_context.result = NULL;
args->seg_readers = VA_new(num_segs);
Err *error = NULL;
for (uint32_t seg_tick = 0; seg_tick < num_segs; seg_tick++) {
seg_context.seg_tick = seg_tick;
error = Err_trap(S_try_open_segreader, &seg_context);
if (error) {
break;
}
VA_Push(args->seg_readers, (Obj*)seg_context.result);
seg_context.result = NULL;
}
DECREF(segments);
DECREF(files);
if (error) {
DECREF(args->seg_readers);
args->seg_readers = NULL;
RETHROW(error);
}
}
void
FilePurger_purge(FilePurger *self)
{
Lock *deletion_lock = IxManager_Make_Deletion_Lock(self->manager);
// Obtain deletion lock, purge files, release deletion lock.
Lock_Clear_Stale(deletion_lock);
if (Lock_Obtain(deletion_lock)) {
Folder *folder = self->folder;
Hash *failures = Hash_new(0);
VArray *purgables;
VArray *snapshots;
S_discover_unused(self, &purgables, &snapshots);
// Attempt to delete entries -- if failure, no big deal, just try
// again later. Proceed in reverse lexical order so that directories
// get deleted after they've been emptied.
VA_Sort(purgables, NULL, NULL);
for (uint32_t i = VA_Get_Size(purgables); i--; ) {
CharBuf *entry = (CharBuf*)VA_fetch(purgables, i);
if (Hash_Fetch(self->disallowed, (Obj*)entry)) { continue; }
if (!Folder_Delete(folder, entry)) {
if (Folder_Exists(folder, entry)) {
Hash_Store(failures, (Obj*)entry, INCREF(&EMPTY));
}
}
}
for (uint32_t i = 0, max = VA_Get_Size(snapshots); i < max; i++) {
Snapshot *snapshot = (Snapshot*)VA_Fetch(snapshots, i);
bool_t snapshot_has_failures = false;
if (Hash_Get_Size(failures)) {
// Only delete snapshot files if all of their entries were
// successfully deleted.
VArray *entries = Snapshot_List(snapshot);
for (uint32_t j = VA_Get_Size(entries); j--; ) {
CharBuf *entry = (CharBuf*)VA_Fetch(entries, j);
if (Hash_Fetch(failures, (Obj*)entry)) {
snapshot_has_failures = true;
break;
}
}
DECREF(entries);
}
if (!snapshot_has_failures) {
CharBuf *snapfile = Snapshot_Get_Path(snapshot);
Folder_Delete(folder, snapfile);
}
}
DECREF(failures);
DECREF(purgables);
DECREF(snapshots);
Lock_Release(deletion_lock);
}
else {
WARN("Can't obtain deletion lock, skipping deletion of "
"obsolete files");
}
DECREF(deletion_lock);
}
static void
S_discover_unused(FilePurger *self, VArray **purgables_ptr,
VArray **snapshots_ptr)
{
Folder *folder = self->folder;
DirHandle *dh = Folder_Open_Dir(folder, NULL);
if (!dh) { RETHROW(INCREF(Err_get_error())); }
VArray *spared = VA_new(1);
VArray *snapshots = VA_new(1);
CharBuf *snapfile = NULL;
// Start off with the list of files in the current snapshot.
if (self->snapshot) {
VArray *entries = Snapshot_List(self->snapshot);
VArray *referenced = S_find_all_referenced(folder, entries);
VA_Push_VArray(spared, referenced);
DECREF(entries);
DECREF(referenced);
snapfile = Snapshot_Get_Path(self->snapshot);
if (snapfile) { VA_Push(spared, INCREF(snapfile)); }
}
CharBuf *entry = DH_Get_Entry(dh);
Hash *candidates = Hash_new(64);
while (DH_Next(dh)) {
if (!CB_Starts_With_Str(entry, "snapshot_", 9)) { continue; }
else if (!CB_Ends_With_Str(entry, ".json", 5)) { continue; }
else if (snapfile && CB_Equals(entry, (Obj*)snapfile)) { continue; }
else {
Snapshot *snapshot
= Snapshot_Read_File(Snapshot_new(), folder, entry);
Lock *lock
= IxManager_Make_Snapshot_Read_Lock(self->manager, entry);
VArray *snap_list = Snapshot_List(snapshot);
VArray *referenced = S_find_all_referenced(folder, snap_list);
// DON'T obtain the lock -- only see whether another
// entity holds a lock on the snapshot file.
if (lock) {
Lock_Clear_Stale(lock);
}
if (lock && Lock_Is_Locked(lock)) {
// The snapshot file is locked, which means someone's using
// that version of the index -- protect all of its entries.
uint32_t new_size = VA_Get_Size(spared)
+ VA_Get_Size(referenced) + 1;
VA_Grow(spared, new_size);
VA_Push(spared, (Obj*)CB_Clone(entry));
VA_Push_VArray(spared, referenced);
}
else {
// No one's using this snapshot, so all of its entries are
// candidates for deletion.
for (uint32_t i = 0, max = VA_Get_Size(referenced); i < max; i++) {
CharBuf *file = (CharBuf*)VA_Fetch(referenced, i);
Hash_Store(candidates, (Obj*)file, INCREF(&EMPTY));
}
VA_Push(snapshots, INCREF(snapshot));
}
DECREF(referenced);
DECREF(snap_list);
DECREF(snapshot);
DECREF(lock);
}
}
DECREF(dh);
// Clean up after a dead segment consolidation.
S_zap_dead_merge(self, candidates);
// Eliminate any current files from the list of files to be purged.
for (uint32_t i = 0, max = VA_Get_Size(spared); i < max; i++) {
CharBuf *filename = (CharBuf*)VA_Fetch(spared, i);
DECREF(Hash_Delete(candidates, (Obj*)filename));
}
// Pass back purgables and Snapshots.
*purgables_ptr = Hash_Keys(candidates);
*snapshots_ptr = snapshots;
DECREF(candidates);
DECREF(spared);
}
void
BGMerger_Prepare_Commit_IMP(BackgroundMerger *self) {
BackgroundMergerIVARS *const ivars = BGMerger_IVARS(self);
Vector *seg_readers = PolyReader_Get_Seg_Readers(ivars->polyreader);
uint32_t num_seg_readers = Vec_Get_Size(seg_readers);
uint32_t segs_merged = 0;
if (ivars->prepared) {
THROW(ERR, "Can't call Prepare_Commit() more than once");
}
// Maybe merge existing index data.
if (num_seg_readers) {
segs_merged = S_maybe_merge(self);
}
if (!segs_merged) {
// Nothing merged. Leave `needs_commit` false and bail out.
ivars->prepared = true;
return;
}
// Finish the segment and write a new snapshot file.
else {
Folder *folder = ivars->folder;
Snapshot *snapshot = ivars->snapshot;
// Write out new deletions.
if (DelWriter_Updated(ivars->del_writer)) {
// Only write out if they haven't all been applied.
if (segs_merged != num_seg_readers) {
DelWriter_Finish(ivars->del_writer);
}
}
// Finish the segment.
SegWriter_Finish(ivars->seg_writer);
// Grab the write lock.
S_obtain_write_lock(self);
if (!ivars->write_lock) {
RETHROW(INCREF(Err_get_error()));
}
// Write temporary snapshot file.
DECREF(ivars->snapfile);
String *snapfile = IxManager_Make_Snapshot_Filename(ivars->manager);
ivars->snapfile = Str_Cat_Trusted_Utf8(snapfile, ".temp", 5);
DECREF(snapfile);
Folder_Delete(folder, ivars->snapfile);
Snapshot_Write_File(snapshot, folder, ivars->snapfile);
// Determine whether the index has been updated while this background
// merge process was running.
String *start_snapfile
= Snapshot_Get_Path(PolyReader_Get_Snapshot(ivars->polyreader));
Snapshot *latest_snapshot
= Snapshot_Read_File(Snapshot_new(), ivars->folder, NULL);
String *latest_snapfile = Snapshot_Get_Path(latest_snapshot);
bool index_updated
= !Str_Equals(start_snapfile, (Obj*)latest_snapfile);
if (index_updated) {
/* See if new deletions have been applied since this
* background merge process started against any of the
* segments we just merged away. If that's true, we need to
* write another segment which applies the deletions against
* the new composite segment.
*/
S_merge_updated_deletions(self);
// Add the fresh content to our snapshot. (It's important to
// run this AFTER S_merge_updated_deletions, because otherwise
// we couldn't tell whether the deletion counts changed.)
Vector *files = Snapshot_List(latest_snapshot);
for (uint32_t i = 0, max = Vec_Get_Size(files); i < max; i++) {
String *file = (String*)Vec_Fetch(files, i);
if (Str_Starts_With_Utf8(file, "seg_", 4)) {
int64_t gen = (int64_t)IxFileNames_extract_gen(file);
if (gen > ivars->cutoff) {
Snapshot_Add_Entry(ivars->snapshot, file);
}
}
}
DECREF(files);
// Since the snapshot content has changed, we need to rewrite it.
Folder_Delete(folder, ivars->snapfile);
Snapshot_Write_File(snapshot, folder, ivars->snapfile);
}
DECREF(latest_snapshot);
ivars->needs_commit = true;
}
// Close reader, so that we can delete its files if appropriate.
PolyReader_Close(ivars->polyreader);
ivars->prepared = true;
}
