static void
test_Open_Dir(TestBatch *batch)
{
Folder *folder = (Folder*)RAMFolder_new(NULL);
DirHandle *dh;
Folder_MkDir(folder, &foo);
Folder_MkDir(folder, &foo_bar);
dh = Folder_Open_Dir(folder, &foo);
TEST_TRUE(batch, dh && DH_Is_A(dh, DIRHANDLE), "Open_Dir");
DECREF(dh);
dh = Folder_Open_Dir(folder, &foo_bar);
TEST_TRUE(batch, dh && DH_Is_A(dh, DIRHANDLE), "Open_Dir nested dir");
DECREF(dh);
Err_set_error(NULL);
dh = Folder_Open_Dir(folder, &bar);
TEST_TRUE(batch, dh == NULL,
"Open_Dir on non-existent entry fails");
TEST_TRUE(batch, Err_get_error() != NULL,
"Open_Dir on non-existent entry sets Err_error");
Err_set_error(NULL);
dh = Folder_Open_Dir(folder, &foo_foo);
TEST_TRUE(batch, dh == NULL,
"Open_Dir on non-existent nested entry fails");
TEST_TRUE(batch, Err_get_error() != NULL,
"Open_Dir on non-existent nested entry sets Err_error");
DECREF(folder);
}
String*
IxFileNames_latest_snapshot(Folder *folder) {
DirHandle *dh = Folder_Open_Dir(folder, NULL);
String *retval = NULL;
uint64_t latest_gen = 0;
if (!dh) { RETHROW(INCREF(Err_get_error())); }
while (DH_Next(dh)) {
String *entry = DH_Get_Entry(dh);
if (Str_Starts_With_Utf8(entry, "snapshot_", 9)
&& Str_Ends_With_Utf8(entry, ".json", 5)
) {
uint64_t gen = IxFileNames_extract_gen(entry);
if (gen > latest_gen) {
latest_gen = gen;
DECREF(retval);
retval = Str_Clone(entry);
}
}
DECREF(entry);
}
DECREF(dh);
return retval;
}
String*
IxManager_Make_Snapshot_Filename_IMP(IndexManager *self) {
IndexManagerIVARS *const ivars = IxManager_IVARS(self);
Folder *folder = (Folder*)CERTIFY(ivars->folder, FOLDER);
DirHandle *dh = Folder_Open_Dir(folder, NULL);
uint64_t max_gen = 0;
if (!dh) { RETHROW(INCREF(Err_get_error())); }
while (DH_Next(dh)) {
String *entry = DH_Get_Entry(dh);
if (Str_Starts_With_Utf8(entry, "snapshot_", 9)
&& Str_Ends_With_Utf8(entry, ".json", 5)
) {
uint64_t gen = IxFileNames_extract_gen(entry);
if (gen > max_gen) { max_gen = gen; }
}
DECREF(entry);
}
DECREF(dh);
uint64_t new_gen = max_gen + 1;
char base36[StrHelp_MAX_BASE36_BYTES];
StrHelp_to_base36(new_gen, &base36);
return Str_newf("snapshot_%s.json", &base36);
}
CharBuf*
IxManager_make_snapshot_filename(IndexManager *self)
{
Folder *folder = (Folder*)CERTIFY(self->folder, FOLDER);
DirHandle *dh = Folder_Open_Dir(folder, NULL);
CharBuf *entry;
uint64_t max_gen = 0;
if (!dh) { RETHROW(INCREF(Err_get_error())); }
entry = DH_Get_Entry(dh);
while (DH_Next(dh)) {
if ( CB_Starts_With_Str(entry, "snapshot_", 9)
&& CB_Ends_With_Str(entry, ".json", 5)
) {
uint64_t gen = IxFileNames_extract_gen(entry);
if (gen > max_gen) { max_gen = gen; }
}
}
DECREF(dh);
{
uint64_t new_gen = max_gen + 1;
char base36[StrHelp_MAX_BASE36_BYTES];
StrHelp_to_base36(new_gen, &base36);
return CB_newf("snapshot_%s.json", &base36);
}
}
static void
S_zap_dead_merge(FilePurger *self, Hash *candidates)
{
IndexManager *manager = self->manager;
Lock *merge_lock = IxManager_Make_Merge_Lock(manager);
Lock_Clear_Stale(merge_lock);
if (!Lock_Is_Locked(merge_lock)) {
Hash *merge_data = IxManager_Read_Merge_Data(manager);
Obj *cutoff = merge_data
? Hash_Fetch_Str(merge_data, "cutoff", 6)
: NULL;
if (cutoff) {
CharBuf *cutoff_seg = Seg_num_to_name(Obj_To_I64(cutoff));
if (Folder_Exists(self->folder, cutoff_seg)) {
ZombieCharBuf *merge_json = ZCB_WRAP_STR("merge.json", 10);
DirHandle *dh = Folder_Open_Dir(self->folder, cutoff_seg);
CharBuf *entry = dh ? DH_Get_Entry(dh) : NULL;
CharBuf *filepath = CB_new(32);
if (!dh) {
THROW(ERR, "Can't open segment dir '%o'", filepath);
}
Hash_Store(candidates, (Obj*)cutoff_seg, INCREF(&EMPTY));
Hash_Store(candidates, (Obj*)merge_json, INCREF(&EMPTY));
while (DH_Next(dh)) {
// TODO: recursively delete subdirs within seg dir.
CB_setf(filepath, "%o/%o", cutoff_seg, entry);
Hash_Store(candidates, (Obj*)filepath, INCREF(&EMPTY));
}
DECREF(filepath);
DECREF(dh);
}
DECREF(cutoff_seg);
}
DECREF(merge_data);
}
DECREF(merge_lock);
return;
}
static void
S_zap_dead_merge(FilePurger *self, Hash *candidates) {
FilePurgerIVARS *const ivars = FilePurger_IVARS(self);
IndexManager *manager = ivars->manager;
Lock *merge_lock = IxManager_Make_Merge_Lock(manager);
Lock_Clear_Stale(merge_lock);
if (!Lock_Is_Locked(merge_lock)) {
Hash *merge_data = IxManager_Read_Merge_Data(manager);
Obj *cutoff = merge_data
? Hash_Fetch_Utf8(merge_data, "cutoff", 6)
: NULL;
if (cutoff) {
String *cutoff_seg = Seg_num_to_name(Json_obj_to_i64(cutoff));
if (Folder_Exists(ivars->folder, cutoff_seg)) {
String *merge_json = SSTR_WRAP_UTF8("merge.json", 10);
DirHandle *dh = Folder_Open_Dir(ivars->folder, cutoff_seg);
if (!dh) {
THROW(ERR, "Can't open segment dir '%o'", cutoff_seg);
}
Hash_Store(candidates, cutoff_seg, (Obj*)CFISH_TRUE);
Hash_Store(candidates, merge_json, (Obj*)CFISH_TRUE);
while (DH_Next(dh)) {
// TODO: recursively delete subdirs within seg dir.
String *entry = DH_Get_Entry(dh);
String *filepath = Str_newf("%o/%o", cutoff_seg, entry);
Hash_Store(candidates, filepath, (Obj*)CFISH_TRUE);
DECREF(filepath);
DECREF(entry);
}
DECREF(dh);
}
DECREF(cutoff_seg);
}
DECREF(merge_data);
}
DECREF(merge_lock);
return;
}
static void
S_add_to_file_list(Folder *self, VArray *list, CharBuf *dir, CharBuf *prefix) {
size_t orig_prefix_size = CB_Get_Size(prefix);
DirHandle *dh = Folder_Open_Dir(self, dir);
CharBuf *entry;
if (!dh) {
RETHROW(INCREF(Err_get_error()));
}
entry = DH_Get_Entry(dh);
while (DH_Next(dh)) { // Updates entry
if (!S_is_updir(entry)) {
CharBuf *relpath = CB_newf("%o%o", prefix, entry);
if (VA_Get_Size(list) == VA_Get_Capacity(list)) {
VA_Grow(list, VA_Get_Size(list) * 2);
}
VA_Push(list, (Obj*)relpath);
if (DH_Entry_Is_Dir(dh) && !DH_Entry_Is_Symlink(dh)) {
CharBuf *subdir = CB_Get_Size(dir)
? CB_newf("%o/%o", dir, entry)
: CB_Clone(entry);
CB_catf(prefix, "%o/", entry);
S_add_to_file_list(self, list, subdir, prefix); // recurse
CB_Set_Size(prefix, orig_prefix_size);
DECREF(subdir);
}
}
}
if (!DH_Close(dh)) {
RETHROW(INCREF(Err_get_error()));
}
DECREF(dh);
}
static void
S_discover_unused(FilePurger *self, VArray **purgables_ptr,
VArray **snapshots_ptr)
{
Folder *folder = self->folder;
DirHandle *dh = Folder_Open_Dir(folder, NULL);
if (!dh) { RETHROW(INCREF(Err_get_error())); }
VArray *spared = VA_new(1);
VArray *snapshots = VA_new(1);
CharBuf *snapfile = NULL;
// Start off with the list of files in the current snapshot.
if (self->snapshot) {
VArray *entries = Snapshot_List(self->snapshot);
VArray *referenced = S_find_all_referenced(folder, entries);
VA_Push_VArray(spared, referenced);
DECREF(entries);
DECREF(referenced);
snapfile = Snapshot_Get_Path(self->snapshot);
if (snapfile) { VA_Push(spared, INCREF(snapfile)); }
}
CharBuf *entry = DH_Get_Entry(dh);
Hash *candidates = Hash_new(64);
while (DH_Next(dh)) {
if (!CB_Starts_With_Str(entry, "snapshot_", 9)) { continue; }
else if (!CB_Ends_With_Str(entry, ".json", 5)) { continue; }
else if (snapfile && CB_Equals(entry, (Obj*)snapfile)) { continue; }
else {
Snapshot *snapshot
= Snapshot_Read_File(Snapshot_new(), folder, entry);
Lock *lock
= IxManager_Make_Snapshot_Read_Lock(self->manager, entry);
VArray *snap_list = Snapshot_List(snapshot);
VArray *referenced = S_find_all_referenced(folder, snap_list);
// DON'T obtain the lock -- only see whether another
// entity holds a lock on the snapshot file.
if (lock) {
Lock_Clear_Stale(lock);
}
if (lock && Lock_Is_Locked(lock)) {
// The snapshot file is locked, which means someone's using
// that version of the index -- protect all of its entries.
uint32_t new_size = VA_Get_Size(spared)
+ VA_Get_Size(referenced) + 1;
VA_Grow(spared, new_size);
VA_Push(spared, (Obj*)CB_Clone(entry));
VA_Push_VArray(spared, referenced);
}
else {
// No one's using this snapshot, so all of its entries are
// candidates for deletion.
for (uint32_t i = 0, max = VA_Get_Size(referenced); i < max; i++) {
CharBuf *file = (CharBuf*)VA_Fetch(referenced, i);
Hash_Store(candidates, (Obj*)file, INCREF(&EMPTY));
}
VA_Push(snapshots, INCREF(snapshot));
}
DECREF(referenced);
DECREF(snap_list);
DECREF(snapshot);
DECREF(lock);
}
}
DECREF(dh);
// Clean up after a dead segment consolidation.
S_zap_dead_merge(self, candidates);
// Eliminate any current files from the list of files to be purged.
for (uint32_t i = 0, max = VA_Get_Size(spared); i < max; i++) {
CharBuf *filename = (CharBuf*)VA_Fetch(spared, i);
DECREF(Hash_Delete(candidates, (Obj*)filename));
}
// Pass back purgables and Snapshots.
*purgables_ptr = Hash_Keys(candidates);
*snapshots_ptr = snapshots;
DECREF(candidates);
DECREF(spared);
}