String* IxFileNames_latest_snapshot(Folder *folder) { DirHandle *dh = Folder_Open_Dir(folder, NULL); String *retval = NULL; uint64_t latest_gen = 0; if (!dh) { RETHROW(INCREF(Err_get_error())); } while (DH_Next(dh)) { String *entry = DH_Get_Entry(dh); if (Str_Starts_With_Utf8(entry, "snapshot_", 9) && Str_Ends_With_Utf8(entry, ".json", 5) ) { uint64_t gen = IxFileNames_extract_gen(entry); if (gen > latest_gen) { latest_gen = gen; DECREF(retval); retval = Str_Clone(entry); } } DECREF(entry); } DECREF(dh); return retval; }
static void test_Local_Open_Dir(TestBatchRunner *runner) { Folder *real_folder = S_folder_with_contents(); CompoundFileReader *cf_reader = CFReader_open(real_folder); bool saw_foo = false; bool saw_stuff = false; bool stuff_was_dir = false; CFReader_MkDir(cf_reader, stuff); DirHandle *dh = CFReader_Local_Open_Dir(cf_reader); while (DH_Next(dh)) { String *entry = DH_Get_Entry(dh); if (Str_Equals(entry, (Obj*)foo)) { saw_foo = true; } else if (Str_Equals(entry, (Obj*)stuff)) { saw_stuff = true; stuff_was_dir = DH_Entry_Is_Dir(dh); } DECREF(entry); } TEST_TRUE(runner, saw_foo, "DirHandle iterated over virtual file"); TEST_TRUE(runner, saw_stuff, "DirHandle iterated over real directory"); TEST_TRUE(runner, stuff_was_dir, "DirHandle knew that real entry was dir"); DECREF(dh); DECREF(cf_reader); DECREF(real_folder); }
String* IxManager_Make_Snapshot_Filename_IMP(IndexManager *self) { IndexManagerIVARS *const ivars = IxManager_IVARS(self); Folder *folder = (Folder*)CERTIFY(ivars->folder, FOLDER); DirHandle *dh = Folder_Open_Dir(folder, NULL); uint64_t max_gen = 0; if (!dh) { RETHROW(INCREF(Err_get_error())); } while (DH_Next(dh)) { String *entry = DH_Get_Entry(dh); if (Str_Starts_With_Utf8(entry, "snapshot_", 9) && Str_Ends_With_Utf8(entry, ".json", 5) ) { uint64_t gen = IxFileNames_extract_gen(entry); if (gen > max_gen) { max_gen = gen; } } DECREF(entry); } DECREF(dh); uint64_t new_gen = max_gen + 1; char base36[StrHelp_MAX_BASE36_BYTES]; StrHelp_to_base36(new_gen, &base36); return Str_newf("snapshot_%s.json", &base36); }
CharBuf* IxManager_make_snapshot_filename(IndexManager *self) { Folder *folder = (Folder*)CERTIFY(self->folder, FOLDER); DirHandle *dh = Folder_Open_Dir(folder, NULL); CharBuf *entry; uint64_t max_gen = 0; if (!dh) { RETHROW(INCREF(Err_get_error())); } entry = DH_Get_Entry(dh); while (DH_Next(dh)) { if ( CB_Starts_With_Str(entry, "snapshot_", 9) && CB_Ends_With_Str(entry, ".json", 5) ) { uint64_t gen = IxFileNames_extract_gen(entry); if (gen > max_gen) { max_gen = gen; } } } DECREF(dh); { uint64_t new_gen = max_gen + 1; char base36[StrHelp_MAX_BASE36_BYTES]; StrHelp_to_base36(new_gen, &base36); return CB_newf("snapshot_%s.json", &base36); } }
bool_t Folder_delete_tree(Folder *self, const CharBuf *path) { Folder *enclosing_folder = Folder_Enclosing_Folder(self, path); // Don't allow Folder to delete itself. if (!path || !CB_Get_Size(path)) { return false; } if (enclosing_folder) { ZombieCharBuf *local = IxFileNames_local_part(path, ZCB_BLANK()); if (Folder_Local_Is_Directory(enclosing_folder, (CharBuf*)local)) { Folder *inner_folder = Folder_Local_Find_Folder(enclosing_folder, (CharBuf*)local); DirHandle *dh = Folder_Local_Open_Dir(inner_folder); if (dh) { VArray *files = VA_new(20); VArray *dirs = VA_new(20); CharBuf *entry = DH_Get_Entry(dh); while (DH_Next(dh)) { VA_Push(files, (Obj*)CB_Clone(entry)); if (DH_Entry_Is_Dir(dh) && !DH_Entry_Is_Symlink(dh)) { VA_Push(dirs, (Obj*)CB_Clone(entry)); } } for (uint32_t i = 0, max = VA_Get_Size(dirs); i < max; i++) { CharBuf *name = (CharBuf*)VA_Fetch(files, i); bool_t success = Folder_Delete_Tree(inner_folder, name); if (!success && Folder_Local_Exists(inner_folder, name)) { break; } } for (uint32_t i = 0, max = VA_Get_Size(files); i < max; i++) { CharBuf *name = (CharBuf*)VA_Fetch(files, i); bool_t success = Folder_Local_Delete(inner_folder, name); if (!success && Folder_Local_Exists(inner_folder, name)) { break; } } DECREF(dirs); DECREF(files); DECREF(dh); } } return Folder_Local_Delete(enclosing_folder, (CharBuf*)local); } else { // Return failure if the entry wasn't there in the first place. return false; } }
CFReaderDirHandle* CFReaderDH_init(CFReaderDirHandle *self, CompoundFileReader *cf_reader) { DH_init((DirHandle*)self, CFReader_Get_Path(cf_reader)); self->cf_reader = (CompoundFileReader*)INCREF(cf_reader); self->elems = Hash_Keys(self->cf_reader->records); self->tick = -1; // Accumulate entries from real Folder. DirHandle *dh = Folder_Local_Open_Dir(self->cf_reader->real_folder); CharBuf *entry = DH_Get_Entry(dh); while (DH_Next(dh)) { VA_Push(self->elems, (Obj*)CB_Clone(entry)); } DECREF(dh); return self; }
VArray* Folder_list(Folder *self, const CharBuf *path) { Folder *local_folder = Folder_Find_Folder(self, path); VArray *list = NULL; DirHandle *dh = Folder_Local_Open_Dir(local_folder); if (dh) { CharBuf *entry = DH_Get_Entry(dh); list = VA_new(32); while (DH_Next(dh)) { VA_Push(list, (Obj*)CB_Clone(entry)); } DECREF(dh); } else { ERR_ADD_FRAME(Err_get_error()); } return list; }
static void S_zap_dead_merge(FilePurger *self, Hash *candidates) { IndexManager *manager = self->manager; Lock *merge_lock = IxManager_Make_Merge_Lock(manager); Lock_Clear_Stale(merge_lock); if (!Lock_Is_Locked(merge_lock)) { Hash *merge_data = IxManager_Read_Merge_Data(manager); Obj *cutoff = merge_data ? Hash_Fetch_Str(merge_data, "cutoff", 6) : NULL; if (cutoff) { CharBuf *cutoff_seg = Seg_num_to_name(Obj_To_I64(cutoff)); if (Folder_Exists(self->folder, cutoff_seg)) { ZombieCharBuf *merge_json = ZCB_WRAP_STR("merge.json", 10); DirHandle *dh = Folder_Open_Dir(self->folder, cutoff_seg); CharBuf *entry = dh ? DH_Get_Entry(dh) : NULL; CharBuf *filepath = CB_new(32); if (!dh) { THROW(ERR, "Can't open segment dir '%o'", filepath); } Hash_Store(candidates, (Obj*)cutoff_seg, INCREF(&EMPTY)); Hash_Store(candidates, (Obj*)merge_json, INCREF(&EMPTY)); while (DH_Next(dh)) { // TODO: recursively delete subdirs within seg dir. CB_setf(filepath, "%o/%o", cutoff_seg, entry); Hash_Store(candidates, (Obj*)filepath, INCREF(&EMPTY)); } DECREF(filepath); DECREF(dh); } DECREF(cutoff_seg); } DECREF(merge_data); } DECREF(merge_lock); return; }
static void S_zap_dead_merge(FilePurger *self, Hash *candidates) { FilePurgerIVARS *const ivars = FilePurger_IVARS(self); IndexManager *manager = ivars->manager; Lock *merge_lock = IxManager_Make_Merge_Lock(manager); Lock_Clear_Stale(merge_lock); if (!Lock_Is_Locked(merge_lock)) { Hash *merge_data = IxManager_Read_Merge_Data(manager); Obj *cutoff = merge_data ? Hash_Fetch_Utf8(merge_data, "cutoff", 6) : NULL; if (cutoff) { String *cutoff_seg = Seg_num_to_name(Json_obj_to_i64(cutoff)); if (Folder_Exists(ivars->folder, cutoff_seg)) { String *merge_json = SSTR_WRAP_UTF8("merge.json", 10); DirHandle *dh = Folder_Open_Dir(ivars->folder, cutoff_seg); if (!dh) { THROW(ERR, "Can't open segment dir '%o'", cutoff_seg); } Hash_Store(candidates, cutoff_seg, (Obj*)CFISH_TRUE); Hash_Store(candidates, merge_json, (Obj*)CFISH_TRUE); while (DH_Next(dh)) { // TODO: recursively delete subdirs within seg dir. String *entry = DH_Get_Entry(dh); String *filepath = Str_newf("%o/%o", cutoff_seg, entry); Hash_Store(candidates, filepath, (Obj*)CFISH_TRUE); DECREF(filepath); DECREF(entry); } DECREF(dh); } DECREF(cutoff_seg); } DECREF(merge_data); } DECREF(merge_lock); return; }
CFReaderDirHandle* CFReaderDH_init(CFReaderDirHandle *self, CompoundFileReader *cf_reader) { DH_init((DirHandle*)self, CFReader_Get_Path(cf_reader)); CFReaderDirHandleIVARS *const ivars = CFReaderDH_IVARS(self); ivars->cf_reader = (CompoundFileReader*)INCREF(cf_reader); Hash *cf_records = CFReader_IVARS(ivars->cf_reader)->records; ivars->elems = Hash_Keys(cf_records); ivars->tick = -1; // Accumulate entries from real Folder. Folder *real_folder = CFReader_Get_Real_Folder(ivars->cf_reader); DirHandle *dh = Folder_Local_Open_Dir(real_folder); while (DH_Next(dh)) { String *entry = DH_Get_Entry(dh); Vec_Push(ivars->elems, (Obj*)Str_Clone(entry)); DECREF(entry); } DECREF(dh); return self; }
static void S_add_to_file_list(Folder *self, VArray *list, CharBuf *dir, CharBuf *prefix) { size_t orig_prefix_size = CB_Get_Size(prefix); DirHandle *dh = Folder_Open_Dir(self, dir); CharBuf *entry; if (!dh) { RETHROW(INCREF(Err_get_error())); } entry = DH_Get_Entry(dh); while (DH_Next(dh)) { // Updates entry if (!S_is_updir(entry)) { CharBuf *relpath = CB_newf("%o%o", prefix, entry); if (VA_Get_Size(list) == VA_Get_Capacity(list)) { VA_Grow(list, VA_Get_Size(list) * 2); } VA_Push(list, (Obj*)relpath); if (DH_Entry_Is_Dir(dh) && !DH_Entry_Is_Symlink(dh)) { CharBuf *subdir = CB_Get_Size(dir) ? CB_newf("%o/%o", dir, entry) : CB_Clone(entry); CB_catf(prefix, "%o/", entry); S_add_to_file_list(self, list, subdir, prefix); // recurse CB_Set_Size(prefix, orig_prefix_size); DECREF(subdir); } } } if (!DH_Close(dh)) { RETHROW(INCREF(Err_get_error())); } DECREF(dh); }
static void S_discover_unused(FilePurger *self, VArray **purgables_ptr, VArray **snapshots_ptr) { Folder *folder = self->folder; DirHandle *dh = Folder_Open_Dir(folder, NULL); if (!dh) { RETHROW(INCREF(Err_get_error())); } VArray *spared = VA_new(1); VArray *snapshots = VA_new(1); CharBuf *snapfile = NULL; // Start off with the list of files in the current snapshot. if (self->snapshot) { VArray *entries = Snapshot_List(self->snapshot); VArray *referenced = S_find_all_referenced(folder, entries); VA_Push_VArray(spared, referenced); DECREF(entries); DECREF(referenced); snapfile = Snapshot_Get_Path(self->snapshot); if (snapfile) { VA_Push(spared, INCREF(snapfile)); } } CharBuf *entry = DH_Get_Entry(dh); Hash *candidates = Hash_new(64); while (DH_Next(dh)) { if (!CB_Starts_With_Str(entry, "snapshot_", 9)) { continue; } else if (!CB_Ends_With_Str(entry, ".json", 5)) { continue; } else if (snapfile && CB_Equals(entry, (Obj*)snapfile)) { continue; } else { Snapshot *snapshot = Snapshot_Read_File(Snapshot_new(), folder, entry); Lock *lock = IxManager_Make_Snapshot_Read_Lock(self->manager, entry); VArray *snap_list = Snapshot_List(snapshot); VArray *referenced = S_find_all_referenced(folder, snap_list); // DON'T obtain the lock -- only see whether another // entity holds a lock on the snapshot file. if (lock) { Lock_Clear_Stale(lock); } if (lock && Lock_Is_Locked(lock)) { // The snapshot file is locked, which means someone's using // that version of the index -- protect all of its entries. uint32_t new_size = VA_Get_Size(spared) + VA_Get_Size(referenced) + 1; VA_Grow(spared, new_size); VA_Push(spared, (Obj*)CB_Clone(entry)); VA_Push_VArray(spared, referenced); } else { // No one's using this snapshot, so all of its entries are // candidates for deletion. for (uint32_t i = 0, max = VA_Get_Size(referenced); i < max; i++) { CharBuf *file = (CharBuf*)VA_Fetch(referenced, i); Hash_Store(candidates, (Obj*)file, INCREF(&EMPTY)); } VA_Push(snapshots, INCREF(snapshot)); } DECREF(referenced); DECREF(snap_list); DECREF(snapshot); DECREF(lock); } } DECREF(dh); // Clean up after a dead segment consolidation. S_zap_dead_merge(self, candidates); // Eliminate any current files from the list of files to be purged. for (uint32_t i = 0, max = VA_Get_Size(spared); i < max; i++) { CharBuf *filename = (CharBuf*)VA_Fetch(spared, i); DECREF(Hash_Delete(candidates, (Obj*)filename)); } // Pass back purgables and Snapshots. *purgables_ptr = Hash_Keys(candidates); *snapshots_ptr = snapshots; DECREF(candidates); DECREF(spared); }