String*
IxFileNames_latest_snapshot(Folder *folder) {
DirHandle *dh = Folder_Open_Dir(folder, NULL);
String *retval = NULL;
uint64_t latest_gen = 0;
if (!dh) { RETHROW(INCREF(Err_get_error())); }
while (DH_Next(dh)) {
String *entry = DH_Get_Entry(dh);
if (Str_Starts_With_Utf8(entry, "snapshot_", 9)
&& Str_Ends_With_Utf8(entry, ".json", 5)
) {
uint64_t gen = IxFileNames_extract_gen(entry);
if (gen > latest_gen) {
latest_gen = gen;
DECREF(retval);
retval = Str_Clone(entry);
}
}
DECREF(entry);
}
DECREF(dh);
return retval;
}
static void
test_Local_Open_Dir(TestBatchRunner *runner) {
Folder *real_folder = S_folder_with_contents();
CompoundFileReader *cf_reader = CFReader_open(real_folder);
bool saw_foo = false;
bool saw_stuff = false;
bool stuff_was_dir = false;
CFReader_MkDir(cf_reader, stuff);
DirHandle *dh = CFReader_Local_Open_Dir(cf_reader);
while (DH_Next(dh)) {
String *entry = DH_Get_Entry(dh);
if (Str_Equals(entry, (Obj*)foo)) {
saw_foo = true;
}
else if (Str_Equals(entry, (Obj*)stuff)) {
saw_stuff = true;
stuff_was_dir = DH_Entry_Is_Dir(dh);
}
DECREF(entry);
}
TEST_TRUE(runner, saw_foo, "DirHandle iterated over virtual file");
TEST_TRUE(runner, saw_stuff, "DirHandle iterated over real directory");
TEST_TRUE(runner, stuff_was_dir,
"DirHandle knew that real entry was dir");
DECREF(dh);
DECREF(cf_reader);
DECREF(real_folder);
}
String*
IxManager_Make_Snapshot_Filename_IMP(IndexManager *self) {
IndexManagerIVARS *const ivars = IxManager_IVARS(self);
Folder *folder = (Folder*)CERTIFY(ivars->folder, FOLDER);
DirHandle *dh = Folder_Open_Dir(folder, NULL);
uint64_t max_gen = 0;
if (!dh) { RETHROW(INCREF(Err_get_error())); }
while (DH_Next(dh)) {
String *entry = DH_Get_Entry(dh);
if (Str_Starts_With_Utf8(entry, "snapshot_", 9)
&& Str_Ends_With_Utf8(entry, ".json", 5)
) {
uint64_t gen = IxFileNames_extract_gen(entry);
if (gen > max_gen) { max_gen = gen; }
}
DECREF(entry);
}
DECREF(dh);
uint64_t new_gen = max_gen + 1;
char base36[StrHelp_MAX_BASE36_BYTES];
StrHelp_to_base36(new_gen, &base36);
return Str_newf("snapshot_%s.json", &base36);
}
CharBuf*
IxManager_make_snapshot_filename(IndexManager *self)
{
Folder *folder = (Folder*)CERTIFY(self->folder, FOLDER);
DirHandle *dh = Folder_Open_Dir(folder, NULL);
CharBuf *entry;
uint64_t max_gen = 0;
if (!dh) { RETHROW(INCREF(Err_get_error())); }
entry = DH_Get_Entry(dh);
while (DH_Next(dh)) {
if ( CB_Starts_With_Str(entry, "snapshot_", 9)
&& CB_Ends_With_Str(entry, ".json", 5)
) {
uint64_t gen = IxFileNames_extract_gen(entry);
if (gen > max_gen) { max_gen = gen; }
}
}
DECREF(dh);
{
uint64_t new_gen = max_gen + 1;
char base36[StrHelp_MAX_BASE36_BYTES];
StrHelp_to_base36(new_gen, &base36);
return CB_newf("snapshot_%s.json", &base36);
}
}
bool_t
Folder_delete_tree(Folder *self, const CharBuf *path) {
Folder *enclosing_folder = Folder_Enclosing_Folder(self, path);
// Don't allow Folder to delete itself.
if (!path || !CB_Get_Size(path)) {
return false;
}
if (enclosing_folder) {
ZombieCharBuf *local = IxFileNames_local_part(path, ZCB_BLANK());
if (Folder_Local_Is_Directory(enclosing_folder, (CharBuf*)local)) {
Folder *inner_folder
= Folder_Local_Find_Folder(enclosing_folder, (CharBuf*)local);
DirHandle *dh = Folder_Local_Open_Dir(inner_folder);
if (dh) {
VArray *files = VA_new(20);
VArray *dirs = VA_new(20);
CharBuf *entry = DH_Get_Entry(dh);
while (DH_Next(dh)) {
VA_Push(files, (Obj*)CB_Clone(entry));
if (DH_Entry_Is_Dir(dh) && !DH_Entry_Is_Symlink(dh)) {
VA_Push(dirs, (Obj*)CB_Clone(entry));
}
}
for (uint32_t i = 0, max = VA_Get_Size(dirs); i < max; i++) {
CharBuf *name = (CharBuf*)VA_Fetch(files, i);
bool_t success = Folder_Delete_Tree(inner_folder, name);
if (!success && Folder_Local_Exists(inner_folder, name)) {
break;
}
}
for (uint32_t i = 0, max = VA_Get_Size(files); i < max; i++) {
CharBuf *name = (CharBuf*)VA_Fetch(files, i);
bool_t success = Folder_Local_Delete(inner_folder, name);
if (!success && Folder_Local_Exists(inner_folder, name)) {
break;
}
}
DECREF(dirs);
DECREF(files);
DECREF(dh);
}
}
return Folder_Local_Delete(enclosing_folder, (CharBuf*)local);
}
else {
// Return failure if the entry wasn't there in the first place.
return false;
}
}
CFReaderDirHandle*
CFReaderDH_init(CFReaderDirHandle *self, CompoundFileReader *cf_reader) {
DH_init((DirHandle*)self, CFReader_Get_Path(cf_reader));
self->cf_reader = (CompoundFileReader*)INCREF(cf_reader);
self->elems = Hash_Keys(self->cf_reader->records);
self->tick = -1;
// Accumulate entries from real Folder.
DirHandle *dh = Folder_Local_Open_Dir(self->cf_reader->real_folder);
CharBuf *entry = DH_Get_Entry(dh);
while (DH_Next(dh)) {
VA_Push(self->elems, (Obj*)CB_Clone(entry));
}
DECREF(dh);
return self;
}
VArray*
Folder_list(Folder *self, const CharBuf *path) {
Folder *local_folder = Folder_Find_Folder(self, path);
VArray *list = NULL;
DirHandle *dh = Folder_Local_Open_Dir(local_folder);
if (dh) {
CharBuf *entry = DH_Get_Entry(dh);
list = VA_new(32);
while (DH_Next(dh)) { VA_Push(list, (Obj*)CB_Clone(entry)); }
DECREF(dh);
}
else {
ERR_ADD_FRAME(Err_get_error());
}
return list;
}
static void
S_zap_dead_merge(FilePurger *self, Hash *candidates)
{
IndexManager *manager = self->manager;
Lock *merge_lock = IxManager_Make_Merge_Lock(manager);
Lock_Clear_Stale(merge_lock);
if (!Lock_Is_Locked(merge_lock)) {
Hash *merge_data = IxManager_Read_Merge_Data(manager);
Obj *cutoff = merge_data
? Hash_Fetch_Str(merge_data, "cutoff", 6)
: NULL;
if (cutoff) {
CharBuf *cutoff_seg = Seg_num_to_name(Obj_To_I64(cutoff));
if (Folder_Exists(self->folder, cutoff_seg)) {
ZombieCharBuf *merge_json = ZCB_WRAP_STR("merge.json", 10);
DirHandle *dh = Folder_Open_Dir(self->folder, cutoff_seg);
CharBuf *entry = dh ? DH_Get_Entry(dh) : NULL;
CharBuf *filepath = CB_new(32);
if (!dh) {
THROW(ERR, "Can't open segment dir '%o'", filepath);
}
Hash_Store(candidates, (Obj*)cutoff_seg, INCREF(&EMPTY));
Hash_Store(candidates, (Obj*)merge_json, INCREF(&EMPTY));
while (DH_Next(dh)) {
// TODO: recursively delete subdirs within seg dir.
CB_setf(filepath, "%o/%o", cutoff_seg, entry);
Hash_Store(candidates, (Obj*)filepath, INCREF(&EMPTY));
}
DECREF(filepath);
DECREF(dh);
}
DECREF(cutoff_seg);
}
DECREF(merge_data);
}
DECREF(merge_lock);
return;
}
static void
S_zap_dead_merge(FilePurger *self, Hash *candidates) {
FilePurgerIVARS *const ivars = FilePurger_IVARS(self);
IndexManager *manager = ivars->manager;
Lock *merge_lock = IxManager_Make_Merge_Lock(manager);
Lock_Clear_Stale(merge_lock);
if (!Lock_Is_Locked(merge_lock)) {
Hash *merge_data = IxManager_Read_Merge_Data(manager);
Obj *cutoff = merge_data
? Hash_Fetch_Utf8(merge_data, "cutoff", 6)
: NULL;
if (cutoff) {
String *cutoff_seg = Seg_num_to_name(Json_obj_to_i64(cutoff));
if (Folder_Exists(ivars->folder, cutoff_seg)) {
String *merge_json = SSTR_WRAP_UTF8("merge.json", 10);
DirHandle *dh = Folder_Open_Dir(ivars->folder, cutoff_seg);
if (!dh) {
THROW(ERR, "Can't open segment dir '%o'", cutoff_seg);
}
Hash_Store(candidates, cutoff_seg, (Obj*)CFISH_TRUE);
Hash_Store(candidates, merge_json, (Obj*)CFISH_TRUE);
while (DH_Next(dh)) {
// TODO: recursively delete subdirs within seg dir.
String *entry = DH_Get_Entry(dh);
String *filepath = Str_newf("%o/%o", cutoff_seg, entry);
Hash_Store(candidates, filepath, (Obj*)CFISH_TRUE);
DECREF(filepath);
DECREF(entry);
}
DECREF(dh);
}
DECREF(cutoff_seg);
}
DECREF(merge_data);
}
DECREF(merge_lock);
return;
}
CFReaderDirHandle*
CFReaderDH_init(CFReaderDirHandle *self, CompoundFileReader *cf_reader) {
DH_init((DirHandle*)self, CFReader_Get_Path(cf_reader));
CFReaderDirHandleIVARS *const ivars = CFReaderDH_IVARS(self);
ivars->cf_reader = (CompoundFileReader*)INCREF(cf_reader);
Hash *cf_records = CFReader_IVARS(ivars->cf_reader)->records;
ivars->elems = Hash_Keys(cf_records);
ivars->tick = -1;
// Accumulate entries from real Folder.
Folder *real_folder = CFReader_Get_Real_Folder(ivars->cf_reader);
DirHandle *dh = Folder_Local_Open_Dir(real_folder);
while (DH_Next(dh)) {
String *entry = DH_Get_Entry(dh);
Vec_Push(ivars->elems, (Obj*)Str_Clone(entry));
DECREF(entry);
}
DECREF(dh);
return self;
}
static void
S_add_to_file_list(Folder *self, VArray *list, CharBuf *dir, CharBuf *prefix) {
size_t orig_prefix_size = CB_Get_Size(prefix);
DirHandle *dh = Folder_Open_Dir(self, dir);
CharBuf *entry;
if (!dh) {
RETHROW(INCREF(Err_get_error()));
}
entry = DH_Get_Entry(dh);
while (DH_Next(dh)) { // Updates entry
if (!S_is_updir(entry)) {
CharBuf *relpath = CB_newf("%o%o", prefix, entry);
if (VA_Get_Size(list) == VA_Get_Capacity(list)) {
VA_Grow(list, VA_Get_Size(list) * 2);
}
VA_Push(list, (Obj*)relpath);
if (DH_Entry_Is_Dir(dh) && !DH_Entry_Is_Symlink(dh)) {
CharBuf *subdir = CB_Get_Size(dir)
? CB_newf("%o/%o", dir, entry)
: CB_Clone(entry);
CB_catf(prefix, "%o/", entry);
S_add_to_file_list(self, list, subdir, prefix); // recurse
CB_Set_Size(prefix, orig_prefix_size);
DECREF(subdir);
}
}
}
if (!DH_Close(dh)) {
RETHROW(INCREF(Err_get_error()));
}
DECREF(dh);
}
static void
S_discover_unused(FilePurger *self, VArray **purgables_ptr,
VArray **snapshots_ptr)
{
Folder *folder = self->folder;
DirHandle *dh = Folder_Open_Dir(folder, NULL);
if (!dh) { RETHROW(INCREF(Err_get_error())); }
VArray *spared = VA_new(1);
VArray *snapshots = VA_new(1);
CharBuf *snapfile = NULL;
// Start off with the list of files in the current snapshot.
if (self->snapshot) {
VArray *entries = Snapshot_List(self->snapshot);
VArray *referenced = S_find_all_referenced(folder, entries);
VA_Push_VArray(spared, referenced);
DECREF(entries);
DECREF(referenced);
snapfile = Snapshot_Get_Path(self->snapshot);
if (snapfile) { VA_Push(spared, INCREF(snapfile)); }
}
CharBuf *entry = DH_Get_Entry(dh);
Hash *candidates = Hash_new(64);
while (DH_Next(dh)) {
if (!CB_Starts_With_Str(entry, "snapshot_", 9)) { continue; }
else if (!CB_Ends_With_Str(entry, ".json", 5)) { continue; }
else if (snapfile && CB_Equals(entry, (Obj*)snapfile)) { continue; }
else {
Snapshot *snapshot
= Snapshot_Read_File(Snapshot_new(), folder, entry);
Lock *lock
= IxManager_Make_Snapshot_Read_Lock(self->manager, entry);
VArray *snap_list = Snapshot_List(snapshot);
VArray *referenced = S_find_all_referenced(folder, snap_list);
// DON'T obtain the lock -- only see whether another
// entity holds a lock on the snapshot file.
if (lock) {
Lock_Clear_Stale(lock);
}
if (lock && Lock_Is_Locked(lock)) {
// The snapshot file is locked, which means someone's using
// that version of the index -- protect all of its entries.
uint32_t new_size = VA_Get_Size(spared)
+ VA_Get_Size(referenced) + 1;
VA_Grow(spared, new_size);
VA_Push(spared, (Obj*)CB_Clone(entry));
VA_Push_VArray(spared, referenced);
}
else {
// No one's using this snapshot, so all of its entries are
// candidates for deletion.
for (uint32_t i = 0, max = VA_Get_Size(referenced); i < max; i++) {
CharBuf *file = (CharBuf*)VA_Fetch(referenced, i);
Hash_Store(candidates, (Obj*)file, INCREF(&EMPTY));
}
VA_Push(snapshots, INCREF(snapshot));
}
DECREF(referenced);
DECREF(snap_list);
DECREF(snapshot);
DECREF(lock);
}
}
DECREF(dh);
// Clean up after a dead segment consolidation.
S_zap_dead_merge(self, candidates);
// Eliminate any current files from the list of files to be purged.
for (uint32_t i = 0, max = VA_Get_Size(spared); i < max; i++) {
CharBuf *filename = (CharBuf*)VA_Fetch(spared, i);
DECREF(Hash_Delete(candidates, (Obj*)filename));
}
// Pass back purgables and Snapshots.
*purgables_ptr = Hash_Keys(candidates);
*snapshots_ptr = snapshots;
DECREF(candidates);
DECREF(spared);
}
