bool CFReader_Local_Delete_IMP(CompoundFileReader *self, String *name) { CompoundFileReaderIVARS *const ivars = CFReader_IVARS(self); Hash *record = (Hash*)Hash_Delete(ivars->records, name); DECREF(record); if (record == NULL) { return Folder_Local_Delete(ivars->real_folder, name); } else { // Once the number of virtual files falls to 0, remove the compound // files. if (Hash_Get_Size(ivars->records) == 0) { String *cf_file = (String*)SSTR_WRAP_UTF8("cf.dat", 6); if (!Folder_Delete(ivars->real_folder, cf_file)) { return false; } String *cfmeta_file = (String*)SSTR_WRAP_UTF8("cfmeta.json", 11); if (!Folder_Delete(ivars->real_folder, cfmeta_file)) { return false; } } return true; } }
bool_t CFReader_local_delete(CompoundFileReader *self, const CharBuf *name) { Hash *record = (Hash*)Hash_Delete(self->records, (Obj*)name); DECREF(record); if (record == NULL) { return Folder_Local_Delete(self->real_folder, name); } else { // Once the number of virtual files falls to 0, remove the compound // files. if (Hash_Get_Size(self->records) == 0) { CharBuf *cf_file = (CharBuf*)ZCB_WRAP_STR("cf.dat", 6); if (!Folder_Delete(self->real_folder, cf_file)) { return false; } CharBuf *cfmeta_file = (CharBuf*)ZCB_WRAP_STR("cfmeta.json", 11); if (!Folder_Delete(self->real_folder, cfmeta_file)) { return false; } } return true; } }
bool LFLock_Maybe_Delete_File_IMP(LockFileLock *self, String *path, bool delete_mine, bool delete_other) { LockFileLockIVARS *const ivars = LFLock_IVARS(self); Folder *folder = ivars->folder; bool success = false; // Only delete locks that start with our lock name. if (!Str_Starts_With_Utf8(path, "locks", 5)) { return false; } StringIterator *iter = Str_Top(path); StrIter_Advance(iter, 5 + 1); if (!StrIter_Starts_With(iter, ivars->name)) { DECREF(iter); return false; } DECREF(iter); // Attempt to delete dead lock file. if (Folder_Exists(folder, path)) { Hash *hash = (Hash*)Json_slurp_json(folder, path); if (hash != NULL && Obj_Is_A((Obj*)hash, HASH)) { String *pid_buf = (String*)Hash_Fetch_Utf8(hash, "pid", 3); String *host = (String*)Hash_Fetch_Utf8(hash, "host", 4); String *name = (String*)Hash_Fetch_Utf8(hash, "name", 4); // Match hostname and lock name. if (host != NULL && Str_Is_A(host, STRING) && Str_Equals(host, (Obj*)ivars->host) && name != NULL && Str_Is_A(name, STRING) && Str_Equals(name, (Obj*)ivars->name) && pid_buf != NULL && Str_Is_A(pid_buf, STRING) ) { // Verify that pid is either mine or dead. int pid = (int)Str_To_I64(pid_buf); if ((delete_mine && pid == PID_getpid()) // This process. || (delete_other && !PID_active(pid)) // Dead pid. ) { if (Folder_Delete(folder, path)) { success = true; } else { String *mess = MAKE_MESS("Can't delete '%o'", path); DECREF(hash); Err_throw_mess(ERR, mess); } } } } DECREF(hash); } return success; }
void Indexer_prepare_commit(Indexer *self) { VArray *seg_readers = PolyReader_Get_Seg_Readers(self->polyreader); uint32_t num_seg_readers = VA_Get_Size(seg_readers); bool_t merge_happened = false; if ( !self->write_lock || self->prepared ) { THROW(ERR, "Can't call Prepare_Commit() more than once"); } // Merge existing index data. if (num_seg_readers) { merge_happened = S_maybe_merge(self, seg_readers); } // Add a new segment and write a new snapshot file if... if ( Seg_Get_Count(self->segment) // Docs/segs added. || merge_happened // Some segs merged. || !Snapshot_Num_Entries(self->snapshot) // Initializing index. || DelWriter_Updated(self->del_writer) ) { Folder *folder = self->folder; Schema *schema = self->schema; Snapshot *snapshot = self->snapshot; CharBuf *old_schema_name = S_find_schema_file(snapshot); uint64_t schema_gen = old_schema_name ? IxFileNames_extract_gen(old_schema_name) + 1 : 1; char base36[StrHelp_MAX_BASE36_BYTES]; CharBuf *new_schema_name; StrHelp_to_base36(schema_gen, &base36); new_schema_name = CB_newf("schema_%s.json", base36); // Finish the segment, write schema file. SegWriter_Finish(self->seg_writer); Schema_Write(schema, folder, new_schema_name); if (old_schema_name) { Snapshot_Delete_Entry(snapshot, old_schema_name); } Snapshot_Add_Entry(snapshot, new_schema_name); DECREF(new_schema_name); // Write temporary snapshot file. DECREF(self->snapfile); self->snapfile = IxManager_Make_Snapshot_Filename(self->manager); CB_Cat_Trusted_Str(self->snapfile, ".temp", 5); Folder_Delete(folder, self->snapfile); Snapshot_Write_File(snapshot, folder, self->snapfile); self->needs_commit = true; } // Close reader, so that we can delete its files if appropriate. PolyReader_Close(self->polyreader); self->prepared = true; }
void Indexer_Prepare_Commit_IMP(Indexer *self) { IndexerIVARS *const ivars = Indexer_IVARS(self); Vector *seg_readers = PolyReader_Get_Seg_Readers(ivars->polyreader); size_t num_seg_readers = Vec_Get_Size(seg_readers); bool merge_happened = false; if (!ivars->write_lock || ivars->prepared) { THROW(ERR, "Can't call Prepare_Commit() more than once"); } // Merge existing index data. if (num_seg_readers) { merge_happened = S_maybe_merge(self, seg_readers); } // Add a new segment and write a new snapshot file if... if (Seg_Get_Count(ivars->segment) // Docs/segs added. || merge_happened // Some segs merged. || !Snapshot_Num_Entries(ivars->snapshot) // Initializing index. || DelWriter_Updated(ivars->del_writer) ) { Folder *folder = ivars->folder; Schema *schema = ivars->schema; Snapshot *snapshot = ivars->snapshot; // Derive snapshot and schema file names. DECREF(ivars->snapfile); String *snapfile = IxManager_Make_Snapshot_Filename(ivars->manager); ivars->snapfile = Str_Cat_Trusted_Utf8(snapfile, ".temp", 5); DECREF(snapfile); uint64_t schema_gen = IxFileNames_extract_gen(ivars->snapfile); char base36[StrHelp_MAX_BASE36_BYTES]; StrHelp_to_base36(schema_gen, &base36); String *new_schema_name = Str_newf("schema_%s.json", base36); // Finish the segment, write schema file. SegWriter_Finish(ivars->seg_writer); Schema_Write(schema, folder, new_schema_name); String *old_schema_name = S_find_schema_file(snapshot); if (old_schema_name) { Snapshot_Delete_Entry(snapshot, old_schema_name); } Snapshot_Add_Entry(snapshot, new_schema_name); DECREF(new_schema_name); // Write temporary snapshot file. Folder_Delete(folder, ivars->snapfile); Snapshot_Write_File(snapshot, folder, ivars->snapfile); ivars->needs_commit = true; } // Close reader, so that we can delete its files if appropriate. PolyReader_Close(ivars->polyreader); ivars->prepared = true; }
static void S_clean_up_old_temp_files(CompoundFileWriter *self, CompoundFileWriterIVARS *ivars) { UNUSED_VAR(self); Folder *folder = ivars->folder; String *cfmeta_temp = (String*)SSTR_WRAP_UTF8("cfmeta.json.temp", 16); String *cf_file = (String*)SSTR_WRAP_UTF8("cf.dat", 6); if (Folder_Exists(folder, cf_file)) { if (!Folder_Delete(folder, cf_file)) { THROW(ERR, "Can't delete '%o'", cf_file); } } if (Folder_Exists(folder, cfmeta_temp)) { if (!Folder_Delete(folder, cfmeta_temp)) { THROW(ERR, "Can't delete '%o'", cfmeta_temp); } } }
bool_t LFLock_maybe_delete_file(LockFileLock *self, const CharBuf *path, bool_t delete_mine, bool_t delete_other) { Folder *folder = self->folder; bool_t success = false; ZombieCharBuf *scratch = ZCB_WRAP(path); // Only delete locks that start with our lock name. CharBuf *lock_dir_name = (CharBuf*)ZCB_WRAP_STR("locks", 5); if (!ZCB_Starts_With(scratch, lock_dir_name)) { return false; } ZCB_Nip(scratch, CB_Get_Size(lock_dir_name) + 1); if (!ZCB_Starts_With(scratch, self->name)) { return false; } // Attempt to delete dead lock file. if (Folder_Exists(folder, path)) { Hash *hash = (Hash*)Json_slurp_json(folder, path); if (hash != NULL && Obj_Is_A((Obj*)hash, HASH)) { CharBuf *pid_buf = (CharBuf*)Hash_Fetch_Str(hash, "pid", 3); CharBuf *host = (CharBuf*)Hash_Fetch_Str(hash, "host", 4); CharBuf *name = (CharBuf*)Hash_Fetch_Str(hash, "name", 4); // Match hostname and lock name. if (host != NULL && CB_Equals(host, (Obj*)self->host) && name != NULL && CB_Equals(name, (Obj*)self->name) && pid_buf != NULL ) { // Verify that pid is either mine or dead. int pid = (int)CB_To_I64(pid_buf); if ((delete_mine && pid == PID_getpid()) // This process. || (delete_other && !PID_active(pid)) // Dead pid. ) { if (Folder_Delete(folder, path)) { success = true; } else { CharBuf *mess = MAKE_MESS("Can't delete '%o'", path); DECREF(hash); Err_throw_mess(ERR, mess); } } } } DECREF(hash); } return success; }
void Schema_Write_IMP(Schema *self, Folder *folder, String *filename) { Hash *dump = Schema_Dump(self); String *schema_temp = SSTR_WRAP_C("schema.temp"); bool success; Folder_Delete(folder, schema_temp); // Just in case. Json_spew_json((Obj*)dump, folder, schema_temp); success = Folder_Rename(folder, schema_temp, filename); DECREF(dump); if (!success) { RETHROW(INCREF(Err_get_error())); } }
void BGMerger_Commit_IMP(BackgroundMerger *self) { BackgroundMergerIVARS *const ivars = BGMerger_IVARS(self); // Safety check. if (!ivars->merge_lock) { THROW(ERR, "Can't call commit() more than once"); } if (!ivars->prepared) { BGMerger_Prepare_Commit(self); } if (ivars->needs_commit) { bool success = false; String *temp_snapfile = ivars->snapfile; // Rename temp snapshot file. size_t ext_len = sizeof(".temp") - 1; size_t snapfile_len = Str_Length(temp_snapfile); if (snapfile_len <= ext_len) { THROW(ERR, "Invalid snapfile name: %o", temp_snapfile); } ivars->snapfile = Str_SubString(temp_snapfile, 0, snapfile_len - ext_len); success = Folder_Hard_Link(ivars->folder, temp_snapfile, ivars->snapfile); Snapshot_Set_Path(ivars->snapshot, ivars->snapfile); if (!success) { String *mess = Str_newf("Can't create hard link from %o to %o", temp_snapfile, ivars->snapfile); DECREF(temp_snapfile); Err_throw_mess(ERR, mess); } if (!Folder_Delete(ivars->folder, temp_snapfile)) { String *mess = Str_newf("Can't delete %o", temp_snapfile); DECREF(temp_snapfile); Err_throw_mess(ERR, mess); } DECREF(temp_snapfile); } // Release the merge lock and remove the merge data file. S_release_merge_lock(self); IxManager_Remove_Merge_Data(ivars->manager); if (ivars->needs_commit) { // Purge obsolete files. FilePurger_Purge(ivars->file_purger); } // Release the write lock. S_release_write_lock(self); }
static void test_Delete(TestBatch *batch) { Folder *folder = (Folder*)RAMFolder_new(NULL); FileHandle *fh; bool_t result; Folder_MkDir(folder, &foo); Folder_MkDir(folder, &foo_bar); fh = Folder_Open_FileHandle(folder, &boffo, FH_CREATE | FH_WRITE_ONLY); DECREF(fh); fh = Folder_Open_FileHandle(folder, &foo_boffo, FH_CREATE | FH_WRITE_ONLY); DECREF(fh); Err_set_error(NULL); result = Folder_Delete(folder, &banana); TEST_FALSE(batch, result, "Delete on non-existent entry returns false"); Err_set_error(NULL); result = Folder_Delete(folder, &foo); TEST_FALSE(batch, result, "Delete on non-empty dir returns false"); TEST_TRUE(batch, Folder_Delete(folder, &foo_boffo), "Delete nested file"); TEST_FALSE(batch, Folder_Exists(folder, &foo_boffo), "File is really gone"); TEST_TRUE(batch, Folder_Delete(folder, &foo_bar), "Delete nested dir"); TEST_FALSE(batch, Folder_Exists(folder, &foo_bar), "Dir is really gone"); TEST_TRUE(batch, Folder_Delete(folder, &foo), "Delete empty dir"); TEST_FALSE(batch, Folder_Exists(folder, &foo), "Dir is really gone"); DECREF(folder); }
void FilePurger_purge(FilePurger *self) { Lock *deletion_lock = IxManager_Make_Deletion_Lock(self->manager); // Obtain deletion lock, purge files, release deletion lock. Lock_Clear_Stale(deletion_lock); if (Lock_Obtain(deletion_lock)) { Folder *folder = self->folder; Hash *failures = Hash_new(0); VArray *purgables; VArray *snapshots; S_discover_unused(self, &purgables, &snapshots); // Attempt to delete entries -- if failure, no big deal, just try // again later. Proceed in reverse lexical order so that directories // get deleted after they've been emptied. VA_Sort(purgables, NULL, NULL); for (uint32_t i = VA_Get_Size(purgables); i--; ) { CharBuf *entry = (CharBuf*)VA_fetch(purgables, i); if (Hash_Fetch(self->disallowed, (Obj*)entry)) { continue; } if (!Folder_Delete(folder, entry)) { if (Folder_Exists(folder, entry)) { Hash_Store(failures, (Obj*)entry, INCREF(&EMPTY)); } } } for (uint32_t i = 0, max = VA_Get_Size(snapshots); i < max; i++) { Snapshot *snapshot = (Snapshot*)VA_Fetch(snapshots, i); bool_t snapshot_has_failures = false; if (Hash_Get_Size(failures)) { // Only delete snapshot files if all of their entries were // successfully deleted. VArray *entries = Snapshot_List(snapshot); for (uint32_t j = VA_Get_Size(entries); j--; ) { CharBuf *entry = (CharBuf*)VA_Fetch(entries, j); if (Hash_Fetch(failures, (Obj*)entry)) { snapshot_has_failures = true; break; } } DECREF(entries); } if (!snapshot_has_failures) { CharBuf *snapfile = Snapshot_Get_Path(snapshot); Folder_Delete(folder, snapfile); } } DECREF(failures); DECREF(purgables); DECREF(snapshots); Lock_Release(deletion_lock); } else { WARN("Can't obtain deletion lock, skipping deletion of " "obsolete files"); } DECREF(deletion_lock); }
bool_t LFLock_request(LockFileLock *self) { Hash *file_data; bool_t wrote_json; bool_t success = false; bool_t deletion_failed = false; if (Folder_Exists(self->folder, self->lock_path)) { Err_set_error((Err*)LockErr_new(CB_newf("Can't obtain lock: '%o' exists", self->lock_path))); return false; } // Create the "locks" subdirectory if necessary. CharBuf *lock_dir_name = (CharBuf*)ZCB_WRAP_STR("locks", 5); if (!Folder_Exists(self->folder, lock_dir_name)) { if (!Folder_MkDir(self->folder, lock_dir_name)) { Err *mkdir_err = (Err*)CERTIFY(Err_get_error(), ERR); LockErr *err = LockErr_new(CB_newf("Can't create 'locks' directory: %o", Err_Get_Mess(mkdir_err))); // Maybe our attempt failed because another process succeeded. if (Folder_Find_Folder(self->folder, lock_dir_name)) { DECREF(err); } else { // Nope, everything failed, so bail out. Err_set_error((Err*)err); return false; } } } // Prepare to write pid, lock name, and host to the lock file as JSON. file_data = Hash_new(3); Hash_Store_Str(file_data, "pid", 3, (Obj*)CB_newf("%i32", (int32_t)PID_getpid())); Hash_Store_Str(file_data, "host", 4, INCREF(self->host)); Hash_Store_Str(file_data, "name", 4, INCREF(self->name)); // Write to a temporary file, then use the creation of a hard link to // ensure atomic but non-destructive creation of the lockfile with its // complete contents. wrote_json = Json_spew_json((Obj*)file_data, self->folder, self->link_path); if (wrote_json) { success = Folder_Hard_Link(self->folder, self->link_path, self->lock_path); if (!success) { Err *hard_link_err = (Err*)CERTIFY(Err_get_error(), ERR); Err_set_error((Err*)LockErr_new(CB_newf("Failed to obtain lock at '%o': %o", self->lock_path, Err_Get_Mess(hard_link_err)))); } deletion_failed = !Folder_Delete(self->folder, self->link_path); } else { Err *spew_json_err = (Err*)CERTIFY(Err_get_error(), ERR); Err_set_error((Err*)LockErr_new(CB_newf("Failed to obtain lock at '%o': %o", self->lock_path, Err_Get_Mess(spew_json_err)))); } DECREF(file_data); // Verify that our temporary file got zapped. if (wrote_json && deletion_failed) { CharBuf *mess = MAKE_MESS("Failed to delete '%o'", self->link_path); Err_throw_mess(ERR, mess); } return success; }
bool LFLock_Request_IMP(LockFileLock *self) { LockFileLockIVARS *const ivars = LFLock_IVARS(self); bool success = false; if (Folder_Exists(ivars->folder, ivars->lock_path)) { Err_set_error((Err*)LockErr_new(Str_newf("Can't obtain lock: '%o' exists", ivars->lock_path))); return false; } // Create the "locks" subdirectory if necessary. String *lock_dir_name = (String*)SSTR_WRAP_UTF8("locks", 5); if (!Folder_Exists(ivars->folder, lock_dir_name)) { if (!Folder_MkDir(ivars->folder, lock_dir_name)) { Err *mkdir_err = (Err*)CERTIFY(Err_get_error(), ERR); LockErr *err = LockErr_new(Str_newf("Can't create 'locks' directory: %o", Err_Get_Mess(mkdir_err))); // Maybe our attempt failed because another process succeeded. if (Folder_Find_Folder(ivars->folder, lock_dir_name)) { DECREF(err); } else { // Nope, everything failed, so bail out. Err_set_error((Err*)err); return false; } } } // Prepare to write pid, lock name, and host to the lock file as JSON. Hash *file_data = Hash_new(3); Hash_Store_Utf8(file_data, "pid", 3, (Obj*)Str_newf("%i32", (int32_t)PID_getpid())); Hash_Store_Utf8(file_data, "host", 4, INCREF(ivars->host)); Hash_Store_Utf8(file_data, "name", 4, INCREF(ivars->name)); String *json = Json_to_json((Obj*)file_data); DECREF(file_data); // Write to a temporary file, then use the creation of a hard link to // ensure atomic but non-destructive creation of the lockfile with its // complete contents. OutStream *outstream = Folder_Open_Out(ivars->folder, ivars->link_path); if (!outstream) { ERR_ADD_FRAME(Err_get_error()); DECREF(json); return false; } struct lockfile_context context; context.outstream = outstream; context.json = json; Err *json_error = Err_trap(S_write_lockfile_json, &context); bool wrote_json = !json_error; DECREF(outstream); DECREF(json); if (wrote_json) { success = Folder_Hard_Link(ivars->folder, ivars->link_path, ivars->lock_path); if (!success) { Err *hard_link_err = (Err*)CERTIFY(Err_get_error(), ERR); Err_set_error((Err*)LockErr_new(Str_newf("Failed to obtain lock at '%o': %o", ivars->lock_path, Err_Get_Mess(hard_link_err)))); } } else { Err_set_error((Err*)LockErr_new(Str_newf("Failed to obtain lock at '%o': %o", ivars->lock_path, Err_Get_Mess(json_error)))); DECREF(json_error); } // Verify that our temporary file got zapped. bool deletion_failed = !Folder_Delete(ivars->folder, ivars->link_path); if (deletion_failed) { String *mess = MAKE_MESS("Failed to delete '%o'", ivars->link_path); Err_throw_mess(ERR, mess); } return success; }
bool IxManager_Remove_Merge_Data_IMP(IndexManager *self) { IndexManagerIVARS *const ivars = IxManager_IVARS(self); String *merge_json = SSTR_WRAP_C("merge.json"); return Folder_Delete(ivars->folder, merge_json) != 0; }
void SortWriter_finish(SortWriter *self) { SortWriterIVARS *const ivars = SortWriter_IVARS(self); VArray *const field_writers = ivars->field_writers; // If we have no data, bail out. if (!ivars->temp_ord_out) { return; } // If we've either flushed or added segments, flush everything so that any // one field can use the entire margin up to mem_thresh. if (ivars->flush_at_finish) { for (uint32_t i = 1, max = VA_Get_Size(field_writers); i < max; i++) { SortFieldWriter *field_writer = (SortFieldWriter*)VA_Fetch(field_writers, i); if (field_writer) { SortFieldWriter_Flush(field_writer); } } } // Close down temp streams. OutStream_Close(ivars->temp_ord_out); OutStream_Close(ivars->temp_ix_out); OutStream_Close(ivars->temp_dat_out); for (uint32_t i = 1, max = VA_Get_Size(field_writers); i < max; i++) { SortFieldWriter *field_writer = (SortFieldWriter*)VA_Delete(field_writers, i); if (field_writer) { CharBuf *field = Seg_Field_Name(ivars->segment, i); SortFieldWriter_Flip(field_writer); int32_t count = SortFieldWriter_Finish(field_writer); Hash_Store(ivars->counts, (Obj*)field, (Obj*)CB_newf("%i32", count)); int32_t null_ord = SortFieldWriter_Get_Null_Ord(field_writer); if (null_ord != -1) { Hash_Store(ivars->null_ords, (Obj*)field, (Obj*)CB_newf("%i32", null_ord)); } int32_t ord_width = SortFieldWriter_Get_Ord_Width(field_writer); Hash_Store(ivars->ord_widths, (Obj*)field, (Obj*)CB_newf("%i32", ord_width)); } DECREF(field_writer); } VA_Clear(field_writers); // Store metadata. Seg_Store_Metadata_Str(ivars->segment, "sort", 4, (Obj*)SortWriter_Metadata(self)); // Clean up. Folder *folder = ivars->folder; CharBuf *seg_name = Seg_Get_Name(ivars->segment); CharBuf *path = CB_newf("%o/sort_ord_temp", seg_name); Folder_Delete(folder, path); CB_setf(path, "%o/sort_ix_temp", seg_name); Folder_Delete(folder, path); CB_setf(path, "%o/sort_dat_temp", seg_name); Folder_Delete(folder, path); DECREF(path); }
bool_t IxManager_remove_merge_data(IndexManager *self) { ZombieCharBuf *merge_json = ZCB_WRAP_STR("merge.json", 10); return Folder_Delete(self->folder, (CharBuf*)merge_json) != 0; }
void BGMerger_Prepare_Commit_IMP(BackgroundMerger *self) { BackgroundMergerIVARS *const ivars = BGMerger_IVARS(self); Vector *seg_readers = PolyReader_Get_Seg_Readers(ivars->polyreader); uint32_t num_seg_readers = Vec_Get_Size(seg_readers); uint32_t segs_merged = 0; if (ivars->prepared) { THROW(ERR, "Can't call Prepare_Commit() more than once"); } // Maybe merge existing index data. if (num_seg_readers) { segs_merged = S_maybe_merge(self); } if (!segs_merged) { // Nothing merged. Leave `needs_commit` false and bail out. ivars->prepared = true; return; } // Finish the segment and write a new snapshot file. else { Folder *folder = ivars->folder; Snapshot *snapshot = ivars->snapshot; // Write out new deletions. if (DelWriter_Updated(ivars->del_writer)) { // Only write out if they haven't all been applied. if (segs_merged != num_seg_readers) { DelWriter_Finish(ivars->del_writer); } } // Finish the segment. SegWriter_Finish(ivars->seg_writer); // Grab the write lock. S_obtain_write_lock(self); if (!ivars->write_lock) { RETHROW(INCREF(Err_get_error())); } // Write temporary snapshot file. DECREF(ivars->snapfile); String *snapfile = IxManager_Make_Snapshot_Filename(ivars->manager); ivars->snapfile = Str_Cat_Trusted_Utf8(snapfile, ".temp", 5); DECREF(snapfile); Folder_Delete(folder, ivars->snapfile); Snapshot_Write_File(snapshot, folder, ivars->snapfile); // Determine whether the index has been updated while this background // merge process was running. String *start_snapfile = Snapshot_Get_Path(PolyReader_Get_Snapshot(ivars->polyreader)); Snapshot *latest_snapshot = Snapshot_Read_File(Snapshot_new(), ivars->folder, NULL); String *latest_snapfile = Snapshot_Get_Path(latest_snapshot); bool index_updated = !Str_Equals(start_snapfile, (Obj*)latest_snapfile); if (index_updated) { /* See if new deletions have been applied since this * background merge process started against any of the * segments we just merged away. If that's true, we need to * write another segment which applies the deletions against * the new composite segment. */ S_merge_updated_deletions(self); // Add the fresh content to our snapshot. (It's important to // run this AFTER S_merge_updated_deletions, because otherwise // we couldn't tell whether the deletion counts changed.) Vector *files = Snapshot_List(latest_snapshot); for (uint32_t i = 0, max = Vec_Get_Size(files); i < max; i++) { String *file = (String*)Vec_Fetch(files, i); if (Str_Starts_With_Utf8(file, "seg_", 4)) { int64_t gen = (int64_t)IxFileNames_extract_gen(file); if (gen > ivars->cutoff) { Snapshot_Add_Entry(ivars->snapshot, file); } } } DECREF(files); // Since the snapshot content has changed, we need to rewrite it. Folder_Delete(folder, ivars->snapfile); Snapshot_Write_File(snapshot, folder, ivars->snapfile); } DECREF(latest_snapshot); ivars->needs_commit = true; } // Close reader, so that we can delete its files if appropriate. PolyReader_Close(ivars->polyreader); ivars->prepared = true; }
static void test_open(TestBatchRunner *runner) { Folder *real_folder; CompoundFileReader *cf_reader; Hash *metadata; Err_set_error(NULL); real_folder = S_folder_with_contents(); Folder_Delete(real_folder, cfmeta_file); cf_reader = CFReader_open(real_folder); TEST_TRUE(runner, cf_reader == NULL, "Return NULL when cfmeta file missing"); TEST_TRUE(runner, Err_get_error() != NULL, "Set global error when cfmeta file missing"); DECREF(real_folder); Err_set_error(NULL); real_folder = S_folder_with_contents(); Folder_Delete(real_folder, cf_file); cf_reader = CFReader_open(real_folder); TEST_TRUE(runner, cf_reader == NULL, "Return NULL when cf.dat file missing"); TEST_TRUE(runner, Err_get_error() != NULL, "Set global error when cf.dat file missing"); DECREF(real_folder); Err_set_error(NULL); real_folder = S_folder_with_contents(); metadata = (Hash*)Json_slurp_json(real_folder, cfmeta_file); Hash_Store_Utf8(metadata, "format", 6, (Obj*)Str_newf("%i32", -1)); Folder_Delete(real_folder, cfmeta_file); Json_spew_json((Obj*)metadata, real_folder, cfmeta_file); cf_reader = CFReader_open(real_folder); TEST_TRUE(runner, cf_reader == NULL, "Return NULL when format is invalid"); TEST_TRUE(runner, Err_get_error() != NULL, "Set global error when format is invalid"); Err_set_error(NULL); Hash_Store_Utf8(metadata, "format", 6, (Obj*)Str_newf("%i32", 1000)); Folder_Delete(real_folder, cfmeta_file); Json_spew_json((Obj*)metadata, real_folder, cfmeta_file); cf_reader = CFReader_open(real_folder); TEST_TRUE(runner, cf_reader == NULL, "Return NULL when format is too recent"); TEST_TRUE(runner, Err_get_error() != NULL, "Set global error when format too recent"); Err_set_error(NULL); DECREF(Hash_Delete_Utf8(metadata, "format", 6)); Folder_Delete(real_folder, cfmeta_file); Json_spew_json((Obj*)metadata, real_folder, cfmeta_file); cf_reader = CFReader_open(real_folder); TEST_TRUE(runner, cf_reader == NULL, "Return NULL when format key is missing"); TEST_TRUE(runner, Err_get_error() != NULL, "Set global error when format key is missing"); Hash_Store_Utf8(metadata, "format", 6, (Obj*)Str_newf("%i32", CFWriter_current_file_format)); DECREF(Hash_Delete_Utf8(metadata, "files", 5)); Folder_Delete(real_folder, cfmeta_file); Json_spew_json((Obj*)metadata, real_folder, cfmeta_file); cf_reader = CFReader_open(real_folder); TEST_TRUE(runner, cf_reader == NULL, "Return NULL when files key is missing"); TEST_TRUE(runner, Err_get_error() != NULL, "Set global error when files key is missing"); DECREF(metadata); DECREF(real_folder); }
static void S_do_consolidate(CompoundFileWriter *self, CompoundFileWriterIVARS *ivars) { UNUSED_VAR(self); Folder *folder = ivars->folder; Hash *metadata = Hash_new(0); Hash *sub_files = Hash_new(0); Vector *files = Folder_List(folder, NULL); Vector *merged = Vec_new(Vec_Get_Size(files)); String *cf_file = (String*)SSTR_WRAP_UTF8("cf.dat", 6); OutStream *outstream = Folder_Open_Out(folder, (String*)cf_file); bool rename_success; if (!outstream) { RETHROW(INCREF(Err_get_error())); } // Start metadata. Hash_Store_Utf8(metadata, "files", 5, INCREF(sub_files)); Hash_Store_Utf8(metadata, "format", 6, (Obj*)Str_newf("%i32", CFWriter_current_file_format)); Vec_Sort(files); for (uint32_t i = 0, max = Vec_Get_Size(files); i < max; i++) { String *infilename = (String*)Vec_Fetch(files, i); if (!Str_Ends_With_Utf8(infilename, ".json", 5)) { InStream *instream = Folder_Open_In(folder, infilename); Hash *file_data = Hash_new(2); int64_t offset, len; if (!instream) { RETHROW(INCREF(Err_get_error())); } // Absorb the file. offset = OutStream_Tell(outstream); OutStream_Absorb(outstream, instream); len = OutStream_Tell(outstream) - offset; // Record offset and length. Hash_Store_Utf8(file_data, "offset", 6, (Obj*)Str_newf("%i64", offset)); Hash_Store_Utf8(file_data, "length", 6, (Obj*)Str_newf("%i64", len)); Hash_Store(sub_files, infilename, (Obj*)file_data); Vec_Push(merged, INCREF(infilename)); // Add filler NULL bytes so that every sub-file begins on a file // position multiple of 8. OutStream_Align(outstream, 8); InStream_Close(instream); DECREF(instream); } } // Write metadata to cfmeta file. String *cfmeta_temp = (String*)SSTR_WRAP_UTF8("cfmeta.json.temp", 16); String *cfmeta_file = (String*)SSTR_WRAP_UTF8("cfmeta.json", 11); Json_spew_json((Obj*)metadata, (Folder*)ivars->folder, cfmeta_temp); rename_success = Folder_Rename(ivars->folder, cfmeta_temp, cfmeta_file); if (!rename_success) { RETHROW(INCREF(Err_get_error())); } // Clean up. OutStream_Close(outstream); DECREF(outstream); DECREF(files); DECREF(metadata); /* HashIterator *iter = HashIter_new(sub_files); while (HashIter_Next(iter)) { String *merged_file = HashIter_Get_Key(iter); if (!Folder_Delete(folder, merged_file)) { String *mess = MAKE_MESS("Can't delete '%o'", merged_file); DECREF(sub_files); Err_throw_mess(ERR, mess); } } DECREF(iter); */ DECREF(sub_files); for (uint32_t i = 0, max = Vec_Get_Size(merged); i < max; i++) { String *merged_file = (String*)Vec_Fetch(merged, i); if (!Folder_Delete(folder, merged_file)) { String *mess = MAKE_MESS("Can't delete '%o'", merged_file); DECREF(merged); Err_throw_mess(ERR, mess); } } DECREF(merged); }