bool
CFReader_Local_Delete_IMP(CompoundFileReader *self, String *name) {
CompoundFileReaderIVARS *const ivars = CFReader_IVARS(self);
Hash *record = (Hash*)Hash_Delete(ivars->records, name);
DECREF(record);
if (record == NULL) {
return Folder_Local_Delete(ivars->real_folder, name);
}
else {
// Once the number of virtual files falls to 0, remove the compound
// files.
if (Hash_Get_Size(ivars->records) == 0) {
String *cf_file = (String*)SSTR_WRAP_UTF8("cf.dat", 6);
if (!Folder_Delete(ivars->real_folder, cf_file)) {
return false;
}
String *cfmeta_file = (String*)SSTR_WRAP_UTF8("cfmeta.json", 11);
if (!Folder_Delete(ivars->real_folder, cfmeta_file)) {
return false;
}
}
return true;
}
}
bool_t
CFReader_local_delete(CompoundFileReader *self, const CharBuf *name) {
Hash *record = (Hash*)Hash_Delete(self->records, (Obj*)name);
DECREF(record);
if (record == NULL) {
return Folder_Local_Delete(self->real_folder, name);
}
else {
// Once the number of virtual files falls to 0, remove the compound
// files.
if (Hash_Get_Size(self->records) == 0) {
CharBuf *cf_file = (CharBuf*)ZCB_WRAP_STR("cf.dat", 6);
if (!Folder_Delete(self->real_folder, cf_file)) {
return false;
}
CharBuf *cfmeta_file = (CharBuf*)ZCB_WRAP_STR("cfmeta.json", 11);
if (!Folder_Delete(self->real_folder, cfmeta_file)) {
return false;
}
}
return true;
}
}
bool
LFLock_Maybe_Delete_File_IMP(LockFileLock *self, String *path,
bool delete_mine, bool delete_other) {
LockFileLockIVARS *const ivars = LFLock_IVARS(self);
Folder *folder = ivars->folder;
bool success = false;
// Only delete locks that start with our lock name.
if (!Str_Starts_With_Utf8(path, "locks", 5)) {
return false;
}
StringIterator *iter = Str_Top(path);
StrIter_Advance(iter, 5 + 1);
if (!StrIter_Starts_With(iter, ivars->name)) {
DECREF(iter);
return false;
}
DECREF(iter);
// Attempt to delete dead lock file.
if (Folder_Exists(folder, path)) {
Hash *hash = (Hash*)Json_slurp_json(folder, path);
if (hash != NULL && Obj_Is_A((Obj*)hash, HASH)) {
String *pid_buf = (String*)Hash_Fetch_Utf8(hash, "pid", 3);
String *host = (String*)Hash_Fetch_Utf8(hash, "host", 4);
String *name = (String*)Hash_Fetch_Utf8(hash, "name", 4);
// Match hostname and lock name.
if (host != NULL
&& Str_Is_A(host, STRING)
&& Str_Equals(host, (Obj*)ivars->host)
&& name != NULL
&& Str_Is_A(name, STRING)
&& Str_Equals(name, (Obj*)ivars->name)
&& pid_buf != NULL
&& Str_Is_A(pid_buf, STRING)
) {
// Verify that pid is either mine or dead.
int pid = (int)Str_To_I64(pid_buf);
if ((delete_mine && pid == PID_getpid()) // This process.
|| (delete_other && !PID_active(pid)) // Dead pid.
) {
if (Folder_Delete(folder, path)) {
success = true;
}
else {
String *mess
= MAKE_MESS("Can't delete '%o'", path);
DECREF(hash);
Err_throw_mess(ERR, mess);
}
}
}
}
DECREF(hash);
}
return success;
}
void
Indexer_prepare_commit(Indexer *self)
{
VArray *seg_readers = PolyReader_Get_Seg_Readers(self->polyreader);
uint32_t num_seg_readers = VA_Get_Size(seg_readers);
bool_t merge_happened = false;
if ( !self->write_lock || self->prepared ) {
THROW(ERR, "Can't call Prepare_Commit() more than once");
}
// Merge existing index data.
if (num_seg_readers) {
merge_happened = S_maybe_merge(self, seg_readers);
}
// Add a new segment and write a new snapshot file if...
if ( Seg_Get_Count(self->segment) // Docs/segs added.
|| merge_happened // Some segs merged.
|| !Snapshot_Num_Entries(self->snapshot) // Initializing index.
|| DelWriter_Updated(self->del_writer)
) {
Folder *folder = self->folder;
Schema *schema = self->schema;
Snapshot *snapshot = self->snapshot;
CharBuf *old_schema_name = S_find_schema_file(snapshot);
uint64_t schema_gen = old_schema_name
? IxFileNames_extract_gen(old_schema_name) + 1
: 1;
char base36[StrHelp_MAX_BASE36_BYTES];
CharBuf *new_schema_name;
StrHelp_to_base36(schema_gen, &base36);
new_schema_name = CB_newf("schema_%s.json", base36);
// Finish the segment, write schema file.
SegWriter_Finish(self->seg_writer);
Schema_Write(schema, folder, new_schema_name);
if (old_schema_name) {
Snapshot_Delete_Entry(snapshot, old_schema_name);
}
Snapshot_Add_Entry(snapshot, new_schema_name);
DECREF(new_schema_name);
// Write temporary snapshot file.
DECREF(self->snapfile);
self->snapfile = IxManager_Make_Snapshot_Filename(self->manager);
CB_Cat_Trusted_Str(self->snapfile, ".temp", 5);
Folder_Delete(folder, self->snapfile);
Snapshot_Write_File(snapshot, folder, self->snapfile);
self->needs_commit = true;
}
// Close reader, so that we can delete its files if appropriate.
PolyReader_Close(self->polyreader);
self->prepared = true;
}
void
Indexer_Prepare_Commit_IMP(Indexer *self) {
IndexerIVARS *const ivars = Indexer_IVARS(self);
Vector *seg_readers = PolyReader_Get_Seg_Readers(ivars->polyreader);
size_t num_seg_readers = Vec_Get_Size(seg_readers);
bool merge_happened = false;
if (!ivars->write_lock || ivars->prepared) {
THROW(ERR, "Can't call Prepare_Commit() more than once");
}
// Merge existing index data.
if (num_seg_readers) {
merge_happened = S_maybe_merge(self, seg_readers);
}
// Add a new segment and write a new snapshot file if...
if (Seg_Get_Count(ivars->segment) // Docs/segs added.
|| merge_happened // Some segs merged.
|| !Snapshot_Num_Entries(ivars->snapshot) // Initializing index.
|| DelWriter_Updated(ivars->del_writer)
) {
Folder *folder = ivars->folder;
Schema *schema = ivars->schema;
Snapshot *snapshot = ivars->snapshot;
// Derive snapshot and schema file names.
DECREF(ivars->snapfile);
String *snapfile = IxManager_Make_Snapshot_Filename(ivars->manager);
ivars->snapfile = Str_Cat_Trusted_Utf8(snapfile, ".temp", 5);
DECREF(snapfile);
uint64_t schema_gen = IxFileNames_extract_gen(ivars->snapfile);
char base36[StrHelp_MAX_BASE36_BYTES];
StrHelp_to_base36(schema_gen, &base36);
String *new_schema_name = Str_newf("schema_%s.json", base36);
// Finish the segment, write schema file.
SegWriter_Finish(ivars->seg_writer);
Schema_Write(schema, folder, new_schema_name);
String *old_schema_name = S_find_schema_file(snapshot);
if (old_schema_name) {
Snapshot_Delete_Entry(snapshot, old_schema_name);
}
Snapshot_Add_Entry(snapshot, new_schema_name);
DECREF(new_schema_name);
// Write temporary snapshot file.
Folder_Delete(folder, ivars->snapfile);
Snapshot_Write_File(snapshot, folder, ivars->snapfile);
ivars->needs_commit = true;
}
// Close reader, so that we can delete its files if appropriate.
PolyReader_Close(ivars->polyreader);
ivars->prepared = true;
}
static void
S_clean_up_old_temp_files(CompoundFileWriter *self,
CompoundFileWriterIVARS *ivars) {
UNUSED_VAR(self);
Folder *folder = ivars->folder;
String *cfmeta_temp = (String*)SSTR_WRAP_UTF8("cfmeta.json.temp", 16);
String *cf_file = (String*)SSTR_WRAP_UTF8("cf.dat", 6);
if (Folder_Exists(folder, cf_file)) {
if (!Folder_Delete(folder, cf_file)) {
THROW(ERR, "Can't delete '%o'", cf_file);
}
}
if (Folder_Exists(folder, cfmeta_temp)) {
if (!Folder_Delete(folder, cfmeta_temp)) {
THROW(ERR, "Can't delete '%o'", cfmeta_temp);
}
}
}
bool_t
LFLock_maybe_delete_file(LockFileLock *self, const CharBuf *path,
bool_t delete_mine, bool_t delete_other) {
Folder *folder = self->folder;
bool_t success = false;
ZombieCharBuf *scratch = ZCB_WRAP(path);
// Only delete locks that start with our lock name.
CharBuf *lock_dir_name = (CharBuf*)ZCB_WRAP_STR("locks", 5);
if (!ZCB_Starts_With(scratch, lock_dir_name)) {
return false;
}
ZCB_Nip(scratch, CB_Get_Size(lock_dir_name) + 1);
if (!ZCB_Starts_With(scratch, self->name)) {
return false;
}
// Attempt to delete dead lock file.
if (Folder_Exists(folder, path)) {
Hash *hash = (Hash*)Json_slurp_json(folder, path);
if (hash != NULL && Obj_Is_A((Obj*)hash, HASH)) {
CharBuf *pid_buf = (CharBuf*)Hash_Fetch_Str(hash, "pid", 3);
CharBuf *host = (CharBuf*)Hash_Fetch_Str(hash, "host", 4);
CharBuf *name
= (CharBuf*)Hash_Fetch_Str(hash, "name", 4);
// Match hostname and lock name.
if (host != NULL
&& CB_Equals(host, (Obj*)self->host)
&& name != NULL
&& CB_Equals(name, (Obj*)self->name)
&& pid_buf != NULL
) {
// Verify that pid is either mine or dead.
int pid = (int)CB_To_I64(pid_buf);
if ((delete_mine && pid == PID_getpid()) // This process.
|| (delete_other && !PID_active(pid)) // Dead pid.
) {
if (Folder_Delete(folder, path)) {
success = true;
}
else {
CharBuf *mess
= MAKE_MESS("Can't delete '%o'", path);
DECREF(hash);
Err_throw_mess(ERR, mess);
}
}
}
}
DECREF(hash);
}
return success;
}
void
Schema_Write_IMP(Schema *self, Folder *folder, String *filename) {
Hash *dump = Schema_Dump(self);
String *schema_temp = SSTR_WRAP_C("schema.temp");
bool success;
Folder_Delete(folder, schema_temp); // Just in case.
Json_spew_json((Obj*)dump, folder, schema_temp);
success = Folder_Rename(folder, schema_temp, filename);
DECREF(dump);
if (!success) { RETHROW(INCREF(Err_get_error())); }
}
void
BGMerger_Commit_IMP(BackgroundMerger *self) {
BackgroundMergerIVARS *const ivars = BGMerger_IVARS(self);
// Safety check.
if (!ivars->merge_lock) {
THROW(ERR, "Can't call commit() more than once");
}
if (!ivars->prepared) {
BGMerger_Prepare_Commit(self);
}
if (ivars->needs_commit) {
bool success = false;
String *temp_snapfile = ivars->snapfile;
// Rename temp snapshot file.
size_t ext_len = sizeof(".temp") - 1;
size_t snapfile_len = Str_Length(temp_snapfile);
if (snapfile_len <= ext_len) {
THROW(ERR, "Invalid snapfile name: %o", temp_snapfile);
}
ivars->snapfile = Str_SubString(temp_snapfile, 0,
snapfile_len - ext_len);
success = Folder_Hard_Link(ivars->folder, temp_snapfile,
ivars->snapfile);
Snapshot_Set_Path(ivars->snapshot, ivars->snapfile);
if (!success) {
String *mess = Str_newf("Can't create hard link from %o to %o",
temp_snapfile, ivars->snapfile);
DECREF(temp_snapfile);
Err_throw_mess(ERR, mess);
}
if (!Folder_Delete(ivars->folder, temp_snapfile)) {
String *mess = Str_newf("Can't delete %o", temp_snapfile);
DECREF(temp_snapfile);
Err_throw_mess(ERR, mess);
}
DECREF(temp_snapfile);
}
// Release the merge lock and remove the merge data file.
S_release_merge_lock(self);
IxManager_Remove_Merge_Data(ivars->manager);
if (ivars->needs_commit) {
// Purge obsolete files.
FilePurger_Purge(ivars->file_purger);
}
// Release the write lock.
S_release_write_lock(self);
}
static void
test_Delete(TestBatch *batch)
{
Folder *folder = (Folder*)RAMFolder_new(NULL);
FileHandle *fh;
bool_t result;
Folder_MkDir(folder, &foo);
Folder_MkDir(folder, &foo_bar);
fh = Folder_Open_FileHandle(folder, &boffo, FH_CREATE | FH_WRITE_ONLY);
DECREF(fh);
fh = Folder_Open_FileHandle(folder, &foo_boffo,
FH_CREATE | FH_WRITE_ONLY);
DECREF(fh);
Err_set_error(NULL);
result = Folder_Delete(folder, &banana);
TEST_FALSE(batch, result, "Delete on non-existent entry returns false");
Err_set_error(NULL);
result = Folder_Delete(folder, &foo);
TEST_FALSE(batch, result, "Delete on non-empty dir returns false");
TEST_TRUE(batch, Folder_Delete(folder, &foo_boffo),
"Delete nested file");
TEST_FALSE(batch, Folder_Exists(folder, &foo_boffo),
"File is really gone");
TEST_TRUE(batch, Folder_Delete(folder, &foo_bar),
"Delete nested dir");
TEST_FALSE(batch, Folder_Exists(folder, &foo_bar),
"Dir is really gone");
TEST_TRUE(batch, Folder_Delete(folder, &foo), "Delete empty dir");
TEST_FALSE(batch, Folder_Exists(folder, &foo), "Dir is really gone");
DECREF(folder);
}
void
FilePurger_purge(FilePurger *self)
{
Lock *deletion_lock = IxManager_Make_Deletion_Lock(self->manager);
// Obtain deletion lock, purge files, release deletion lock.
Lock_Clear_Stale(deletion_lock);
if (Lock_Obtain(deletion_lock)) {
Folder *folder = self->folder;
Hash *failures = Hash_new(0);
VArray *purgables;
VArray *snapshots;
S_discover_unused(self, &purgables, &snapshots);
// Attempt to delete entries -- if failure, no big deal, just try
// again later. Proceed in reverse lexical order so that directories
// get deleted after they've been emptied.
VA_Sort(purgables, NULL, NULL);
for (uint32_t i = VA_Get_Size(purgables); i--; ) {
CharBuf *entry = (CharBuf*)VA_fetch(purgables, i);
if (Hash_Fetch(self->disallowed, (Obj*)entry)) { continue; }
if (!Folder_Delete(folder, entry)) {
if (Folder_Exists(folder, entry)) {
Hash_Store(failures, (Obj*)entry, INCREF(&EMPTY));
}
}
}
for (uint32_t i = 0, max = VA_Get_Size(snapshots); i < max; i++) {
Snapshot *snapshot = (Snapshot*)VA_Fetch(snapshots, i);
bool_t snapshot_has_failures = false;
if (Hash_Get_Size(failures)) {
// Only delete snapshot files if all of their entries were
// successfully deleted.
VArray *entries = Snapshot_List(snapshot);
for (uint32_t j = VA_Get_Size(entries); j--; ) {
CharBuf *entry = (CharBuf*)VA_Fetch(entries, j);
if (Hash_Fetch(failures, (Obj*)entry)) {
snapshot_has_failures = true;
break;
}
}
DECREF(entries);
}
if (!snapshot_has_failures) {
CharBuf *snapfile = Snapshot_Get_Path(snapshot);
Folder_Delete(folder, snapfile);
}
}
DECREF(failures);
DECREF(purgables);
DECREF(snapshots);
Lock_Release(deletion_lock);
}
else {
WARN("Can't obtain deletion lock, skipping deletion of "
"obsolete files");
}
DECREF(deletion_lock);
}
bool_t
LFLock_request(LockFileLock *self) {
Hash *file_data;
bool_t wrote_json;
bool_t success = false;
bool_t deletion_failed = false;
if (Folder_Exists(self->folder, self->lock_path)) {
Err_set_error((Err*)LockErr_new(CB_newf("Can't obtain lock: '%o' exists",
self->lock_path)));
return false;
}
// Create the "locks" subdirectory if necessary.
CharBuf *lock_dir_name = (CharBuf*)ZCB_WRAP_STR("locks", 5);
if (!Folder_Exists(self->folder, lock_dir_name)) {
if (!Folder_MkDir(self->folder, lock_dir_name)) {
Err *mkdir_err = (Err*)CERTIFY(Err_get_error(), ERR);
LockErr *err = LockErr_new(CB_newf("Can't create 'locks' directory: %o",
Err_Get_Mess(mkdir_err)));
// Maybe our attempt failed because another process succeeded.
if (Folder_Find_Folder(self->folder, lock_dir_name)) {
DECREF(err);
}
else {
// Nope, everything failed, so bail out.
Err_set_error((Err*)err);
return false;
}
}
}
// Prepare to write pid, lock name, and host to the lock file as JSON.
file_data = Hash_new(3);
Hash_Store_Str(file_data, "pid", 3,
(Obj*)CB_newf("%i32", (int32_t)PID_getpid()));
Hash_Store_Str(file_data, "host", 4, INCREF(self->host));
Hash_Store_Str(file_data, "name", 4, INCREF(self->name));
// Write to a temporary file, then use the creation of a hard link to
// ensure atomic but non-destructive creation of the lockfile with its
// complete contents.
wrote_json = Json_spew_json((Obj*)file_data, self->folder, self->link_path);
if (wrote_json) {
success = Folder_Hard_Link(self->folder, self->link_path,
self->lock_path);
if (!success) {
Err *hard_link_err = (Err*)CERTIFY(Err_get_error(), ERR);
Err_set_error((Err*)LockErr_new(CB_newf("Failed to obtain lock at '%o': %o",
self->lock_path,
Err_Get_Mess(hard_link_err))));
}
deletion_failed = !Folder_Delete(self->folder, self->link_path);
}
else {
Err *spew_json_err = (Err*)CERTIFY(Err_get_error(), ERR);
Err_set_error((Err*)LockErr_new(CB_newf("Failed to obtain lock at '%o': %o",
self->lock_path,
Err_Get_Mess(spew_json_err))));
}
DECREF(file_data);
// Verify that our temporary file got zapped.
if (wrote_json && deletion_failed) {
CharBuf *mess = MAKE_MESS("Failed to delete '%o'", self->link_path);
Err_throw_mess(ERR, mess);
}
return success;
}
bool
LFLock_Request_IMP(LockFileLock *self) {
LockFileLockIVARS *const ivars = LFLock_IVARS(self);
bool success = false;
if (Folder_Exists(ivars->folder, ivars->lock_path)) {
Err_set_error((Err*)LockErr_new(Str_newf("Can't obtain lock: '%o' exists",
ivars->lock_path)));
return false;
}
// Create the "locks" subdirectory if necessary.
String *lock_dir_name = (String*)SSTR_WRAP_UTF8("locks", 5);
if (!Folder_Exists(ivars->folder, lock_dir_name)) {
if (!Folder_MkDir(ivars->folder, lock_dir_name)) {
Err *mkdir_err = (Err*)CERTIFY(Err_get_error(), ERR);
LockErr *err = LockErr_new(Str_newf("Can't create 'locks' directory: %o",
Err_Get_Mess(mkdir_err)));
// Maybe our attempt failed because another process succeeded.
if (Folder_Find_Folder(ivars->folder, lock_dir_name)) {
DECREF(err);
}
else {
// Nope, everything failed, so bail out.
Err_set_error((Err*)err);
return false;
}
}
}
// Prepare to write pid, lock name, and host to the lock file as JSON.
Hash *file_data = Hash_new(3);
Hash_Store_Utf8(file_data, "pid", 3,
(Obj*)Str_newf("%i32", (int32_t)PID_getpid()));
Hash_Store_Utf8(file_data, "host", 4, INCREF(ivars->host));
Hash_Store_Utf8(file_data, "name", 4, INCREF(ivars->name));
String *json = Json_to_json((Obj*)file_data);
DECREF(file_data);
// Write to a temporary file, then use the creation of a hard link to
// ensure atomic but non-destructive creation of the lockfile with its
// complete contents.
OutStream *outstream = Folder_Open_Out(ivars->folder, ivars->link_path);
if (!outstream) {
ERR_ADD_FRAME(Err_get_error());
DECREF(json);
return false;
}
struct lockfile_context context;
context.outstream = outstream;
context.json = json;
Err *json_error = Err_trap(S_write_lockfile_json, &context);
bool wrote_json = !json_error;
DECREF(outstream);
DECREF(json);
if (wrote_json) {
success = Folder_Hard_Link(ivars->folder, ivars->link_path,
ivars->lock_path);
if (!success) {
Err *hard_link_err = (Err*)CERTIFY(Err_get_error(), ERR);
Err_set_error((Err*)LockErr_new(Str_newf("Failed to obtain lock at '%o': %o",
ivars->lock_path,
Err_Get_Mess(hard_link_err))));
}
}
else {
Err_set_error((Err*)LockErr_new(Str_newf("Failed to obtain lock at '%o': %o",
ivars->lock_path,
Err_Get_Mess(json_error))));
DECREF(json_error);
}
// Verify that our temporary file got zapped.
bool deletion_failed = !Folder_Delete(ivars->folder, ivars->link_path);
if (deletion_failed) {
String *mess = MAKE_MESS("Failed to delete '%o'", ivars->link_path);
Err_throw_mess(ERR, mess);
}
return success;
}
bool
IxManager_Remove_Merge_Data_IMP(IndexManager *self) {
IndexManagerIVARS *const ivars = IxManager_IVARS(self);
String *merge_json = SSTR_WRAP_C("merge.json");
return Folder_Delete(ivars->folder, merge_json) != 0;
}
void
SortWriter_finish(SortWriter *self) {
SortWriterIVARS *const ivars = SortWriter_IVARS(self);
VArray *const field_writers = ivars->field_writers;
// If we have no data, bail out.
if (!ivars->temp_ord_out) { return; }
// If we've either flushed or added segments, flush everything so that any
// one field can use the entire margin up to mem_thresh.
if (ivars->flush_at_finish) {
for (uint32_t i = 1, max = VA_Get_Size(field_writers); i < max; i++) {
SortFieldWriter *field_writer
= (SortFieldWriter*)VA_Fetch(field_writers, i);
if (field_writer) {
SortFieldWriter_Flush(field_writer);
}
}
}
// Close down temp streams.
OutStream_Close(ivars->temp_ord_out);
OutStream_Close(ivars->temp_ix_out);
OutStream_Close(ivars->temp_dat_out);
for (uint32_t i = 1, max = VA_Get_Size(field_writers); i < max; i++) {
SortFieldWriter *field_writer
= (SortFieldWriter*)VA_Delete(field_writers, i);
if (field_writer) {
CharBuf *field = Seg_Field_Name(ivars->segment, i);
SortFieldWriter_Flip(field_writer);
int32_t count = SortFieldWriter_Finish(field_writer);
Hash_Store(ivars->counts, (Obj*)field,
(Obj*)CB_newf("%i32", count));
int32_t null_ord = SortFieldWriter_Get_Null_Ord(field_writer);
if (null_ord != -1) {
Hash_Store(ivars->null_ords, (Obj*)field,
(Obj*)CB_newf("%i32", null_ord));
}
int32_t ord_width = SortFieldWriter_Get_Ord_Width(field_writer);
Hash_Store(ivars->ord_widths, (Obj*)field,
(Obj*)CB_newf("%i32", ord_width));
}
DECREF(field_writer);
}
VA_Clear(field_writers);
// Store metadata.
Seg_Store_Metadata_Str(ivars->segment, "sort", 4,
(Obj*)SortWriter_Metadata(self));
// Clean up.
Folder *folder = ivars->folder;
CharBuf *seg_name = Seg_Get_Name(ivars->segment);
CharBuf *path = CB_newf("%o/sort_ord_temp", seg_name);
Folder_Delete(folder, path);
CB_setf(path, "%o/sort_ix_temp", seg_name);
Folder_Delete(folder, path);
CB_setf(path, "%o/sort_dat_temp", seg_name);
Folder_Delete(folder, path);
DECREF(path);
}
bool_t
IxManager_remove_merge_data(IndexManager *self)
{
ZombieCharBuf *merge_json = ZCB_WRAP_STR("merge.json", 10);
return Folder_Delete(self->folder, (CharBuf*)merge_json) != 0;
}
void
BGMerger_Prepare_Commit_IMP(BackgroundMerger *self) {
BackgroundMergerIVARS *const ivars = BGMerger_IVARS(self);
Vector *seg_readers = PolyReader_Get_Seg_Readers(ivars->polyreader);
uint32_t num_seg_readers = Vec_Get_Size(seg_readers);
uint32_t segs_merged = 0;
if (ivars->prepared) {
THROW(ERR, "Can't call Prepare_Commit() more than once");
}
// Maybe merge existing index data.
if (num_seg_readers) {
segs_merged = S_maybe_merge(self);
}
if (!segs_merged) {
// Nothing merged. Leave `needs_commit` false and bail out.
ivars->prepared = true;
return;
}
// Finish the segment and write a new snapshot file.
else {
Folder *folder = ivars->folder;
Snapshot *snapshot = ivars->snapshot;
// Write out new deletions.
if (DelWriter_Updated(ivars->del_writer)) {
// Only write out if they haven't all been applied.
if (segs_merged != num_seg_readers) {
DelWriter_Finish(ivars->del_writer);
}
}
// Finish the segment.
SegWriter_Finish(ivars->seg_writer);
// Grab the write lock.
S_obtain_write_lock(self);
if (!ivars->write_lock) {
RETHROW(INCREF(Err_get_error()));
}
// Write temporary snapshot file.
DECREF(ivars->snapfile);
String *snapfile = IxManager_Make_Snapshot_Filename(ivars->manager);
ivars->snapfile = Str_Cat_Trusted_Utf8(snapfile, ".temp", 5);
DECREF(snapfile);
Folder_Delete(folder, ivars->snapfile);
Snapshot_Write_File(snapshot, folder, ivars->snapfile);
// Determine whether the index has been updated while this background
// merge process was running.
String *start_snapfile
= Snapshot_Get_Path(PolyReader_Get_Snapshot(ivars->polyreader));
Snapshot *latest_snapshot
= Snapshot_Read_File(Snapshot_new(), ivars->folder, NULL);
String *latest_snapfile = Snapshot_Get_Path(latest_snapshot);
bool index_updated
= !Str_Equals(start_snapfile, (Obj*)latest_snapfile);
if (index_updated) {
/* See if new deletions have been applied since this
* background merge process started against any of the
* segments we just merged away. If that's true, we need to
* write another segment which applies the deletions against
* the new composite segment.
*/
S_merge_updated_deletions(self);
// Add the fresh content to our snapshot. (It's important to
// run this AFTER S_merge_updated_deletions, because otherwise
// we couldn't tell whether the deletion counts changed.)
Vector *files = Snapshot_List(latest_snapshot);
for (uint32_t i = 0, max = Vec_Get_Size(files); i < max; i++) {
String *file = (String*)Vec_Fetch(files, i);
if (Str_Starts_With_Utf8(file, "seg_", 4)) {
int64_t gen = (int64_t)IxFileNames_extract_gen(file);
if (gen > ivars->cutoff) {
Snapshot_Add_Entry(ivars->snapshot, file);
}
}
}
DECREF(files);
// Since the snapshot content has changed, we need to rewrite it.
Folder_Delete(folder, ivars->snapfile);
Snapshot_Write_File(snapshot, folder, ivars->snapfile);
}
DECREF(latest_snapshot);
ivars->needs_commit = true;
}
// Close reader, so that we can delete its files if appropriate.
PolyReader_Close(ivars->polyreader);
ivars->prepared = true;
}
static void
test_open(TestBatchRunner *runner) {
Folder *real_folder;
CompoundFileReader *cf_reader;
Hash *metadata;
Err_set_error(NULL);
real_folder = S_folder_with_contents();
Folder_Delete(real_folder, cfmeta_file);
cf_reader = CFReader_open(real_folder);
TEST_TRUE(runner, cf_reader == NULL,
"Return NULL when cfmeta file missing");
TEST_TRUE(runner, Err_get_error() != NULL,
"Set global error when cfmeta file missing");
DECREF(real_folder);
Err_set_error(NULL);
real_folder = S_folder_with_contents();
Folder_Delete(real_folder, cf_file);
cf_reader = CFReader_open(real_folder);
TEST_TRUE(runner, cf_reader == NULL,
"Return NULL when cf.dat file missing");
TEST_TRUE(runner, Err_get_error() != NULL,
"Set global error when cf.dat file missing");
DECREF(real_folder);
Err_set_error(NULL);
real_folder = S_folder_with_contents();
metadata = (Hash*)Json_slurp_json(real_folder, cfmeta_file);
Hash_Store_Utf8(metadata, "format", 6, (Obj*)Str_newf("%i32", -1));
Folder_Delete(real_folder, cfmeta_file);
Json_spew_json((Obj*)metadata, real_folder, cfmeta_file);
cf_reader = CFReader_open(real_folder);
TEST_TRUE(runner, cf_reader == NULL,
"Return NULL when format is invalid");
TEST_TRUE(runner, Err_get_error() != NULL,
"Set global error when format is invalid");
Err_set_error(NULL);
Hash_Store_Utf8(metadata, "format", 6, (Obj*)Str_newf("%i32", 1000));
Folder_Delete(real_folder, cfmeta_file);
Json_spew_json((Obj*)metadata, real_folder, cfmeta_file);
cf_reader = CFReader_open(real_folder);
TEST_TRUE(runner, cf_reader == NULL,
"Return NULL when format is too recent");
TEST_TRUE(runner, Err_get_error() != NULL,
"Set global error when format too recent");
Err_set_error(NULL);
DECREF(Hash_Delete_Utf8(metadata, "format", 6));
Folder_Delete(real_folder, cfmeta_file);
Json_spew_json((Obj*)metadata, real_folder, cfmeta_file);
cf_reader = CFReader_open(real_folder);
TEST_TRUE(runner, cf_reader == NULL,
"Return NULL when format key is missing");
TEST_TRUE(runner, Err_get_error() != NULL,
"Set global error when format key is missing");
Hash_Store_Utf8(metadata, "format", 6,
(Obj*)Str_newf("%i32", CFWriter_current_file_format));
DECREF(Hash_Delete_Utf8(metadata, "files", 5));
Folder_Delete(real_folder, cfmeta_file);
Json_spew_json((Obj*)metadata, real_folder, cfmeta_file);
cf_reader = CFReader_open(real_folder);
TEST_TRUE(runner, cf_reader == NULL,
"Return NULL when files key is missing");
TEST_TRUE(runner, Err_get_error() != NULL,
"Set global error when files key is missing");
DECREF(metadata);
DECREF(real_folder);
}
static void
S_do_consolidate(CompoundFileWriter *self, CompoundFileWriterIVARS *ivars) {
UNUSED_VAR(self);
Folder *folder = ivars->folder;
Hash *metadata = Hash_new(0);
Hash *sub_files = Hash_new(0);
Vector *files = Folder_List(folder, NULL);
Vector *merged = Vec_new(Vec_Get_Size(files));
String *cf_file = (String*)SSTR_WRAP_UTF8("cf.dat", 6);
OutStream *outstream = Folder_Open_Out(folder, (String*)cf_file);
bool rename_success;
if (!outstream) { RETHROW(INCREF(Err_get_error())); }
// Start metadata.
Hash_Store_Utf8(metadata, "files", 5, INCREF(sub_files));
Hash_Store_Utf8(metadata, "format", 6,
(Obj*)Str_newf("%i32", CFWriter_current_file_format));
Vec_Sort(files);
for (uint32_t i = 0, max = Vec_Get_Size(files); i < max; i++) {
String *infilename = (String*)Vec_Fetch(files, i);
if (!Str_Ends_With_Utf8(infilename, ".json", 5)) {
InStream *instream = Folder_Open_In(folder, infilename);
Hash *file_data = Hash_new(2);
int64_t offset, len;
if (!instream) { RETHROW(INCREF(Err_get_error())); }
// Absorb the file.
offset = OutStream_Tell(outstream);
OutStream_Absorb(outstream, instream);
len = OutStream_Tell(outstream) - offset;
// Record offset and length.
Hash_Store_Utf8(file_data, "offset", 6,
(Obj*)Str_newf("%i64", offset));
Hash_Store_Utf8(file_data, "length", 6,
(Obj*)Str_newf("%i64", len));
Hash_Store(sub_files, infilename, (Obj*)file_data);
Vec_Push(merged, INCREF(infilename));
// Add filler NULL bytes so that every sub-file begins on a file
// position multiple of 8.
OutStream_Align(outstream, 8);
InStream_Close(instream);
DECREF(instream);
}
}
// Write metadata to cfmeta file.
String *cfmeta_temp = (String*)SSTR_WRAP_UTF8("cfmeta.json.temp", 16);
String *cfmeta_file = (String*)SSTR_WRAP_UTF8("cfmeta.json", 11);
Json_spew_json((Obj*)metadata, (Folder*)ivars->folder, cfmeta_temp);
rename_success = Folder_Rename(ivars->folder, cfmeta_temp, cfmeta_file);
if (!rename_success) { RETHROW(INCREF(Err_get_error())); }
// Clean up.
OutStream_Close(outstream);
DECREF(outstream);
DECREF(files);
DECREF(metadata);
/*
HashIterator *iter = HashIter_new(sub_files);
while (HashIter_Next(iter)) {
String *merged_file = HashIter_Get_Key(iter);
if (!Folder_Delete(folder, merged_file)) {
String *mess = MAKE_MESS("Can't delete '%o'", merged_file);
DECREF(sub_files);
Err_throw_mess(ERR, mess);
}
}
DECREF(iter);
*/
DECREF(sub_files);
for (uint32_t i = 0, max = Vec_Get_Size(merged); i < max; i++) {
String *merged_file = (String*)Vec_Fetch(merged, i);
if (!Folder_Delete(folder, merged_file)) {
String *mess = MAKE_MESS("Can't delete '%o'", merged_file);
DECREF(merged);
Err_throw_mess(ERR, mess);
}
}
DECREF(merged);
}
