void
Indexer_commit(Indexer *self)
{
// Safety check.
if ( !self->write_lock ) {
THROW(ERR, "Can't call commit() more than once");
}
if (!self->prepared) {
Indexer_Prepare_Commit(self);
}
if (self->needs_commit) {
bool_t success;
// Rename temp snapshot file.
CharBuf *temp_snapfile = CB_Clone(self->snapfile);
CB_Chop(self->snapfile, sizeof(".temp") - 1);
Snapshot_Set_Path(self->snapshot, self->snapfile);
success = Folder_Rename(self->folder, temp_snapfile, self->snapfile);
DECREF(temp_snapfile);
if (!success) { RETHROW(INCREF(Err_get_error())); }
// Purge obsolete files.
FilePurger_Purge(self->file_purger);
}
// Release locks, invalidating the Indexer.
S_release_merge_lock(self);
S_release_write_lock(self);
}
void
Schema_Write_IMP(Schema *self, Folder *folder, String *filename) {
Hash *dump = Schema_Dump(self);
String *schema_temp = SSTR_WRAP_C("schema.temp");
bool success;
Folder_Delete(folder, schema_temp); // Just in case.
Json_spew_json((Obj*)dump, folder, schema_temp);
success = Folder_Rename(folder, schema_temp, filename);
DECREF(dump);
if (!success) { RETHROW(INCREF(Err_get_error())); }
}
void
Indexer_Commit_IMP(Indexer *self) {
IndexerIVARS *const ivars = Indexer_IVARS(self);
// Safety check.
if (!ivars->write_lock) {
THROW(ERR, "Can't call commit() more than once");
}
if (!ivars->prepared) {
Indexer_Prepare_Commit(self);
}
if (ivars->needs_commit) {
bool success;
// Rename temp snapshot file.
String *temp_snapfile = ivars->snapfile;
size_t ext_len = sizeof(".temp") - 1;
size_t snapfile_len = Str_Length(temp_snapfile);
if (snapfile_len <= ext_len) {
THROW(ERR, "Invalid snapfile name: %o", temp_snapfile);
}
ivars->snapfile = Str_SubString(temp_snapfile, 0,
snapfile_len - ext_len);
Snapshot_Set_Path(ivars->snapshot, ivars->snapfile);
success = Folder_Rename(ivars->folder, temp_snapfile, ivars->snapfile);
DECREF(temp_snapfile);
if (!success) { RETHROW(INCREF(Err_get_error())); }
// Purge obsolete files.
FilePurger_Purge(ivars->file_purger);
}
// Release locks, invalidating the Indexer.
S_release_merge_lock(self);
S_release_write_lock(self);
}
static void
test_path_handling(TestBatchRunner *runner) {
Snapshot *snapshot = Snapshot_new();
Folder *folder = (Folder*)RAMFolder_new(NULL);
String *snap = (String*)SSTR_WRAP_UTF8("snap", 4);
String *crackle = (String*)SSTR_WRAP_UTF8("crackle", 7);
Snapshot_Write_File(snapshot, folder, snap);
TEST_TRUE(runner, Str_Equals(snap, (Obj*)Snapshot_Get_Path(snapshot)),
"Write_File() sets path as a side effect");
Folder_Rename(folder, snap, crackle);
Snapshot_Read_File(snapshot, folder, crackle);
TEST_TRUE(runner, Str_Equals(crackle, (Obj*)Snapshot_Get_Path(snapshot)),
"Read_File() sets path as a side effect");
Snapshot_Set_Path(snapshot, snap);
TEST_TRUE(runner, Str_Equals(snap, (Obj*)Snapshot_Get_Path(snapshot)),
"Set_Path()");
DECREF(folder);
DECREF(snapshot);
}
PolyReader*
PolyReader_do_open(PolyReader *self, Obj *index, Snapshot *snapshot,
IndexManager *manager) {
PolyReaderIVARS *const ivars = PolyReader_IVARS(self);
Folder *folder = S_derive_folder(index);
uint64_t last_gen = 0;
PolyReader_init(self, NULL, folder, snapshot, manager, NULL);
DECREF(folder);
if (manager) {
if (!S_obtain_deletion_lock(self)) {
DECREF(self);
THROW(LOCKERR, "Couldn't get deletion lock");
}
}
while (1) {
CharBuf *target_snap_file;
// If a Snapshot was supplied, use its file.
if (snapshot) {
target_snap_file = Snapshot_Get_Path(snapshot);
if (!target_snap_file) {
THROW(ERR, "Supplied snapshot objects must not be empty");
}
else {
CB_Inc_RefCount(target_snap_file);
}
}
else {
// Otherwise, pick the most recent snap file.
target_snap_file = IxFileNames_latest_snapshot(folder);
// No snap file? Looks like the index is empty. We can stop now
// and return NULL.
if (!target_snap_file) { break; }
}
// Derive "generation" of this snapshot file from its name.
uint64_t gen = IxFileNames_extract_gen(target_snap_file);
// Get a read lock on the most recent snapshot file if indicated.
if (manager) {
if (!S_obtain_read_lock(self, target_snap_file)) {
DECREF(self);
THROW(LOCKERR, "Couldn't get read lock for %o",
target_snap_file);
}
}
// Testing only.
if (PolyReader_race_condition_debug1) {
ZombieCharBuf *temp = ZCB_WRAP_STR("temp", 4);
if (Folder_Exists(folder, (CharBuf*)temp)) {
bool success = Folder_Rename(folder, (CharBuf*)temp,
PolyReader_race_condition_debug1);
if (!success) { RETHROW(INCREF(Err_get_error())); }
}
PolyReader_debug1_num_passes++;
}
// If a Snapshot object was passed in, the file has already been read.
// If that's not the case, we must read the file we just picked.
if (!snapshot) {
struct try_read_snapshot_context context;
context.snapshot = ivars->snapshot;
context.folder = folder;
context.path = target_snap_file;
Err *error = Err_trap(S_try_read_snapshot, &context);
if (error) {
S_release_read_lock(self);
DECREF(target_snap_file);
if (last_gen < gen) { // Index updated, so try again.
DECREF(error);
last_gen = gen;
continue;
}
else { // Real error.
if (manager) { S_release_deletion_lock(self); }
RETHROW(error);
}
}
}
/* It's possible, though unlikely, for an Indexer to delete files
* out from underneath us after the snapshot file is read but before
* we've got SegReaders holding open all the required files. If we
* failed to open something, see if we can find a newer snapshot file.
* If we can, then the exception was due to the race condition. If
* not, we have a real exception, so throw an error. */
struct try_open_elements_context context;
context.self = self;
context.seg_readers = NULL;
Err *error = Err_trap(S_try_open_elements, &context);
if (error) {
S_release_read_lock(self);
DECREF(target_snap_file);
if (last_gen < gen) { // Index updated, so try again.
DECREF(error);
last_gen = gen;
}
else { // Real error.
if (manager) { S_release_deletion_lock(self); }
RETHROW(error);
}
}
else { // Succeeded.
S_init_sub_readers(self, (VArray*)context.seg_readers);
DECREF(context.seg_readers);
DECREF(target_snap_file);
break;
}
}
if (manager) { S_release_deletion_lock(self); }
return self;
}
static void
S_do_consolidate(CompoundFileWriter *self, CompoundFileWriterIVARS *ivars) {
UNUSED_VAR(self);
Folder *folder = ivars->folder;
Hash *metadata = Hash_new(0);
Hash *sub_files = Hash_new(0);
Vector *files = Folder_List(folder, NULL);
Vector *merged = Vec_new(Vec_Get_Size(files));
String *cf_file = (String*)SSTR_WRAP_UTF8("cf.dat", 6);
OutStream *outstream = Folder_Open_Out(folder, (String*)cf_file);
bool rename_success;
if (!outstream) { RETHROW(INCREF(Err_get_error())); }
// Start metadata.
Hash_Store_Utf8(metadata, "files", 5, INCREF(sub_files));
Hash_Store_Utf8(metadata, "format", 6,
(Obj*)Str_newf("%i32", CFWriter_current_file_format));
Vec_Sort(files);
for (uint32_t i = 0, max = Vec_Get_Size(files); i < max; i++) {
String *infilename = (String*)Vec_Fetch(files, i);
if (!Str_Ends_With_Utf8(infilename, ".json", 5)) {
InStream *instream = Folder_Open_In(folder, infilename);
Hash *file_data = Hash_new(2);
int64_t offset, len;
if (!instream) { RETHROW(INCREF(Err_get_error())); }
// Absorb the file.
offset = OutStream_Tell(outstream);
OutStream_Absorb(outstream, instream);
len = OutStream_Tell(outstream) - offset;
// Record offset and length.
Hash_Store_Utf8(file_data, "offset", 6,
(Obj*)Str_newf("%i64", offset));
Hash_Store_Utf8(file_data, "length", 6,
(Obj*)Str_newf("%i64", len));
Hash_Store(sub_files, infilename, (Obj*)file_data);
Vec_Push(merged, INCREF(infilename));
// Add filler NULL bytes so that every sub-file begins on a file
// position multiple of 8.
OutStream_Align(outstream, 8);
InStream_Close(instream);
DECREF(instream);
}
}
// Write metadata to cfmeta file.
String *cfmeta_temp = (String*)SSTR_WRAP_UTF8("cfmeta.json.temp", 16);
String *cfmeta_file = (String*)SSTR_WRAP_UTF8("cfmeta.json", 11);
Json_spew_json((Obj*)metadata, (Folder*)ivars->folder, cfmeta_temp);
rename_success = Folder_Rename(ivars->folder, cfmeta_temp, cfmeta_file);
if (!rename_success) { RETHROW(INCREF(Err_get_error())); }
// Clean up.
OutStream_Close(outstream);
DECREF(outstream);
DECREF(files);
DECREF(metadata);
/*
HashIterator *iter = HashIter_new(sub_files);
while (HashIter_Next(iter)) {
String *merged_file = HashIter_Get_Key(iter);
if (!Folder_Delete(folder, merged_file)) {
String *mess = MAKE_MESS("Can't delete '%o'", merged_file);
DECREF(sub_files);
Err_throw_mess(ERR, mess);
}
}
DECREF(iter);
*/
DECREF(sub_files);
for (uint32_t i = 0, max = Vec_Get_Size(merged); i < max; i++) {
String *merged_file = (String*)Vec_Fetch(merged, i);
if (!Folder_Delete(folder, merged_file)) {
String *mess = MAKE_MESS("Can't delete '%o'", merged_file);
DECREF(merged);
Err_throw_mess(ERR, mess);
}
}
DECREF(merged);
}
