SegReader*
SegReader_init(SegReader *self, Schema *schema, Folder *folder,
Snapshot *snapshot, Vector *segments, int32_t seg_tick) {
Segment *segment;
IxReader_init((IndexReader*)self, schema, folder, snapshot, segments,
seg_tick, NULL);
SegReaderIVARS *const ivars = SegReader_IVARS(self);
segment = SegReader_Get_Segment(self);
ivars->doc_max = (int32_t)Seg_Get_Count(segment);
ivars->seg_name = (String*)INCREF(Seg_Get_Name(segment));
ivars->seg_num = Seg_Get_Number(segment);
Err *error = Err_trap(S_try_init_components, self);
if (error) {
// An error occurred, so clean up self and rethrow the exception.
DECREF(self);
RETHROW(error);
}
DeletionsReader *del_reader
= (DeletionsReader*)Hash_Fetch(
ivars->components, Class_Get_Name(DELETIONSREADER));
ivars->del_count = del_reader ? DelReader_Del_Count(del_reader) : 0;
return self;
}
bool
TestBatchRunner_Run_Batch_IMP(TestBatchRunner *self, TestBatch *batch) {
struct try_run_tests_context args;
args.runner = self;
args.batch = batch;
Err *err = Err_trap(S_try_run_tests, &args);
bool failed = false;
if (err) {
failed = true;
String *mess = Err_Get_Mess(err);
Err_warn_mess((String*)INCREF(mess));
}
if (self->num_failed > 0) {
failed = true;
TestFormatter_batch_comment(self->formatter, "%d/%d tests failed.\n",
self->num_failed, self->test_num);
}
if (self->test_num != self->num_planned) {
failed = true;
TestFormatter_batch_comment(self->formatter,
"Bad plan: You planned %d tests but ran"
" %d.\n",
self->num_planned, self->test_num);
}
return !failed;
}
static void
S_verify_abstract_error(TestBatch *batch, Err_Attempt_t routine,
void *context, const char *name) {
char message[100];
sprintf(message, "%s() is abstract", name);
Err *error = Err_trap(routine, context);
TEST_TRUE(batch, error != NULL
&& Err_Is_A(error, ERR)
&& CB_Find_Str(Err_Get_Mess(error), "bstract", 7) != -1,
message);
DECREF(error);
}
bool
LFLock_Request_IMP(LockFileLock *self) {
LockFileLockIVARS *const ivars = LFLock_IVARS(self);
bool success = false;
if (Folder_Exists(ivars->folder, ivars->lock_path)) {
Err_set_error((Err*)LockErr_new(Str_newf("Can't obtain lock: '%o' exists",
ivars->lock_path)));
return false;
}
// Create the "locks" subdirectory if necessary.
String *lock_dir_name = (String*)SSTR_WRAP_UTF8("locks", 5);
if (!Folder_Exists(ivars->folder, lock_dir_name)) {
if (!Folder_MkDir(ivars->folder, lock_dir_name)) {
Err *mkdir_err = (Err*)CERTIFY(Err_get_error(), ERR);
LockErr *err = LockErr_new(Str_newf("Can't create 'locks' directory: %o",
Err_Get_Mess(mkdir_err)));
// Maybe our attempt failed because another process succeeded.
if (Folder_Find_Folder(ivars->folder, lock_dir_name)) {
DECREF(err);
}
else {
// Nope, everything failed, so bail out.
Err_set_error((Err*)err);
return false;
}
}
}
// Prepare to write pid, lock name, and host to the lock file as JSON.
Hash *file_data = Hash_new(3);
Hash_Store_Utf8(file_data, "pid", 3,
(Obj*)Str_newf("%i32", (int32_t)PID_getpid()));
Hash_Store_Utf8(file_data, "host", 4, INCREF(ivars->host));
Hash_Store_Utf8(file_data, "name", 4, INCREF(ivars->name));
String *json = Json_to_json((Obj*)file_data);
DECREF(file_data);
// Write to a temporary file, then use the creation of a hard link to
// ensure atomic but non-destructive creation of the lockfile with its
// complete contents.
OutStream *outstream = Folder_Open_Out(ivars->folder, ivars->link_path);
if (!outstream) {
ERR_ADD_FRAME(Err_get_error());
DECREF(json);
return false;
}
struct lockfile_context context;
context.outstream = outstream;
context.json = json;
Err *json_error = Err_trap(S_write_lockfile_json, &context);
bool wrote_json = !json_error;
DECREF(outstream);
DECREF(json);
if (wrote_json) {
success = Folder_Hard_Link(ivars->folder, ivars->link_path,
ivars->lock_path);
if (!success) {
Err *hard_link_err = (Err*)CERTIFY(Err_get_error(), ERR);
Err_set_error((Err*)LockErr_new(Str_newf("Failed to obtain lock at '%o': %o",
ivars->lock_path,
Err_Get_Mess(hard_link_err))));
}
}
else {
Err_set_error((Err*)LockErr_new(Str_newf("Failed to obtain lock at '%o': %o",
ivars->lock_path,
Err_Get_Mess(json_error))));
DECREF(json_error);
}
// Verify that our temporary file got zapped.
bool deletion_failed = !Folder_Delete(ivars->folder, ivars->link_path);
if (deletion_failed) {
String *mess = MAKE_MESS("Failed to delete '%o'", ivars->link_path);
Err_throw_mess(ERR, mess);
}
return success;
}
PolyReader*
PolyReader_do_open(PolyReader *self, Obj *index, Snapshot *snapshot,
IndexManager *manager) {
PolyReaderIVARS *const ivars = PolyReader_IVARS(self);
Folder *folder = S_derive_folder(index);
uint64_t last_gen = 0;
PolyReader_init(self, NULL, folder, snapshot, manager, NULL);
DECREF(folder);
if (manager) {
if (!S_obtain_deletion_lock(self)) {
DECREF(self);
THROW(LOCKERR, "Couldn't get deletion lock");
}
}
while (1) {
CharBuf *target_snap_file;
// If a Snapshot was supplied, use its file.
if (snapshot) {
target_snap_file = Snapshot_Get_Path(snapshot);
if (!target_snap_file) {
THROW(ERR, "Supplied snapshot objects must not be empty");
}
else {
CB_Inc_RefCount(target_snap_file);
}
}
else {
// Otherwise, pick the most recent snap file.
target_snap_file = IxFileNames_latest_snapshot(folder);
// No snap file? Looks like the index is empty. We can stop now
// and return NULL.
if (!target_snap_file) { break; }
}
// Derive "generation" of this snapshot file from its name.
uint64_t gen = IxFileNames_extract_gen(target_snap_file);
// Get a read lock on the most recent snapshot file if indicated.
if (manager) {
if (!S_obtain_read_lock(self, target_snap_file)) {
DECREF(self);
THROW(LOCKERR, "Couldn't get read lock for %o",
target_snap_file);
}
}
// Testing only.
if (PolyReader_race_condition_debug1) {
ZombieCharBuf *temp = ZCB_WRAP_STR("temp", 4);
if (Folder_Exists(folder, (CharBuf*)temp)) {
bool success = Folder_Rename(folder, (CharBuf*)temp,
PolyReader_race_condition_debug1);
if (!success) { RETHROW(INCREF(Err_get_error())); }
}
PolyReader_debug1_num_passes++;
}
// If a Snapshot object was passed in, the file has already been read.
// If that's not the case, we must read the file we just picked.
if (!snapshot) {
struct try_read_snapshot_context context;
context.snapshot = ivars->snapshot;
context.folder = folder;
context.path = target_snap_file;
Err *error = Err_trap(S_try_read_snapshot, &context);
if (error) {
S_release_read_lock(self);
DECREF(target_snap_file);
if (last_gen < gen) { // Index updated, so try again.
DECREF(error);
last_gen = gen;
continue;
}
else { // Real error.
if (manager) { S_release_deletion_lock(self); }
RETHROW(error);
}
}
}
/* It's possible, though unlikely, for an Indexer to delete files
* out from underneath us after the snapshot file is read but before
* we've got SegReaders holding open all the required files. If we
* failed to open something, see if we can find a newer snapshot file.
* If we can, then the exception was due to the race condition. If
* not, we have a real exception, so throw an error. */
struct try_open_elements_context context;
context.self = self;
context.seg_readers = NULL;
Err *error = Err_trap(S_try_open_elements, &context);
if (error) {
S_release_read_lock(self);
DECREF(target_snap_file);
if (last_gen < gen) { // Index updated, so try again.
DECREF(error);
last_gen = gen;
}
else { // Real error.
if (manager) { S_release_deletion_lock(self); }
RETHROW(error);
}
}
else { // Succeeded.
S_init_sub_readers(self, (VArray*)context.seg_readers);
DECREF(context.seg_readers);
DECREF(target_snap_file);
break;
}
}
if (manager) { S_release_deletion_lock(self); }
return self;
}
void
S_try_open_elements(void *context) {
struct try_open_elements_context *args
= (struct try_open_elements_context*)context;
PolyReader *self = args->self;
PolyReaderIVARS *const ivars = PolyReader_IVARS(self);
VArray *files = Snapshot_List(ivars->snapshot);
Folder *folder = PolyReader_Get_Folder(self);
uint32_t num_segs = 0;
uint64_t latest_schema_gen = 0;
CharBuf *schema_file = NULL;
// Find schema file, count segments.
for (uint32_t i = 0, max = VA_Get_Size(files); i < max; i++) {
CharBuf *entry = (CharBuf*)VA_Fetch(files, i);
if (Seg_valid_seg_name(entry)) {
num_segs++;
}
else if (CB_Starts_With_Str(entry, "schema_", 7)
&& CB_Ends_With_Str(entry, ".json", 5)
) {
uint64_t gen = IxFileNames_extract_gen(entry);
if (gen > latest_schema_gen) {
latest_schema_gen = gen;
if (!schema_file) { schema_file = CB_Clone(entry); }
else { CB_Mimic(schema_file, (Obj*)entry); }
}
}
}
// Read Schema.
if (!schema_file) {
DECREF(files);
THROW(ERR, "Can't find a schema file.");
}
else {
Hash *dump = (Hash*)Json_slurp_json(folder, schema_file);
if (dump) { // read file successfully
DECREF(ivars->schema);
ivars->schema = (Schema*)CERTIFY(
VTable_Load_Obj(SCHEMA, (Obj*)dump), SCHEMA);
DECREF(dump);
DECREF(schema_file);
schema_file = NULL;
}
else {
CharBuf *mess = MAKE_MESS("Failed to parse %o", schema_file);
DECREF(schema_file);
DECREF(files);
Err_throw_mess(ERR, mess);
}
}
VArray *segments = VA_new(num_segs);
for (uint32_t i = 0, max = VA_Get_Size(files); i < max; i++) {
CharBuf *entry = (CharBuf*)VA_Fetch(files, i);
// Create a Segment for each segmeta.
if (Seg_valid_seg_name(entry)) {
int64_t seg_num = IxFileNames_extract_gen(entry);
Segment *segment = Seg_new(seg_num);
// Bail if reading the file fails (probably because it's been
// deleted and a new snapshot file has been written so we need to
// retry).
if (Seg_Read_File(segment, folder)) {
VA_Push(segments, (Obj*)segment);
}
else {
CharBuf *mess = MAKE_MESS("Failed to read %o", entry);
DECREF(segment);
DECREF(segments);
DECREF(files);
Err_throw_mess(ERR, mess);
}
}
}
// Sort the segments by age.
VA_Sort(segments, NULL, NULL);
// Open individual SegReaders.
struct try_open_segreader_context seg_context;
seg_context.schema = PolyReader_Get_Schema(self);
seg_context.folder = folder;
seg_context.snapshot = PolyReader_Get_Snapshot(self);
seg_context.segments = segments;
seg_context.result = NULL;
args->seg_readers = VA_new(num_segs);
Err *error = NULL;
for (uint32_t seg_tick = 0; seg_tick < num_segs; seg_tick++) {
seg_context.seg_tick = seg_tick;
error = Err_trap(S_try_open_segreader, &seg_context);
if (error) {
break;
}
VA_Push(args->seg_readers, (Obj*)seg_context.result);
seg_context.result = NULL;
}
DECREF(segments);
DECREF(files);
if (error) {
DECREF(args->seg_readers);
args->seg_readers = NULL;
RETHROW(error);
}
}
