static void
test_Consolidate(TestBatchRunner *runner) {
Folder *folder = S_folder_with_contents();
FileHandle *fh;
// Fake up detritus from failed consolidation.
fh = Folder_Open_FileHandle(folder, cf_file,
FH_CREATE | FH_WRITE_ONLY | FH_EXCLUSIVE);
DECREF(fh);
fh = Folder_Open_FileHandle(folder, cfmeta_temp,
FH_CREATE | FH_WRITE_ONLY | FH_EXCLUSIVE);
DECREF(fh);
CompoundFileWriter *cf_writer = CFWriter_new(folder);
CFWriter_Consolidate(cf_writer);
PASS(runner, "Consolidate completes despite leftover files");
DECREF(cf_writer);
TEST_TRUE(runner, Folder_Exists(folder, cf_file),
"cf.dat file written");
TEST_TRUE(runner, Folder_Exists(folder, cfmeta_file),
"cfmeta.json file written");
TEST_FALSE(runner, Folder_Exists(folder, foo),
"original file zapped");
TEST_FALSE(runner, Folder_Exists(folder, cfmeta_temp),
"detritus from failed consolidation zapped");
DECREF(folder);
}
static void
test_Delete_Tree(TestBatch *batch)
{
Folder *folder = (Folder*)RAMFolder_new(NULL);
FileHandle *fh;
bool_t result;
// Create tree to be deleted.
Folder_MkDir(folder, &foo);
Folder_MkDir(folder, &foo_bar);
Folder_MkDir(folder, &foo_bar_baz);
fh = Folder_Open_FileHandle(folder, &foo_bar_baz_boffo,
FH_CREATE | FH_WRITE_ONLY);
DECREF(fh);
// Create bystanders.
Folder_MkDir(folder, &bar);
fh = Folder_Open_FileHandle(folder, &baz, FH_CREATE | FH_WRITE_ONLY);
DECREF(fh);
result = Folder_Delete_Tree(folder, &foo);
TEST_TRUE(batch, result, "Delete_Tree() succeeded");
TEST_FALSE(batch, Folder_Exists(folder, &foo), "Tree really gone");
TEST_TRUE(batch, Folder_Exists(folder, &bar),
"local dir with same name as nested dir left intact");
TEST_TRUE(batch, Folder_Exists(folder, &baz),
"local file with same name as nested dir left intact");
// Kill off the bystanders.
result = Folder_Delete_Tree(folder, &bar);
TEST_TRUE(batch, result, "Delete_Tree() on empty dir");
result = Folder_Delete_Tree(folder, &baz);
TEST_TRUE(batch, result, "Delete_Tree() on file");
// Create new tree to be deleted.
Folder_MkDir(folder, &foo);
Folder_MkDir(folder, &foo_bar);
Folder_MkDir(folder, &foo_bar_baz);
fh = Folder_Open_FileHandle(folder, &foo_bar_baz_boffo,
FH_CREATE | FH_WRITE_ONLY);
DECREF(fh);
// Remove tree in subdir.
result = Folder_Delete_Tree(folder, &foo_bar);
TEST_TRUE(batch, result, "Delete_Tree() of subdir succeeded");
TEST_FALSE(batch, Folder_Exists(folder, &foo_bar),
"subdir really gone");
TEST_TRUE(batch, Folder_Exists(folder, &foo),
"enclosing dir left intact");
DECREF(folder);
}
void
LFLock_Release_IMP(LockFileLock *self) {
LockFileLockIVARS *const ivars = LFLock_IVARS(self);
if (Folder_Exists(ivars->folder, ivars->lock_path)) {
LFLock_Maybe_Delete_File(self, ivars->lock_path, true, false);
}
}
bool
LFLock_Maybe_Delete_File_IMP(LockFileLock *self, String *path,
bool delete_mine, bool delete_other) {
LockFileLockIVARS *const ivars = LFLock_IVARS(self);
Folder *folder = ivars->folder;
bool success = false;
// Only delete locks that start with our lock name.
if (!Str_Starts_With_Utf8(path, "locks", 5)) {
return false;
}
StringIterator *iter = Str_Top(path);
StrIter_Advance(iter, 5 + 1);
if (!StrIter_Starts_With(iter, ivars->name)) {
DECREF(iter);
return false;
}
DECREF(iter);
// Attempt to delete dead lock file.
if (Folder_Exists(folder, path)) {
Hash *hash = (Hash*)Json_slurp_json(folder, path);
if (hash != NULL && Obj_Is_A((Obj*)hash, HASH)) {
String *pid_buf = (String*)Hash_Fetch_Utf8(hash, "pid", 3);
String *host = (String*)Hash_Fetch_Utf8(hash, "host", 4);
String *name = (String*)Hash_Fetch_Utf8(hash, "name", 4);
// Match hostname and lock name.
if (host != NULL
&& Str_Is_A(host, STRING)
&& Str_Equals(host, (Obj*)ivars->host)
&& name != NULL
&& Str_Is_A(name, STRING)
&& Str_Equals(name, (Obj*)ivars->name)
&& pid_buf != NULL
&& Str_Is_A(pid_buf, STRING)
) {
// Verify that pid is either mine or dead.
int pid = (int)Str_To_I64(pid_buf);
if ((delete_mine && pid == PID_getpid()) // This process.
|| (delete_other && !PID_active(pid)) // Dead pid.
) {
if (Folder_Delete(folder, path)) {
success = true;
}
else {
String *mess
= MAKE_MESS("Can't delete '%o'", path);
DECREF(hash);
Err_throw_mess(ERR, mess);
}
}
}
}
DECREF(hash);
}
return success;
}
DefaultHighlightReader*
DefHLReader_init(DefaultHighlightReader *self, Schema *schema,
Folder *folder, Snapshot *snapshot, VArray *segments,
int32_t seg_tick)
{
Segment *segment;
Hash *metadata;
HLReader_init((HighlightReader*)self, schema, folder, snapshot,
segments, seg_tick);
segment = DefHLReader_Get_Segment(self);
metadata = (Hash*)Seg_Fetch_Metadata_Str(segment, "highlight", 9);
if (!metadata) {
metadata = (Hash*)Seg_Fetch_Metadata_Str(segment, "term_vectors", 12);
}
// Check format.
if (metadata) {
Obj *format = Hash_Fetch_Str(metadata, "format", 6);
if (!format) { THROW(ERR, "Missing 'format' var"); }
else {
if (Obj_To_I64(format) != HLWriter_current_file_format) {
THROW(ERR, "Unsupported highlight data format: %i64",
Obj_To_I64(format));
}
}
}
// Open instreams.
{
CharBuf *seg_name = Seg_Get_Name(segment);
CharBuf *ix_file = CB_newf("%o/highlight.ix", seg_name);
CharBuf *dat_file = CB_newf("%o/highlight.dat", seg_name);
if (Folder_Exists(folder, ix_file)) {
self->ix_in = Folder_Open_In(folder, ix_file);
if (!self->ix_in) {
Err *error = (Err*)INCREF(Err_get_error());
DECREF(ix_file);
DECREF(dat_file);
DECREF(self);
RETHROW(error);
}
self->dat_in = Folder_Open_In(folder, dat_file);
if (!self->dat_in) {
Err *error = (Err*)INCREF(Err_get_error());
DECREF(ix_file);
DECREF(dat_file);
DECREF(self);
RETHROW(error);
}
}
DECREF(ix_file);
DECREF(dat_file);
}
return self;
}
static void
S_clean_up_old_temp_files(CompoundFileWriter *self,
CompoundFileWriterIVARS *ivars) {
UNUSED_VAR(self);
Folder *folder = ivars->folder;
String *cfmeta_temp = (String*)SSTR_WRAP_UTF8("cfmeta.json.temp", 16);
String *cf_file = (String*)SSTR_WRAP_UTF8("cf.dat", 6);
if (Folder_Exists(folder, cf_file)) {
if (!Folder_Delete(folder, cf_file)) {
THROW(ERR, "Can't delete '%o'", cf_file);
}
}
if (Folder_Exists(folder, cfmeta_temp)) {
if (!Folder_Delete(folder, cfmeta_temp)) {
THROW(ERR, "Can't delete '%o'", cfmeta_temp);
}
}
}
DefaultDocReader*
DefDocReader_init(DefaultDocReader *self, Schema *schema, Folder *folder,
Snapshot *snapshot, Vector *segments, int32_t seg_tick) {
Hash *metadata;
Segment *segment;
DocReader_init((DocReader*)self, schema, folder, snapshot, segments,
seg_tick);
DefaultDocReaderIVARS *const ivars = DefDocReader_IVARS(self);
segment = DefDocReader_Get_Segment(self);
metadata = (Hash*)Seg_Fetch_Metadata_Utf8(segment, "documents", 9);
if (metadata) {
String *seg_name = Seg_Get_Name(segment);
String *ix_file = Str_newf("%o/documents.ix", seg_name);
String *dat_file = Str_newf("%o/documents.dat", seg_name);
Obj *format = Hash_Fetch_Utf8(metadata, "format", 6);
// Check format.
if (!format) { THROW(ERR, "Missing 'format' var"); }
else {
int64_t format_val = Json_obj_to_i64(format);
if (format_val < DocWriter_current_file_format) {
THROW(ERR, "Obsolete doc storage format %i64; "
"Index regeneration is required", format_val);
}
else if (format_val != DocWriter_current_file_format) {
THROW(ERR, "Unsupported doc storage format: %i64", format_val);
}
}
// Get streams.
if (Folder_Exists(folder, ix_file)) {
ivars->ix_in = Folder_Open_In(folder, ix_file);
if (!ivars->ix_in) {
Err *error = (Err*)INCREF(Err_get_error());
DECREF(ix_file);
DECREF(dat_file);
DECREF(self);
RETHROW(error);
}
ivars->dat_in = Folder_Open_In(folder, dat_file);
if (!ivars->dat_in) {
Err *error = (Err*)INCREF(Err_get_error());
DECREF(ix_file);
DECREF(dat_file);
DECREF(self);
RETHROW(error);
}
}
DECREF(ix_file);
DECREF(dat_file);
}
return self;
}
bool_t
LFLock_maybe_delete_file(LockFileLock *self, const CharBuf *path,
bool_t delete_mine, bool_t delete_other) {
Folder *folder = self->folder;
bool_t success = false;
ZombieCharBuf *scratch = ZCB_WRAP(path);
// Only delete locks that start with our lock name.
CharBuf *lock_dir_name = (CharBuf*)ZCB_WRAP_STR("locks", 5);
if (!ZCB_Starts_With(scratch, lock_dir_name)) {
return false;
}
ZCB_Nip(scratch, CB_Get_Size(lock_dir_name) + 1);
if (!ZCB_Starts_With(scratch, self->name)) {
return false;
}
// Attempt to delete dead lock file.
if (Folder_Exists(folder, path)) {
Hash *hash = (Hash*)Json_slurp_json(folder, path);
if (hash != NULL && Obj_Is_A((Obj*)hash, HASH)) {
CharBuf *pid_buf = (CharBuf*)Hash_Fetch_Str(hash, "pid", 3);
CharBuf *host = (CharBuf*)Hash_Fetch_Str(hash, "host", 4);
CharBuf *name
= (CharBuf*)Hash_Fetch_Str(hash, "name", 4);
// Match hostname and lock name.
if (host != NULL
&& CB_Equals(host, (Obj*)self->host)
&& name != NULL
&& CB_Equals(name, (Obj*)self->name)
&& pid_buf != NULL
) {
// Verify that pid is either mine or dead.
int pid = (int)CB_To_I64(pid_buf);
if ((delete_mine && pid == PID_getpid()) // This process.
|| (delete_other && !PID_active(pid)) // Dead pid.
) {
if (Folder_Delete(folder, path)) {
success = true;
}
else {
CharBuf *mess
= MAKE_MESS("Can't delete '%o'", path);
DECREF(hash);
Err_throw_mess(ERR, mess);
}
}
}
}
DECREF(hash);
}
return success;
}
static void
test_Exists(TestBatch *batch)
{
Folder *folder = (Folder*)RAMFolder_new(NULL);
FileHandle *fh;
Folder_MkDir(folder, &foo);
Folder_MkDir(folder, &foo_bar);
fh = Folder_Open_FileHandle(folder, &boffo, FH_CREATE | FH_WRITE_ONLY);
DECREF(fh);
fh = Folder_Open_FileHandle(folder, &foo_boffo,
FH_CREATE | FH_WRITE_ONLY);
DECREF(fh);
TEST_TRUE(batch, Folder_Exists(folder, &foo), "Dir exists");
TEST_TRUE(batch, Folder_Exists(folder, &boffo), "File exists");
TEST_TRUE(batch, Folder_Exists(folder, &foo_bar),
"Nested dir exists");
TEST_TRUE(batch, Folder_Exists(folder, &foo_boffo),
"Nested file exists");
TEST_FALSE(batch, Folder_Exists(folder, &banana),
"Non-existent entry");
TEST_FALSE(batch, Folder_Exists(folder, &foo_foo),
"Non-existent nested entry");
DECREF(folder);
}
DefaultHighlightReader*
DefHLReader_init(DefaultHighlightReader *self, Schema *schema,
Folder *folder, Snapshot *snapshot, Vector *segments,
int32_t seg_tick) {
HLReader_init((HighlightReader*)self, schema, folder, snapshot,
segments, seg_tick);
DefaultHighlightReaderIVARS *const ivars = DefHLReader_IVARS(self);
Segment *segment = DefHLReader_Get_Segment(self);
Hash *metadata = (Hash*)Seg_Fetch_Metadata_Utf8(segment, "highlight", 9);
if (!metadata) {
metadata = (Hash*)Seg_Fetch_Metadata_Utf8(segment, "term_vectors", 12);
}
// Check format.
if (metadata) {
Obj *format = Hash_Fetch_Utf8(metadata, "format", 6);
if (!format) { THROW(ERR, "Missing 'format' var"); }
else {
if (Json_obj_to_i64(format) != HLWriter_current_file_format) {
THROW(ERR, "Unsupported highlight data format: %i64",
Json_obj_to_i64(format));
}
}
}
// Open instreams.
String *seg_name = Seg_Get_Name(segment);
String *ix_file = Str_newf("%o/highlight.ix", seg_name);
String *dat_file = Str_newf("%o/highlight.dat", seg_name);
if (Folder_Exists(folder, ix_file)) {
ivars->ix_in = Folder_Open_In(folder, ix_file);
if (!ivars->ix_in) {
Err *error = (Err*)INCREF(Err_get_error());
DECREF(ix_file);
DECREF(dat_file);
DECREF(self);
RETHROW(error);
}
ivars->dat_in = Folder_Open_In(folder, dat_file);
if (!ivars->dat_in) {
Err *error = (Err*)INCREF(Err_get_error());
DECREF(ix_file);
DECREF(dat_file);
DECREF(self);
RETHROW(error);
}
}
DECREF(ix_file);
DECREF(dat_file);
return self;
}
void
CFWriter_Consolidate_IMP(CompoundFileWriter *self) {
CompoundFileWriterIVARS *const ivars = CFWriter_IVARS(self);
String *cfmeta_file = (String*)SSTR_WRAP_UTF8("cfmeta.json", 11);
if (Folder_Exists(ivars->folder, cfmeta_file)) {
THROW(ERR, "Merge already performed for %o",
Folder_Get_Path(ivars->folder));
}
else {
S_clean_up_old_temp_files(self, ivars);
S_do_consolidate(self, ivars);
}
}
DefaultDocReader*
DefDocReader_init(DefaultDocReader *self, Schema *schema, Folder *folder,
Snapshot *snapshot, VArray *segments, i32_t seg_tick)
{
Hash *metadata;
Segment *segment;
DocReader_init((DocReader*)self, schema, folder, snapshot, segments,
seg_tick);
segment = DefDocReader_Get_Segment(self);
metadata = (Hash*)Seg_Fetch_Metadata_Str(segment, "documents", 9);
if (metadata) {
CharBuf *seg_name = Seg_Get_Name(segment);
CharBuf *ix_file = CB_newf("%o/documents.ix", seg_name);
CharBuf *dat_file = CB_newf("%o/documents.dat", seg_name);
Obj *format = Hash_Fetch_Str(metadata, "format", 6);
/* Check format. */
if (!format) { THROW("Missing 'format' var"); }
else {
i64_t format_val = Obj_To_I64(format);
if (format_val < DocWriter_current_file_format) {
THROW("Obsolete doc storage format %i64; "
"Index regeneration is required", format_val);
}
else if (format_val != DocWriter_current_file_format) {
THROW("Unsupported doc storage format: %i64", format_val);
}
}
/* Get streams. */
if (Folder_Exists(folder, ix_file)) {
self->ix_in = Folder_Open_In(folder, ix_file);
self->dat_in = Folder_Open_In(folder, dat_file);
if (!self->ix_in || !self->dat_in) {
CharBuf *mess = MAKE_MESS("Can't open either %o or %o",
ix_file, dat_file);
DECREF(ix_file);
DECREF(dat_file);
DECREF(self);
Err_throw_mess(mess);
}
}
DECREF(ix_file);
DECREF(dat_file);
}
return self;
}
static void
test_Delete(TestBatch *batch)
{
Folder *folder = (Folder*)RAMFolder_new(NULL);
FileHandle *fh;
bool_t result;
Folder_MkDir(folder, &foo);
Folder_MkDir(folder, &foo_bar);
fh = Folder_Open_FileHandle(folder, &boffo, FH_CREATE | FH_WRITE_ONLY);
DECREF(fh);
fh = Folder_Open_FileHandle(folder, &foo_boffo,
FH_CREATE | FH_WRITE_ONLY);
DECREF(fh);
Err_set_error(NULL);
result = Folder_Delete(folder, &banana);
TEST_FALSE(batch, result, "Delete on non-existent entry returns false");
Err_set_error(NULL);
result = Folder_Delete(folder, &foo);
TEST_FALSE(batch, result, "Delete on non-empty dir returns false");
TEST_TRUE(batch, Folder_Delete(folder, &foo_boffo),
"Delete nested file");
TEST_FALSE(batch, Folder_Exists(folder, &foo_boffo),
"File is really gone");
TEST_TRUE(batch, Folder_Delete(folder, &foo_bar),
"Delete nested dir");
TEST_FALSE(batch, Folder_Exists(folder, &foo_bar),
"Dir is really gone");
TEST_TRUE(batch, Folder_Delete(folder, &foo), "Delete empty dir");
TEST_FALSE(batch, Folder_Exists(folder, &foo), "Dir is really gone");
DECREF(folder);
}
static void
test_spew_and_slurp(TestBatch *batch) {
Obj *dump = S_make_dump();
Folder *folder = (Folder*)RAMFolder_new(NULL);
CharBuf *foo = (CharBuf*)ZCB_WRAP_STR("foo", 3);
bool_t result = Json_spew_json(dump, folder, foo);
TEST_TRUE(batch, result, "spew_json returns true on success");
TEST_TRUE(batch, Folder_Exists(folder, foo),
"spew_json wrote file");
Obj *got = Json_slurp_json(folder, foo);
TEST_TRUE(batch, got && Obj_Equals(dump, got),
"Round trip through spew_json and slurp_json");
DECREF(got);
Err_set_error(NULL);
result = Json_spew_json(dump, folder, foo);
TEST_FALSE(batch, result, "Can't spew_json when file exists");
TEST_TRUE(batch, Err_get_error() != NULL,
"Failed spew_json sets Err_error");
Err_set_error(NULL);
CharBuf *bar = (CharBuf*)ZCB_WRAP_STR("bar", 3);
got = Json_slurp_json(folder, bar);
TEST_TRUE(batch, got == NULL,
"slurp_json returns NULL when file doesn't exist");
TEST_TRUE(batch, Err_get_error() != NULL,
"Failed slurp_json sets Err_error");
CharBuf *boffo = (CharBuf*)ZCB_WRAP_STR("boffo", 5);
FileHandle *fh
= Folder_Open_FileHandle(folder, boffo, FH_CREATE | FH_WRITE_ONLY);
FH_Write(fh, "garbage", 7);
DECREF(fh);
Err_set_error(NULL);
got = Json_slurp_json(folder, boffo);
TEST_TRUE(batch, got == NULL,
"slurp_json returns NULL when file doesn't contain valid JSON");
TEST_TRUE(batch, Err_get_error() != NULL,
"Failed slurp_json sets Err_error");
DECREF(got);
DECREF(dump);
DECREF(folder);
}
Hash*
IxManager_Read_Merge_Data_IMP(IndexManager *self) {
IndexManagerIVARS *const ivars = IxManager_IVARS(self);
String *merge_json = SSTR_WRAP_C("merge.json");
if (Folder_Exists(ivars->folder, merge_json)) {
Hash *stuff = (Hash*)Json_slurp_json(ivars->folder, merge_json);
if (stuff) {
CERTIFY(stuff, HASH);
return stuff;
}
else {
return Hash_new(0);
}
}
else {
return NULL;
}
}
// Indicate whether it is safe to build a SegLexicon using the given
// parameters. Will return false if the field is not indexed or if no terms
// are present for this field in this segment.
static bool
S_has_data(Schema *schema, Folder *folder, Segment *segment, String *field) {
FieldType *type = Schema_Fetch_Type(schema, field);
if (!type || !FType_Indexed(type)) {
// If the field isn't indexed, bail out.
return false;
}
else {
// Bail out if there are no terms for this field in this segment.
int32_t field_num = Seg_Field_Num(segment, field);
String *seg_name = Seg_Get_Name(segment);
String *file = Str_newf("%o/lexicon-%i32.dat", seg_name, field_num);
bool retval = Folder_Exists(folder, file);
DECREF(file);
return retval;
}
}
void
SegWriter_Prep_Seg_Dir_IMP(SegWriter *self) {
SegWriterIVARS *const ivars = SegWriter_IVARS(self);
Folder *folder = SegWriter_Get_Folder(self);
String *seg_name = Seg_Get_Name(ivars->segment);
// Clear stale segment files from crashed indexing sessions.
if (Folder_Exists(folder, seg_name)) {
bool result = Folder_Delete_Tree(folder, seg_name);
if (!result) {
THROW(ERR, "Couldn't completely remove '%o'", seg_name);
}
}
// Create the segment directory.
bool result = Folder_MkDir(folder, seg_name);
if (!result) { RETHROW(INCREF(Err_get_error())); }
}
static void
S_zap_dead_merge(FilePurger *self, Hash *candidates)
{
IndexManager *manager = self->manager;
Lock *merge_lock = IxManager_Make_Merge_Lock(manager);
Lock_Clear_Stale(merge_lock);
if (!Lock_Is_Locked(merge_lock)) {
Hash *merge_data = IxManager_Read_Merge_Data(manager);
Obj *cutoff = merge_data
? Hash_Fetch_Str(merge_data, "cutoff", 6)
: NULL;
if (cutoff) {
CharBuf *cutoff_seg = Seg_num_to_name(Obj_To_I64(cutoff));
if (Folder_Exists(self->folder, cutoff_seg)) {
ZombieCharBuf *merge_json = ZCB_WRAP_STR("merge.json", 10);
DirHandle *dh = Folder_Open_Dir(self->folder, cutoff_seg);
CharBuf *entry = dh ? DH_Get_Entry(dh) : NULL;
CharBuf *filepath = CB_new(32);
if (!dh) {
THROW(ERR, "Can't open segment dir '%o'", filepath);
}
Hash_Store(candidates, (Obj*)cutoff_seg, INCREF(&EMPTY));
Hash_Store(candidates, (Obj*)merge_json, INCREF(&EMPTY));
while (DH_Next(dh)) {
// TODO: recursively delete subdirs within seg dir.
CB_setf(filepath, "%o/%o", cutoff_seg, entry);
Hash_Store(candidates, (Obj*)filepath, INCREF(&EMPTY));
}
DECREF(filepath);
DECREF(dh);
}
DECREF(cutoff_seg);
}
DECREF(merge_data);
}
DECREF(merge_lock);
return;
}
static void
S_zap_dead_merge(FilePurger *self, Hash *candidates) {
FilePurgerIVARS *const ivars = FilePurger_IVARS(self);
IndexManager *manager = ivars->manager;
Lock *merge_lock = IxManager_Make_Merge_Lock(manager);
Lock_Clear_Stale(merge_lock);
if (!Lock_Is_Locked(merge_lock)) {
Hash *merge_data = IxManager_Read_Merge_Data(manager);
Obj *cutoff = merge_data
? Hash_Fetch_Utf8(merge_data, "cutoff", 6)
: NULL;
if (cutoff) {
String *cutoff_seg = Seg_num_to_name(Json_obj_to_i64(cutoff));
if (Folder_Exists(ivars->folder, cutoff_seg)) {
String *merge_json = SSTR_WRAP_UTF8("merge.json", 10);
DirHandle *dh = Folder_Open_Dir(ivars->folder, cutoff_seg);
if (!dh) {
THROW(ERR, "Can't open segment dir '%o'", cutoff_seg);
}
Hash_Store(candidates, cutoff_seg, (Obj*)CFISH_TRUE);
Hash_Store(candidates, merge_json, (Obj*)CFISH_TRUE);
while (DH_Next(dh)) {
// TODO: recursively delete subdirs within seg dir.
String *entry = DH_Get_Entry(dh);
String *filepath = Str_newf("%o/%o", cutoff_seg, entry);
Hash_Store(candidates, filepath, (Obj*)CFISH_TRUE);
DECREF(filepath);
DECREF(entry);
}
DECREF(dh);
}
DECREF(cutoff_seg);
}
DECREF(merge_data);
}
DECREF(merge_lock);
return;
}
Hash*
IxManager_read_merge_data(IndexManager *self)
{
ZombieCharBuf *merge_json = ZCB_WRAP_STR("merge.json", 10);
if (Folder_Exists(self->folder, (CharBuf*)merge_json)) {
Hash *stuff
= (Hash*)Json_slurp_json(self->folder, (CharBuf*)merge_json);
if (stuff) {
CERTIFY(stuff, HASH);
return stuff;
}
else {
return Hash_new(0);
}
}
else {
return NULL;
}
}
void
Snapshot_Write_File_IMP(Snapshot *self, Folder *folder, String *path) {
SnapshotIVARS *const ivars = Snapshot_IVARS(self);
Hash *all_data = Hash_new(0);
Vector *list = Snapshot_List(self);
// Update path.
DECREF(ivars->path);
if (path != NULL && Str_Get_Size(path) != 0) {
ivars->path = Str_Clone(path);
}
else {
String *latest = IxFileNames_latest_snapshot(folder);
uint64_t gen = latest ? IxFileNames_extract_gen(latest) + 1 : 1;
char base36[StrHelp_MAX_BASE36_BYTES];
StrHelp_to_base36(gen, &base36);
ivars->path = Str_newf("snapshot_%s.json", &base36);
DECREF(latest);
}
// Don't overwrite.
if (Folder_Exists(folder, ivars->path)) {
THROW(ERR, "Snapshot file '%o' already exists", ivars->path);
}
// Sort, then store file names.
Vec_Sort(list);
Hash_Store_Utf8(all_data, "entries", 7, (Obj*)list);
// Create a JSON-izable data structure.
Hash_Store_Utf8(all_data, "format", 6,
(Obj*)Str_newf("%i32", (int32_t)Snapshot_current_file_format));
Hash_Store_Utf8(all_data, "subformat", 9,
(Obj*)Str_newf("%i32", (int32_t)Snapshot_current_file_subformat));
// Write out JSON-ized data to the new file.
Json_spew_json((Obj*)all_data, folder, ivars->path);
DECREF(all_data);
}
void
PostPool_assign_seg(PostingPool *self, Folder *other_folder,
Segment *other_segment, i32_t doc_base, I32Array *doc_map)
{
i32_t field_num = Seg_Field_Num(other_segment, self->field);
CharBuf *other_seg_name = Seg_Get_Name(other_segment);
CharBuf *lex_file
= CB_newf("%o/lexicon-%i32.dat", other_seg_name, field_num);
/* Dedicate pool to this task alone. */
if (self->from_seg || self->cache_max > 0 || self->lex_end != 0)
THROW("Can't Assign_Segment to PostingPool with other content");
self->from_seg = true;
/* Prepare to read from existing files. */
if (Folder_Exists(other_folder, lex_file)) {
CharBuf *post_file
= CB_newf("%o/postings-%i32.dat", other_seg_name, field_num);
/* Open lexicon and postings files. */
self->lex_instream = Folder_Open_In(other_folder, lex_file);
self->post_instream = Folder_Open_In(other_folder, post_file);
if (!self->lex_instream) { THROW("Can't open %o", lex_file); }
if (!self->post_instream) { THROW("Can't open %o", post_file); }
self->lex_end = InStream_Length(self->lex_instream);
self->post_end = InStream_Length(self->post_instream);
/* Assign doc base and doc map. */
self->doc_base = doc_base;
self->doc_map = doc_map ? (I32Array*)INCREF(doc_map) : NULL;
DECREF(post_file);
}
else {
/* This posting pool will be empty. */
}
/* Clean up. */
DECREF(lex_file);
}
void
Snapshot_write_file(Snapshot *self, Folder *folder, const CharBuf *path)
{
Hash *all_data = Hash_new(0);
VArray *list = Snapshot_List(self);
// Update path.
DECREF(self->path);
if (path) {
self->path = CB_Clone(path);
}
else {
CharBuf *latest = IxFileNames_latest_snapshot(folder);
uint64_t gen = latest ? IxFileNames_extract_gen(latest) + 1 : 1;
char base36[StrHelp_MAX_BASE36_BYTES];
StrHelp_to_base36(gen, &base36);
self->path = CB_newf("snapshot_%s.json", &base36);
DECREF(latest);
}
// Don't overwrite.
if (Folder_Exists(folder, self->path)) {
THROW(ERR, "Snapshot file '%o' already exists", self->path);
}
// Sort, then store file names.
VA_Sort(list, NULL, NULL);
Hash_Store_Str(all_data, "entries", 7, (Obj*)list);
// Create a JSON-izable data structure.
Hash_Store_Str(all_data, "format", 6,
(Obj*)CB_newf("%i32", (int32_t)Snapshot_current_file_format) );
Hash_Store_Str(all_data, "subformat", 9,
(Obj*)CB_newf("%i32", (int32_t)Snapshot_current_file_subformat) );
// Write out JSON-ized data to the new file.
Json_spew_json((Obj*)all_data, folder, self->path);
DECREF(all_data);
}
void
Snapshot_write_file(Snapshot *self, Folder *folder, const CharBuf *filename)
{
Hash *all_data = Hash_new(0);
VArray *list = Snapshot_List(self);
/* Update filename. */
DECREF(self->filename);
if (filename) {
self->filename = CB_Clone(filename);
}
else {
CharBuf *latest = IxFileNames_latest_snapshot(folder);
i32_t gen = latest ? IxFileNames_extract_gen(latest) + 1 : 1;
CharBuf *base_36 = StrHelp_to_base36(gen);
self->filename = CB_newf("snapshot_%o.json", base_36);
DECREF(latest);
DECREF(base_36);
}
/* Don't overwrite. */
if (Folder_Exists(folder, self->filename)) {
THROW("Snapshot file '%o' already exists", self->filename);
}
/* Sort, then store file names. */
VA_Sort(list, NULL);
Hash_Store_Str(all_data, "entries", 7, (Obj*)list);
/* Create a JSON-izable data structure. */
Hash_Store_Str(all_data, "format", 6,
(Obj*)CB_newf("%i32", (i32_t)Snapshot_current_file_format) );
/* Write out JSON-ized data to the new file. */
Json_spew_json((Obj*)all_data, folder, self->filename);
DECREF(all_data);
}
bool
LFLock_Request_IMP(LockFileLock *self) {
LockFileLockIVARS *const ivars = LFLock_IVARS(self);
bool success = false;
if (Folder_Exists(ivars->folder, ivars->lock_path)) {
Err_set_error((Err*)LockErr_new(Str_newf("Can't obtain lock: '%o' exists",
ivars->lock_path)));
return false;
}
// Create the "locks" subdirectory if necessary.
String *lock_dir_name = (String*)SSTR_WRAP_UTF8("locks", 5);
if (!Folder_Exists(ivars->folder, lock_dir_name)) {
if (!Folder_MkDir(ivars->folder, lock_dir_name)) {
Err *mkdir_err = (Err*)CERTIFY(Err_get_error(), ERR);
LockErr *err = LockErr_new(Str_newf("Can't create 'locks' directory: %o",
Err_Get_Mess(mkdir_err)));
// Maybe our attempt failed because another process succeeded.
if (Folder_Find_Folder(ivars->folder, lock_dir_name)) {
DECREF(err);
}
else {
// Nope, everything failed, so bail out.
Err_set_error((Err*)err);
return false;
}
}
}
// Prepare to write pid, lock name, and host to the lock file as JSON.
Hash *file_data = Hash_new(3);
Hash_Store_Utf8(file_data, "pid", 3,
(Obj*)Str_newf("%i32", (int32_t)PID_getpid()));
Hash_Store_Utf8(file_data, "host", 4, INCREF(ivars->host));
Hash_Store_Utf8(file_data, "name", 4, INCREF(ivars->name));
String *json = Json_to_json((Obj*)file_data);
DECREF(file_data);
// Write to a temporary file, then use the creation of a hard link to
// ensure atomic but non-destructive creation of the lockfile with its
// complete contents.
OutStream *outstream = Folder_Open_Out(ivars->folder, ivars->link_path);
if (!outstream) {
ERR_ADD_FRAME(Err_get_error());
DECREF(json);
return false;
}
struct lockfile_context context;
context.outstream = outstream;
context.json = json;
Err *json_error = Err_trap(S_write_lockfile_json, &context);
bool wrote_json = !json_error;
DECREF(outstream);
DECREF(json);
if (wrote_json) {
success = Folder_Hard_Link(ivars->folder, ivars->link_path,
ivars->lock_path);
if (!success) {
Err *hard_link_err = (Err*)CERTIFY(Err_get_error(), ERR);
Err_set_error((Err*)LockErr_new(Str_newf("Failed to obtain lock at '%o': %o",
ivars->lock_path,
Err_Get_Mess(hard_link_err))));
}
}
else {
Err_set_error((Err*)LockErr_new(Str_newf("Failed to obtain lock at '%o': %o",
ivars->lock_path,
Err_Get_Mess(json_error))));
DECREF(json_error);
}
// Verify that our temporary file got zapped.
bool deletion_failed = !Folder_Delete(ivars->folder, ivars->link_path);
if (deletion_failed) {
String *mess = MAKE_MESS("Failed to delete '%o'", ivars->link_path);
Err_throw_mess(ERR, mess);
}
return success;
}
bool_t
LFLock_is_locked(LockFileLock *self) {
return Folder_Exists(self->folder, self->lock_path);
}
void
LFLock_release(LockFileLock *self) {
if (Folder_Exists(self->folder, self->lock_path)) {
LFLock_Maybe_Delete_File(self, self->lock_path, true, false);
}
}
bool_t
LFLock_request(LockFileLock *self) {
Hash *file_data;
bool_t wrote_json;
bool_t success = false;
bool_t deletion_failed = false;
if (Folder_Exists(self->folder, self->lock_path)) {
Err_set_error((Err*)LockErr_new(CB_newf("Can't obtain lock: '%o' exists",
self->lock_path)));
return false;
}
// Create the "locks" subdirectory if necessary.
CharBuf *lock_dir_name = (CharBuf*)ZCB_WRAP_STR("locks", 5);
if (!Folder_Exists(self->folder, lock_dir_name)) {
if (!Folder_MkDir(self->folder, lock_dir_name)) {
Err *mkdir_err = (Err*)CERTIFY(Err_get_error(), ERR);
LockErr *err = LockErr_new(CB_newf("Can't create 'locks' directory: %o",
Err_Get_Mess(mkdir_err)));
// Maybe our attempt failed because another process succeeded.
if (Folder_Find_Folder(self->folder, lock_dir_name)) {
DECREF(err);
}
else {
// Nope, everything failed, so bail out.
Err_set_error((Err*)err);
return false;
}
}
}
// Prepare to write pid, lock name, and host to the lock file as JSON.
file_data = Hash_new(3);
Hash_Store_Str(file_data, "pid", 3,
(Obj*)CB_newf("%i32", (int32_t)PID_getpid()));
Hash_Store_Str(file_data, "host", 4, INCREF(self->host));
Hash_Store_Str(file_data, "name", 4, INCREF(self->name));
// Write to a temporary file, then use the creation of a hard link to
// ensure atomic but non-destructive creation of the lockfile with its
// complete contents.
wrote_json = Json_spew_json((Obj*)file_data, self->folder, self->link_path);
if (wrote_json) {
success = Folder_Hard_Link(self->folder, self->link_path,
self->lock_path);
if (!success) {
Err *hard_link_err = (Err*)CERTIFY(Err_get_error(), ERR);
Err_set_error((Err*)LockErr_new(CB_newf("Failed to obtain lock at '%o': %o",
self->lock_path,
Err_Get_Mess(hard_link_err))));
}
deletion_failed = !Folder_Delete(self->folder, self->link_path);
}
else {
Err *spew_json_err = (Err*)CERTIFY(Err_get_error(), ERR);
Err_set_error((Err*)LockErr_new(CB_newf("Failed to obtain lock at '%o': %o",
self->lock_path,
Err_Get_Mess(spew_json_err))));
}
DECREF(file_data);
// Verify that our temporary file got zapped.
if (wrote_json && deletion_failed) {
CharBuf *mess = MAKE_MESS("Failed to delete '%o'", self->link_path);
Err_throw_mess(ERR, mess);
}
return success;
}
bool
LFLock_Is_Locked_IMP(LockFileLock *self) {
LockFileLockIVARS *const ivars = LFLock_IVARS(self);
return Folder_Exists(ivars->folder, ivars->lock_path);
}
PolyReader*
PolyReader_do_open(PolyReader *self, Obj *index, Snapshot *snapshot,
IndexManager *manager) {
PolyReaderIVARS *const ivars = PolyReader_IVARS(self);
Folder *folder = S_derive_folder(index);
uint64_t last_gen = 0;
PolyReader_init(self, NULL, folder, snapshot, manager, NULL);
DECREF(folder);
if (manager) {
if (!S_obtain_deletion_lock(self)) {
DECREF(self);
THROW(LOCKERR, "Couldn't get deletion lock");
}
}
while (1) {
CharBuf *target_snap_file;
// If a Snapshot was supplied, use its file.
if (snapshot) {
target_snap_file = Snapshot_Get_Path(snapshot);
if (!target_snap_file) {
THROW(ERR, "Supplied snapshot objects must not be empty");
}
else {
CB_Inc_RefCount(target_snap_file);
}
}
else {
// Otherwise, pick the most recent snap file.
target_snap_file = IxFileNames_latest_snapshot(folder);
// No snap file? Looks like the index is empty. We can stop now
// and return NULL.
if (!target_snap_file) { break; }
}
// Derive "generation" of this snapshot file from its name.
uint64_t gen = IxFileNames_extract_gen(target_snap_file);
// Get a read lock on the most recent snapshot file if indicated.
if (manager) {
if (!S_obtain_read_lock(self, target_snap_file)) {
DECREF(self);
THROW(LOCKERR, "Couldn't get read lock for %o",
target_snap_file);
}
}
// Testing only.
if (PolyReader_race_condition_debug1) {
ZombieCharBuf *temp = ZCB_WRAP_STR("temp", 4);
if (Folder_Exists(folder, (CharBuf*)temp)) {
bool success = Folder_Rename(folder, (CharBuf*)temp,
PolyReader_race_condition_debug1);
if (!success) { RETHROW(INCREF(Err_get_error())); }
}
PolyReader_debug1_num_passes++;
}
// If a Snapshot object was passed in, the file has already been read.
// If that's not the case, we must read the file we just picked.
if (!snapshot) {
struct try_read_snapshot_context context;
context.snapshot = ivars->snapshot;
context.folder = folder;
context.path = target_snap_file;
Err *error = Err_trap(S_try_read_snapshot, &context);
if (error) {
S_release_read_lock(self);
DECREF(target_snap_file);
if (last_gen < gen) { // Index updated, so try again.
DECREF(error);
last_gen = gen;
continue;
}
else { // Real error.
if (manager) { S_release_deletion_lock(self); }
RETHROW(error);
}
}
}
/* It's possible, though unlikely, for an Indexer to delete files
* out from underneath us after the snapshot file is read but before
* we've got SegReaders holding open all the required files. If we
* failed to open something, see if we can find a newer snapshot file.
* If we can, then the exception was due to the race condition. If
* not, we have a real exception, so throw an error. */
struct try_open_elements_context context;
context.self = self;
context.seg_readers = NULL;
Err *error = Err_trap(S_try_open_elements, &context);
if (error) {
S_release_read_lock(self);
DECREF(target_snap_file);
if (last_gen < gen) { // Index updated, so try again.
DECREF(error);
last_gen = gen;
}
else { // Real error.
if (manager) { S_release_deletion_lock(self); }
RETHROW(error);
}
}
else { // Succeeded.
S_init_sub_readers(self, (VArray*)context.seg_readers);
DECREF(context.seg_readers);
DECREF(target_snap_file);
break;
}
}
if (manager) { S_release_deletion_lock(self); }
return self;
}
