static void
test_fields(TestBatch *batch)
{
Segment *segment = Seg_new(1);
ZombieCharBuf *foo = ZCB_WRAP_STR("foo",3 );
ZombieCharBuf *bar = ZCB_WRAP_STR("bar", 3);
ZombieCharBuf *baz = ZCB_WRAP_STR("baz", 3);
int32_t field_num;
field_num = Seg_Add_Field(segment, (CharBuf*)foo);
TEST_TRUE(batch, field_num == 1,
"Add_Field returns field number, and field numbers start at 1");
field_num = Seg_Add_Field(segment, (CharBuf*)bar);
TEST_TRUE(batch, field_num == 2, "add a second field");
field_num = Seg_Add_Field(segment, (CharBuf*)foo);
TEST_TRUE(batch, field_num == 1,
"Add_Field returns existing field number if field is already known");
TEST_TRUE(batch, ZCB_Equals(bar, (Obj*)Seg_Field_Name(segment, 2)),
"Field_Name");
TEST_TRUE(batch, Seg_Field_Name(segment, 3) == NULL,
"Field_Name returns NULL for unknown field number");
TEST_TRUE(batch, Seg_Field_Num(segment, (CharBuf*)bar) == 2,
"Field_Num");
TEST_TRUE(batch, Seg_Field_Num(segment, (CharBuf*)baz) == 0,
"Field_Num returns 0 for unknown field name");
DECREF(segment);
}
static void
test_Dump_Load_and_Equals(TestBatch *batch)
{
CharBuf *EN = (CharBuf*)ZCB_WRAP_STR("en", 2);
CharBuf *ES = (CharBuf*)ZCB_WRAP_STR("es", 2);
Stemmer *stemmer = Stemmer_new(EN);
Stemmer *other = Stemmer_new(ES);
Obj *dump = (Obj*)Stemmer_Dump(stemmer);
Obj *other_dump = (Obj*)Stemmer_Dump(other);
Stemmer *clone = (Stemmer*)Stemmer_Load(other, dump);
Stemmer *other_clone = (Stemmer*)Stemmer_Load(other, other_dump);
TEST_FALSE(batch, Stemmer_Equals(stemmer,
(Obj*)other), "Equals() false with different language");
TEST_TRUE(batch, Stemmer_Equals(stemmer,
(Obj*)clone), "Dump => Load round trip");
TEST_TRUE(batch, Stemmer_Equals(other,
(Obj*)other_clone), "Dump => Load round trip");
DECREF(stemmer);
DECREF(dump);
DECREF(clone);
DECREF(other);
DECREF(other_dump);
DECREF(other_clone);
}
static void
test_Write_File_and_Read_File(TestBatch *batch)
{
RAMFolder *folder = RAMFolder_new(NULL);
Segment *segment = Seg_new(100);
Segment *got = Seg_new(100);
CharBuf *meta;
CharBuf *flotsam = (CharBuf*)ZCB_WRAP_STR("flotsam", 7);
CharBuf *jetsam = (CharBuf*)ZCB_WRAP_STR("jetsam", 6);
Seg_Set_Count(segment, 111);
Seg_Store_Metadata_Str(segment, "foo", 3, (Obj*)CB_newf("bar"));
Seg_Add_Field(segment, flotsam);
Seg_Add_Field(segment, jetsam);
RAMFolder_MkDir(folder, Seg_Get_Name(segment));
Seg_Write_File(segment, (Folder*)folder);
Seg_Read_File(got, (Folder*)folder);
TEST_TRUE(batch, Seg_Get_Count(got) == Seg_Get_Count(segment),
"Round-trip count through file");
TEST_TRUE(batch,
Seg_Field_Num(got, jetsam) == Seg_Field_Num(segment, jetsam),
"Round trip field names through file");
meta = (CharBuf*)Seg_Fetch_Metadata_Str(got, "foo", 3);
TEST_TRUE(batch, meta && CB_Is_A(meta, CHARBUF)
&& CB_Equals_Str(meta, "bar", 3), "Round trip metadata through file");
DECREF(got);
DECREF(segment);
DECREF(folder);
}
bool_t
CFReader_local_delete(CompoundFileReader *self, const CharBuf *name) {
Hash *record = (Hash*)Hash_Delete(self->records, (Obj*)name);
DECREF(record);
if (record == NULL) {
return Folder_Local_Delete(self->real_folder, name);
}
else {
// Once the number of virtual files falls to 0, remove the compound
// files.
if (Hash_Get_Size(self->records) == 0) {
CharBuf *cf_file = (CharBuf*)ZCB_WRAP_STR("cf.dat", 6);
if (!Folder_Delete(self->real_folder, cf_file)) {
return false;
}
CharBuf *cfmeta_file = (CharBuf*)ZCB_WRAP_STR("cfmeta.json", 11);
if (!Folder_Delete(self->real_folder, cfmeta_file)) {
return false;
}
}
return true;
}
}
static void
test_all(TestBatch *batch) {
CharBuf *foo = (CharBuf*)ZCB_WRAP_STR("foo", 3);
CharBuf *boffo = (CharBuf*)ZCB_WRAP_STR("boffo", 5);
CharBuf *foo_boffo = (CharBuf*)ZCB_WRAP_STR("foo/boffo", 9);
CharBuf *test_dir = (CharBuf*)ZCB_WRAP_STR("_fsdir_test", 11);
FSFolder *folder = FSFolder_new(test_dir);
bool_t saw_foo = false;
bool_t saw_boffo = false;
bool_t foo_was_dir = false;
bool_t boffo_was_dir = false;
int count = 0;
// Clean up after previous failed runs.
FSFolder_Delete(folder, foo_boffo);
FSFolder_Delete(folder, foo);
FSFolder_Delete(folder, boffo);
rmdir("_fsdir_test");
FSFolder_Initialize(folder);
FSFolder_MkDir(folder, foo);
OutStream *outstream = FSFolder_Open_Out(folder, boffo);
DECREF(outstream);
outstream = FSFolder_Open_Out(folder, foo_boffo);
DECREF(outstream);
FSDirHandle *dh = FSDH_open(test_dir);
CharBuf *entry = FSDH_Get_Entry(dh);
while (FSDH_Next(dh)) {
count++;
if (CB_Equals(entry, (Obj*)foo)) {
saw_foo = true;
foo_was_dir = FSDH_Entry_Is_Dir(dh);
}
else if (CB_Equals(entry, (Obj*)boffo)) {
saw_boffo = true;
boffo_was_dir = FSDH_Entry_Is_Dir(dh);
}
}
TEST_INT_EQ(batch, 2, count, "correct number of entries");
TEST_TRUE(batch, saw_foo, "Directory was iterated over");
TEST_TRUE(batch, foo_was_dir,
"Dir correctly identified by Entry_Is_Dir");
TEST_TRUE(batch, saw_boffo, "File was iterated over");
TEST_FALSE(batch, boffo_was_dir,
"File correctly identified by Entry_Is_Dir");
DECREF(dh);
FSFolder_Delete(folder, foo_boffo);
FSFolder_Delete(folder, foo);
FSFolder_Delete(folder, boffo);
DECREF(folder);
rmdir("_fsdir_test");
}
static void
test_Keys_Values_Iter(TestBatch *batch) {
Hash *hash = Hash_new(0); // trigger multiple rebuilds.
VArray *expected = VA_new(100);
VArray *keys;
VArray *values;
for (uint32_t i = 0; i < 500; i++) {
CharBuf *cb = CB_newf("%u32", i);
Hash_Store(hash, (Obj*)cb, (Obj*)cb);
VA_Push(expected, INCREF(cb));
}
VA_Sort(expected, NULL, NULL);
keys = Hash_Keys(hash);
values = Hash_Values(hash);
VA_Sort(keys, NULL, NULL);
VA_Sort(values, NULL, NULL);
TEST_TRUE(batch, VA_Equals(keys, (Obj*)expected), "Keys");
TEST_TRUE(batch, VA_Equals(values, (Obj*)expected), "Values");
VA_Clear(keys);
VA_Clear(values);
{
Obj *key;
Obj *value;
Hash_Iterate(hash);
while (Hash_Next(hash, &key, &value)) {
VA_Push(keys, INCREF(key));
VA_Push(values, INCREF(value));
}
}
VA_Sort(keys, NULL, NULL);
VA_Sort(values, NULL, NULL);
TEST_TRUE(batch, VA_Equals(keys, (Obj*)expected), "Keys from Iter");
TEST_TRUE(batch, VA_Equals(values, (Obj*)expected), "Values from Iter");
{
ZombieCharBuf *forty = ZCB_WRAP_STR("40", 2);
ZombieCharBuf *nope = ZCB_WRAP_STR("nope", 4);
Obj *key = Hash_Find_Key(hash, (Obj*)forty, ZCB_Hash_Sum(forty));
TEST_TRUE(batch, Obj_Equals(key, (Obj*)forty), "Find_Key");
key = Hash_Find_Key(hash, (Obj*)nope, ZCB_Hash_Sum(nope)),
TEST_TRUE(batch, key == NULL,
"Find_Key returns NULL for non-existent key");
}
DECREF(hash);
DECREF(expected);
DECREF(keys);
DECREF(values);
}
static void
test_spew_and_slurp(TestBatch *batch) {
Obj *dump = S_make_dump();
Folder *folder = (Folder*)RAMFolder_new(NULL);
CharBuf *foo = (CharBuf*)ZCB_WRAP_STR("foo", 3);
bool_t result = Json_spew_json(dump, folder, foo);
TEST_TRUE(batch, result, "spew_json returns true on success");
TEST_TRUE(batch, Folder_Exists(folder, foo),
"spew_json wrote file");
Obj *got = Json_slurp_json(folder, foo);
TEST_TRUE(batch, got && Obj_Equals(dump, got),
"Round trip through spew_json and slurp_json");
DECREF(got);
Err_set_error(NULL);
result = Json_spew_json(dump, folder, foo);
TEST_FALSE(batch, result, "Can't spew_json when file exists");
TEST_TRUE(batch, Err_get_error() != NULL,
"Failed spew_json sets Err_error");
Err_set_error(NULL);
CharBuf *bar = (CharBuf*)ZCB_WRAP_STR("bar", 3);
got = Json_slurp_json(folder, bar);
TEST_TRUE(batch, got == NULL,
"slurp_json returns NULL when file doesn't exist");
TEST_TRUE(batch, Err_get_error() != NULL,
"Failed slurp_json sets Err_error");
CharBuf *boffo = (CharBuf*)ZCB_WRAP_STR("boffo", 5);
FileHandle *fh
= Folder_Open_FileHandle(folder, boffo, FH_CREATE | FH_WRITE_ONLY);
FH_Write(fh, "garbage", 7);
DECREF(fh);
Err_set_error(NULL);
got = Json_slurp_json(folder, boffo);
TEST_TRUE(batch, got == NULL,
"slurp_json returns NULL when file doesn't contain valid JSON");
TEST_TRUE(batch, Err_get_error() != NULL,
"Failed slurp_json sets Err_error");
DECREF(got);
DECREF(dump);
DECREF(folder);
}
Lock*
IxManager_make_merge_lock(IndexManager *self)
{
ZombieCharBuf *merge_lock_name = ZCB_WRAP_STR("merge", 5);
LockFactory *lock_factory = S_obtain_lock_factory(self);
return LockFact_Make_Lock(lock_factory, (CharBuf*)merge_lock_name,
self->merge_lock_timeout, self->merge_lock_interval);
}
Lock*
IxManager_make_deletion_lock(IndexManager *self)
{
ZombieCharBuf *lock_name = ZCB_WRAP_STR("deletion", 8);
LockFactory *lock_factory = S_obtain_lock_factory(self);
return LockFact_Make_Lock(lock_factory, (CharBuf*)lock_name,
self->deletion_lock_timeout, self->deletion_lock_interval);
}
static void
S_verify_bad_syntax(TestBatch *batch, const char *bad, const char *mess) {
ZombieCharBuf *has_errors = ZCB_WRAP_STR(bad, strlen(bad));
Err_set_error(NULL);
Obj *not_json = Json_from_json((CharBuf*)has_errors);
TEST_TRUE(batch, not_json == NULL, "from_json returns NULL: %s", mess);
TEST_TRUE(batch, Err_get_error() != NULL, "from_json sets Err_error: %s",
mess);
}
bool_t
LFLock_maybe_delete_file(LockFileLock *self, const CharBuf *path,
bool_t delete_mine, bool_t delete_other) {
Folder *folder = self->folder;
bool_t success = false;
ZombieCharBuf *scratch = ZCB_WRAP(path);
// Only delete locks that start with our lock name.
CharBuf *lock_dir_name = (CharBuf*)ZCB_WRAP_STR("locks", 5);
if (!ZCB_Starts_With(scratch, lock_dir_name)) {
return false;
}
ZCB_Nip(scratch, CB_Get_Size(lock_dir_name) + 1);
if (!ZCB_Starts_With(scratch, self->name)) {
return false;
}
// Attempt to delete dead lock file.
if (Folder_Exists(folder, path)) {
Hash *hash = (Hash*)Json_slurp_json(folder, path);
if (hash != NULL && Obj_Is_A((Obj*)hash, HASH)) {
CharBuf *pid_buf = (CharBuf*)Hash_Fetch_Str(hash, "pid", 3);
CharBuf *host = (CharBuf*)Hash_Fetch_Str(hash, "host", 4);
CharBuf *name
= (CharBuf*)Hash_Fetch_Str(hash, "name", 4);
// Match hostname and lock name.
if (host != NULL
&& CB_Equals(host, (Obj*)self->host)
&& name != NULL
&& CB_Equals(name, (Obj*)self->name)
&& pid_buf != NULL
) {
// Verify that pid is either mine or dead.
int pid = (int)CB_To_I64(pid_buf);
if ((delete_mine && pid == PID_getpid()) // This process.
|| (delete_other && !PID_active(pid)) // Dead pid.
) {
if (Folder_Delete(folder, path)) {
success = true;
}
else {
CharBuf *mess
= MAKE_MESS("Can't delete '%o'", path);
DECREF(hash);
Err_throw_mess(ERR, mess);
}
}
}
}
DECREF(hash);
}
return success;
}
static Obj*
S_new_testobj() {
ZombieCharBuf *klass = ZCB_WRAP_STR("TestObj", 7);
Obj *obj;
VTable *vtable = VTable_fetch_vtable((CharBuf*)klass);
if (!vtable) {
vtable = VTable_singleton((CharBuf*)klass, OBJ);
}
obj = VTable_Make_Obj(vtable);
return Obj_init(obj);
}
void
IxManager_write_merge_data(IndexManager *self, int64_t cutoff)
{
ZombieCharBuf *merge_json = ZCB_WRAP_STR("merge.json", 10);
Hash *data = Hash_new(1);
bool_t success;
Hash_Store_Str(data, "cutoff", 6, (Obj*)CB_newf("%i64", cutoff));
success = Json_spew_json((Obj*)data, self->folder, (CharBuf*)merge_json);
DECREF(data);
if (!success) {
THROW(ERR, "Failed to write to %o", merge_json);
}
}
static void
S_zap_dead_merge(FilePurger *self, Hash *candidates)
{
IndexManager *manager = self->manager;
Lock *merge_lock = IxManager_Make_Merge_Lock(manager);
Lock_Clear_Stale(merge_lock);
if (!Lock_Is_Locked(merge_lock)) {
Hash *merge_data = IxManager_Read_Merge_Data(manager);
Obj *cutoff = merge_data
? Hash_Fetch_Str(merge_data, "cutoff", 6)
: NULL;
if (cutoff) {
CharBuf *cutoff_seg = Seg_num_to_name(Obj_To_I64(cutoff));
if (Folder_Exists(self->folder, cutoff_seg)) {
ZombieCharBuf *merge_json = ZCB_WRAP_STR("merge.json", 10);
DirHandle *dh = Folder_Open_Dir(self->folder, cutoff_seg);
CharBuf *entry = dh ? DH_Get_Entry(dh) : NULL;
CharBuf *filepath = CB_new(32);
if (!dh) {
THROW(ERR, "Can't open segment dir '%o'", filepath);
}
Hash_Store(candidates, (Obj*)cutoff_seg, INCREF(&EMPTY));
Hash_Store(candidates, (Obj*)merge_json, INCREF(&EMPTY));
while (DH_Next(dh)) {
// TODO: recursively delete subdirs within seg dir.
CB_setf(filepath, "%o/%o", cutoff_seg, entry);
Hash_Store(candidates, (Obj*)filepath, INCREF(&EMPTY));
}
DECREF(filepath);
DECREF(dh);
}
DECREF(cutoff_seg);
}
DECREF(merge_data);
}
DECREF(merge_lock);
return;
}
Hash*
IxManager_read_merge_data(IndexManager *self)
{
ZombieCharBuf *merge_json = ZCB_WRAP_STR("merge.json", 10);
if (Folder_Exists(self->folder, (CharBuf*)merge_json)) {
Hash *stuff
= (Hash*)Json_slurp_json(self->folder, (CharBuf*)merge_json);
if (stuff) {
CERTIFY(stuff, HASH);
return stuff;
}
else {
return Hash_new(0);
}
}
else {
return NULL;
}
}
static void
S_set_error(CharBuf *mess, char *json, char *limit, int line,
const char *func) {
if (func) {
CB_catf(mess, " at %s %s line %i32 near ", func, __FILE__,
(int32_t)line);
}
else {
CB_catf(mess, " at %s line %i32 near ", __FILE__, (int32_t)line);
}
// Append escaped text.
int64_t len = limit - json;
if (len > 32) {
const char *end = StrHelp_back_utf8_char(json + 32, json);
len = end - json;
}
ZombieCharBuf *snippet = ZCB_WRAP_STR(json, len);
S_append_json_string((Obj*)snippet, mess);
// Set Err_error.
Err_set_error(Err_new(mess));
}
static void
test_Equals(TestBatch *batch) {
Hash *hash = Hash_new(0);
Hash *other = Hash_new(0);
ZombieCharBuf *stuff = ZCB_WRAP_STR("stuff", 5);
TEST_TRUE(batch, Hash_Equals(hash, (Obj*)other),
"Empty hashes are equal");
Hash_Store_Str(hash, "foo", 3, (Obj*)CFISH_TRUE);
TEST_FALSE(batch, Hash_Equals(hash, (Obj*)other),
"Add one pair and Equals returns false");
Hash_Store_Str(other, "foo", 3, (Obj*)CFISH_TRUE);
TEST_TRUE(batch, Hash_Equals(hash, (Obj*)other),
"Add a matching pair and Equals returns true");
Hash_Store_Str(other, "foo", 3, INCREF(stuff));
TEST_FALSE(batch, Hash_Equals(hash, (Obj*)other),
"Non-matching value spoils Equals");
DECREF(hash);
DECREF(other);
}
static Folder*
S_create_index() {
Schema *schema = (Schema*)TestSchema_new();
RAMFolder *folder = RAMFolder_new(NULL);
VArray *doc_set = TestUtils_doc_set();
Indexer *indexer = Indexer_new(schema, (Obj*)folder, NULL, 0);
uint32_t i, max;
CharBuf *field = (CharBuf*)ZCB_WRAP_STR("content", 7);
for (i = 0, max = VA_Get_Size(doc_set); i < max; i++) {
Doc *doc = Doc_new(NULL, 0);
Doc_Store(doc, field, VA_Fetch(doc_set, i));
Indexer_Add_Doc(indexer, doc, 1.0f);
DECREF(doc);
}
Indexer_Commit(indexer);
DECREF(doc_set);
DECREF(indexer);
DECREF(schema);
return (Folder*)folder;
}
CompoundFileReader*
CFReader_do_open(CompoundFileReader *self, Folder *folder) {
CharBuf *cfmeta_file = (CharBuf*)ZCB_WRAP_STR("cfmeta.json", 11);
Hash *metadata = (Hash*)Json_slurp_json((Folder*)folder, cfmeta_file);
Err *error = NULL;
Folder_init((Folder*)self, Folder_Get_Path(folder));
// Parse metadata file.
if (!metadata || !Hash_Is_A(metadata, HASH)) {
error = Err_new(CB_newf("Can't read '%o' in '%o'", cfmeta_file,
Folder_Get_Path(folder)));
}
else {
Obj *format = Hash_Fetch_Str(metadata, "format", 6);
self->format = format ? (int32_t)Obj_To_I64(format) : 0;
self->records = (Hash*)INCREF(Hash_Fetch_Str(metadata, "files", 5));
if (self->format < 1) {
error = Err_new(CB_newf("Corrupt %o file: Missing or invalid 'format'",
cfmeta_file));
}
else if (self->format > CFWriter_current_file_format) {
error = Err_new(CB_newf("Unsupported compound file format: %i32 "
"(current = %i32", self->format,
CFWriter_current_file_format));
}
else if (!self->records) {
error = Err_new(CB_newf("Corrupt %o file: missing 'files' key",
cfmeta_file));
}
}
DECREF(metadata);
if (error) {
Err_set_error(error);
DECREF(self);
return NULL;
}
// Open an instream which we'll clone over and over.
CharBuf *cf_file = (CharBuf*)ZCB_WRAP_STR("cf.dat", 6);
self->instream = Folder_Open_In(folder, cf_file);
if (!self->instream) {
ERR_ADD_FRAME(Err_get_error());
DECREF(self);
return NULL;
}
// Assign.
self->real_folder = (Folder*)INCREF(folder);
// Strip directory name from filepaths for old format.
if (self->format == 1) {
VArray *files = Hash_Keys(self->records);
ZombieCharBuf *filename = ZCB_BLANK();
ZombieCharBuf *folder_name
= IxFileNames_local_part(Folder_Get_Path(folder), ZCB_BLANK());
size_t folder_name_len = ZCB_Length(folder_name);
for (uint32_t i = 0, max = VA_Get_Size(files); i < max; i++) {
CharBuf *orig = (CharBuf*)VA_Fetch(files, i);
if (CB_Starts_With(orig, (CharBuf*)folder_name)) {
Obj *record = Hash_Delete(self->records, (Obj*)orig);
ZCB_Assign(filename, orig);
ZCB_Nip(filename, folder_name_len + sizeof(DIR_SEP) - 1);
Hash_Store(self->records, (Obj*)filename, (Obj*)record);
}
}
DECREF(files);
}
return self;
}
bool_t
LFLock_request(LockFileLock *self) {
Hash *file_data;
bool_t wrote_json;
bool_t success = false;
bool_t deletion_failed = false;
if (Folder_Exists(self->folder, self->lock_path)) {
Err_set_error((Err*)LockErr_new(CB_newf("Can't obtain lock: '%o' exists",
self->lock_path)));
return false;
}
// Create the "locks" subdirectory if necessary.
CharBuf *lock_dir_name = (CharBuf*)ZCB_WRAP_STR("locks", 5);
if (!Folder_Exists(self->folder, lock_dir_name)) {
if (!Folder_MkDir(self->folder, lock_dir_name)) {
Err *mkdir_err = (Err*)CERTIFY(Err_get_error(), ERR);
LockErr *err = LockErr_new(CB_newf("Can't create 'locks' directory: %o",
Err_Get_Mess(mkdir_err)));
// Maybe our attempt failed because another process succeeded.
if (Folder_Find_Folder(self->folder, lock_dir_name)) {
DECREF(err);
}
else {
// Nope, everything failed, so bail out.
Err_set_error((Err*)err);
return false;
}
}
}
// Prepare to write pid, lock name, and host to the lock file as JSON.
file_data = Hash_new(3);
Hash_Store_Str(file_data, "pid", 3,
(Obj*)CB_newf("%i32", (int32_t)PID_getpid()));
Hash_Store_Str(file_data, "host", 4, INCREF(self->host));
Hash_Store_Str(file_data, "name", 4, INCREF(self->name));
// Write to a temporary file, then use the creation of a hard link to
// ensure atomic but non-destructive creation of the lockfile with its
// complete contents.
wrote_json = Json_spew_json((Obj*)file_data, self->folder, self->link_path);
if (wrote_json) {
success = Folder_Hard_Link(self->folder, self->link_path,
self->lock_path);
if (!success) {
Err *hard_link_err = (Err*)CERTIFY(Err_get_error(), ERR);
Err_set_error((Err*)LockErr_new(CB_newf("Failed to obtain lock at '%o': %o",
self->lock_path,
Err_Get_Mess(hard_link_err))));
}
deletion_failed = !Folder_Delete(self->folder, self->link_path);
}
else {
Err *spew_json_err = (Err*)CERTIFY(Err_get_error(), ERR);
Err_set_error((Err*)LockErr_new(CB_newf("Failed to obtain lock at '%o': %o",
self->lock_path,
Err_Get_Mess(spew_json_err))));
}
DECREF(file_data);
// Verify that our temporary file got zapped.
if (wrote_json && deletion_failed) {
CharBuf *mess = MAKE_MESS("Failed to delete '%o'", self->link_path);
Err_throw_mess(ERR, mess);
}
return success;
}
void
Hash_store_str(Hash *self, const char *key, size_t key_len, Obj *value) {
ZombieCharBuf *key_buf = ZCB_WRAP_STR((char*)key, key_len);
Hash_do_store(self, (Obj*)key_buf, value,
ZCB_Hash_Sum(key_buf), false);
}
Obj*
Hash_delete_str(Hash *self, const char *key, size_t key_len) {
ZombieCharBuf *key_buf = ZCB_WRAP_STR(key, key_len);
return Hash_delete(self, (Obj*)key_buf);
}
bool_t
IxManager_remove_merge_data(IndexManager *self)
{
ZombieCharBuf *merge_json = ZCB_WRAP_STR("merge.json", 10);
return Folder_Delete(self->folder, (CharBuf*)merge_json) != 0;
}
void
Seg_store_metadata_str(Segment *self, const char *key, size_t key_len,
Obj *value) {
ZombieCharBuf *k = ZCB_WRAP_STR((char*)key, key_len);
Seg_Store_Metadata(self, (CharBuf*)k, value);
}
void
TestQPLogic_run_tests() {
uint32_t i;
TestBatch *batch = TestBatch_new(258);
Folder *folder = S_create_index();
IndexSearcher *searcher = IxSearcher_new((Obj*)folder);
QueryParser *or_parser = QParser_new(IxSearcher_Get_Schema(searcher),
NULL, NULL, NULL);
ZombieCharBuf *AND = ZCB_WRAP_STR("AND", 3);
QueryParser *and_parser = QParser_new(IxSearcher_Get_Schema(searcher),
NULL, (CharBuf*)AND, NULL);
QParser_Set_Heed_Colons(or_parser, true);
QParser_Set_Heed_Colons(and_parser, true);
TestBatch_Plan(batch);
// Run logical tests with default boolop of OR.
for (i = 0; logical_test_funcs[i] != NULL; i++) {
Lucy_TestQPLogic_Logical_Test_t test_func = logical_test_funcs[i];
TestQueryParser *test_case = test_func(BOOLOP_OR);
Query *tree = QParser_Tree(or_parser, test_case->query_string);
Query *parsed = QParser_Parse(or_parser, test_case->query_string);
Hits *hits = IxSearcher_Hits(searcher, (Obj*)parsed, 0, 10, NULL);
TEST_TRUE(batch, Query_Equals(tree, (Obj*)test_case->tree),
"tree() OR %s", (char*)CB_Get_Ptr8(test_case->query_string));
TEST_INT_EQ(batch, Hits_Total_Hits(hits), test_case->num_hits,
"hits: OR %s", (char*)CB_Get_Ptr8(test_case->query_string));
DECREF(hits);
DECREF(parsed);
DECREF(tree);
DECREF(test_case);
}
// Run logical tests with default boolop of AND.
for (i = 0; logical_test_funcs[i] != NULL; i++) {
Lucy_TestQPLogic_Logical_Test_t test_func = logical_test_funcs[i];
TestQueryParser *test_case = test_func(BOOLOP_AND);
Query *tree = QParser_Tree(and_parser, test_case->query_string);
Query *parsed = QParser_Parse(and_parser, test_case->query_string);
Hits *hits = IxSearcher_Hits(searcher, (Obj*)parsed, 0, 10, NULL);
TEST_TRUE(batch, Query_Equals(tree, (Obj*)test_case->tree),
"tree() AND %s", (char*)CB_Get_Ptr8(test_case->query_string));
TEST_INT_EQ(batch, Hits_Total_Hits(hits), test_case->num_hits,
"hits: AND %s", (char*)CB_Get_Ptr8(test_case->query_string));
DECREF(hits);
DECREF(parsed);
DECREF(tree);
DECREF(test_case);
}
// Run tests for QParser_Prune().
for (i = 0; prune_test_funcs[i] != NULL; i++) {
Lucy_TestQPLogic_Prune_Test_t test_func = prune_test_funcs[i];
TestQueryParser *test_case = test_func();
CharBuf *qstring = test_case->tree
? Query_To_String(test_case->tree)
: CB_new_from_trusted_utf8("(NULL)", 6);
Query *tree = test_case->tree;
Query *wanted = test_case->expanded;
Query *pruned = QParser_Prune(or_parser, tree);
Query *expanded;
Hits *hits;
TEST_TRUE(batch, Query_Equals(pruned, (Obj*)wanted),
"prune() %s", (char*)CB_Get_Ptr8(qstring));
expanded = QParser_Expand(or_parser, pruned);
hits = IxSearcher_Hits(searcher, (Obj*)expanded, 0, 10, NULL);
TEST_INT_EQ(batch, Hits_Total_Hits(hits), test_case->num_hits,
"hits: %s", (char*)CB_Get_Ptr8(qstring));
DECREF(hits);
DECREF(expanded);
DECREF(pruned);
DECREF(qstring);
DECREF(test_case);
}
DECREF(and_parser);
DECREF(or_parser);
DECREF(searcher);
DECREF(folder);
DECREF(batch);
}
static void
test_Store_and_Fetch(TestBatch *batch) {
Hash *hash = Hash_new(100);
Hash *dupe = Hash_new(100);
const uint32_t starting_cap = Hash_Get_Capacity(hash);
VArray *expected = VA_new(100);
VArray *got = VA_new(100);
ZombieCharBuf *twenty = ZCB_WRAP_STR("20", 2);
ZombieCharBuf *forty = ZCB_WRAP_STR("40", 2);
ZombieCharBuf *foo = ZCB_WRAP_STR("foo", 3);
for (int32_t i = 0; i < 100; i++) {
CharBuf *cb = CB_newf("%i32", i);
Hash_Store(hash, (Obj*)cb, (Obj*)cb);
Hash_Store(dupe, (Obj*)cb, INCREF(cb));
VA_Push(expected, INCREF(cb));
}
TEST_TRUE(batch, Hash_Equals(hash, (Obj*)dupe), "Equals");
TEST_INT_EQ(batch, Hash_Get_Capacity(hash), starting_cap,
"Initial capacity sufficient (no rebuilds)");
for (int32_t i = 0; i < 100; i++) {
Obj *key = VA_Fetch(expected, i);
Obj *elem = Hash_Fetch(hash, key);
VA_Push(got, (Obj*)INCREF(elem));
}
TEST_TRUE(batch, VA_Equals(got, (Obj*)expected),
"basic Store and Fetch");
TEST_INT_EQ(batch, Hash_Get_Size(hash), 100,
"size incremented properly by Hash_Store");
TEST_TRUE(batch, Hash_Fetch(hash, (Obj*)foo) == NULL,
"Fetch against non-existent key returns NULL");
Hash_Store(hash, (Obj*)forty, INCREF(foo));
TEST_TRUE(batch, ZCB_Equals(foo, Hash_Fetch(hash, (Obj*)forty)),
"Hash_Store replaces existing value");
TEST_FALSE(batch, Hash_Equals(hash, (Obj*)dupe),
"replacement value spoils equals");
TEST_INT_EQ(batch, Hash_Get_Size(hash), 100,
"size unaffected after value replaced");
TEST_TRUE(batch, Hash_Delete(hash, (Obj*)forty) == (Obj*)foo,
"Delete returns value");
DECREF(foo);
TEST_INT_EQ(batch, Hash_Get_Size(hash), 99,
"size decremented by successful Delete");
TEST_TRUE(batch, Hash_Delete(hash, (Obj*)forty) == NULL,
"Delete returns NULL when key not found");
TEST_INT_EQ(batch, Hash_Get_Size(hash), 99,
"size not decremented by unsuccessful Delete");
DECREF(Hash_Delete(dupe, (Obj*)forty));
TEST_TRUE(batch, VA_Equals(got, (Obj*)expected), "Equals after Delete");
Hash_Clear(hash);
TEST_TRUE(batch, Hash_Fetch(hash, (Obj*)twenty) == NULL, "Clear");
TEST_TRUE(batch, Hash_Get_Size(hash) == 0, "size is 0 after Clear");
DECREF(hash);
DECREF(dupe);
DECREF(got);
DECREF(expected);
}
PolyReader*
PolyReader_do_open(PolyReader *self, Obj *index, Snapshot *snapshot,
IndexManager *manager) {
PolyReaderIVARS *const ivars = PolyReader_IVARS(self);
Folder *folder = S_derive_folder(index);
uint64_t last_gen = 0;
PolyReader_init(self, NULL, folder, snapshot, manager, NULL);
DECREF(folder);
if (manager) {
if (!S_obtain_deletion_lock(self)) {
DECREF(self);
THROW(LOCKERR, "Couldn't get deletion lock");
}
}
while (1) {
CharBuf *target_snap_file;
// If a Snapshot was supplied, use its file.
if (snapshot) {
target_snap_file = Snapshot_Get_Path(snapshot);
if (!target_snap_file) {
THROW(ERR, "Supplied snapshot objects must not be empty");
}
else {
CB_Inc_RefCount(target_snap_file);
}
}
else {
// Otherwise, pick the most recent snap file.
target_snap_file = IxFileNames_latest_snapshot(folder);
// No snap file? Looks like the index is empty. We can stop now
// and return NULL.
if (!target_snap_file) { break; }
}
// Derive "generation" of this snapshot file from its name.
uint64_t gen = IxFileNames_extract_gen(target_snap_file);
// Get a read lock on the most recent snapshot file if indicated.
if (manager) {
if (!S_obtain_read_lock(self, target_snap_file)) {
DECREF(self);
THROW(LOCKERR, "Couldn't get read lock for %o",
target_snap_file);
}
}
// Testing only.
if (PolyReader_race_condition_debug1) {
ZombieCharBuf *temp = ZCB_WRAP_STR("temp", 4);
if (Folder_Exists(folder, (CharBuf*)temp)) {
bool success = Folder_Rename(folder, (CharBuf*)temp,
PolyReader_race_condition_debug1);
if (!success) { RETHROW(INCREF(Err_get_error())); }
}
PolyReader_debug1_num_passes++;
}
// If a Snapshot object was passed in, the file has already been read.
// If that's not the case, we must read the file we just picked.
if (!snapshot) {
struct try_read_snapshot_context context;
context.snapshot = ivars->snapshot;
context.folder = folder;
context.path = target_snap_file;
Err *error = Err_trap(S_try_read_snapshot, &context);
if (error) {
S_release_read_lock(self);
DECREF(target_snap_file);
if (last_gen < gen) { // Index updated, so try again.
DECREF(error);
last_gen = gen;
continue;
}
else { // Real error.
if (manager) { S_release_deletion_lock(self); }
RETHROW(error);
}
}
}
/* It's possible, though unlikely, for an Indexer to delete files
* out from underneath us after the snapshot file is read but before
* we've got SegReaders holding open all the required files. If we
* failed to open something, see if we can find a newer snapshot file.
* If we can, then the exception was due to the race condition. If
* not, we have a real exception, so throw an error. */
struct try_open_elements_context context;
context.self = self;
context.seg_readers = NULL;
Err *error = Err_trap(S_try_open_elements, &context);
if (error) {
S_release_read_lock(self);
DECREF(target_snap_file);
if (last_gen < gen) { // Index updated, so try again.
DECREF(error);
last_gen = gen;
}
else { // Real error.
if (manager) { S_release_deletion_lock(self); }
RETHROW(error);
}
}
else { // Succeeded.
S_init_sub_readers(self, (VArray*)context.seg_readers);
DECREF(context.seg_readers);
DECREF(target_snap_file);
break;
}
}
if (manager) { S_release_deletion_lock(self); }
return self;
}