Exemple #1
0
static void
test_fields(TestBatch *batch)
{
    Segment *segment = Seg_new(1);
    ZombieCharBuf *foo = ZCB_WRAP_STR("foo",3 );
    ZombieCharBuf *bar = ZCB_WRAP_STR("bar", 3);
    ZombieCharBuf *baz = ZCB_WRAP_STR("baz", 3);
    int32_t field_num; 
    
    field_num = Seg_Add_Field(segment, (CharBuf*)foo);
    TEST_TRUE(batch, field_num == 1, 
        "Add_Field returns field number, and field numbers start at 1");
    field_num = Seg_Add_Field(segment, (CharBuf*)bar);
    TEST_TRUE(batch, field_num == 2, "add a second field");
    field_num = Seg_Add_Field(segment, (CharBuf*)foo);
    TEST_TRUE(batch, field_num == 1,
        "Add_Field returns existing field number if field is already known");

    TEST_TRUE(batch, ZCB_Equals(bar, (Obj*)Seg_Field_Name(segment, 2)),
        "Field_Name");
    TEST_TRUE(batch, Seg_Field_Name(segment, 3) == NULL, 
        "Field_Name returns NULL for unknown field number");
    TEST_TRUE(batch, Seg_Field_Num(segment, (CharBuf*)bar) == 2,
        "Field_Num");
    TEST_TRUE(batch, Seg_Field_Num(segment, (CharBuf*)baz) == 0, 
        "Field_Num returns 0 for unknown field name");

    DECREF(segment);
}
Exemple #2
0
static void
test_Dump_Load_and_Equals(TestBatch *batch)
{
    CharBuf *EN          = (CharBuf*)ZCB_WRAP_STR("en", 2); 
    CharBuf *ES          = (CharBuf*)ZCB_WRAP_STR("es", 2); 
    Stemmer *stemmer     = Stemmer_new(EN);
    Stemmer *other       = Stemmer_new(ES);
    Obj     *dump        = (Obj*)Stemmer_Dump(stemmer);
    Obj     *other_dump  = (Obj*)Stemmer_Dump(other);
    Stemmer *clone       = (Stemmer*)Stemmer_Load(other, dump);
    Stemmer *other_clone = (Stemmer*)Stemmer_Load(other, other_dump);

    TEST_FALSE(batch, Stemmer_Equals(stemmer,
        (Obj*)other), "Equals() false with different language");
    TEST_TRUE(batch, Stemmer_Equals(stemmer,
        (Obj*)clone), "Dump => Load round trip");
    TEST_TRUE(batch, Stemmer_Equals(other,
        (Obj*)other_clone), "Dump => Load round trip");

    DECREF(stemmer);
    DECREF(dump);
    DECREF(clone);
    DECREF(other);
    DECREF(other_dump);
    DECREF(other_clone);
}
Exemple #3
0
static void
test_Write_File_and_Read_File(TestBatch *batch)
{
    RAMFolder *folder  = RAMFolder_new(NULL);
    Segment   *segment = Seg_new(100);
    Segment   *got     = Seg_new(100);
    CharBuf   *meta;
    CharBuf   *flotsam = (CharBuf*)ZCB_WRAP_STR("flotsam", 7);
    CharBuf   *jetsam  = (CharBuf*)ZCB_WRAP_STR("jetsam", 6);

    Seg_Set_Count(segment, 111);
    Seg_Store_Metadata_Str(segment, "foo", 3, (Obj*)CB_newf("bar"));
    Seg_Add_Field(segment, flotsam);
    Seg_Add_Field(segment, jetsam);
    
    RAMFolder_MkDir(folder, Seg_Get_Name(segment));
    Seg_Write_File(segment, (Folder*)folder);
    Seg_Read_File(got, (Folder*)folder);

    TEST_TRUE(batch, Seg_Get_Count(got) == Seg_Get_Count(segment), 
        "Round-trip count through file");
    TEST_TRUE(batch, 
        Seg_Field_Num(got, jetsam) == Seg_Field_Num(segment, jetsam), 
        "Round trip field names through file");
    meta = (CharBuf*)Seg_Fetch_Metadata_Str(got, "foo", 3);
    TEST_TRUE(batch, meta && CB_Is_A(meta, CHARBUF) 
        && CB_Equals_Str(meta, "bar", 3), "Round trip metadata through file");

    DECREF(got);
    DECREF(segment);
    DECREF(folder);
}
bool_t
CFReader_local_delete(CompoundFileReader *self, const CharBuf *name) {
    Hash *record = (Hash*)Hash_Delete(self->records, (Obj*)name);
    DECREF(record);

    if (record == NULL) {
        return Folder_Local_Delete(self->real_folder, name);
    }
    else {
        // Once the number of virtual files falls to 0, remove the compound
        // files.
        if (Hash_Get_Size(self->records) == 0) {
            CharBuf *cf_file = (CharBuf*)ZCB_WRAP_STR("cf.dat", 6);
            if (!Folder_Delete(self->real_folder, cf_file)) {
                return false;
            }
            CharBuf *cfmeta_file = (CharBuf*)ZCB_WRAP_STR("cfmeta.json", 11);
            if (!Folder_Delete(self->real_folder, cfmeta_file)) {
                return false;

            }
        }
        return true;
    }
}
Exemple #5
0
static void
test_all(TestBatch *batch) {
    CharBuf  *foo           = (CharBuf*)ZCB_WRAP_STR("foo", 3);
    CharBuf  *boffo         = (CharBuf*)ZCB_WRAP_STR("boffo", 5);
    CharBuf  *foo_boffo     = (CharBuf*)ZCB_WRAP_STR("foo/boffo", 9);
    CharBuf  *test_dir      = (CharBuf*)ZCB_WRAP_STR("_fsdir_test", 11);
    FSFolder *folder        = FSFolder_new(test_dir);
    bool_t    saw_foo       = false;
    bool_t    saw_boffo     = false;
    bool_t    foo_was_dir   = false;
    bool_t    boffo_was_dir = false;
    int       count         = 0;

    // Clean up after previous failed runs.
    FSFolder_Delete(folder, foo_boffo);
    FSFolder_Delete(folder, foo);
    FSFolder_Delete(folder, boffo);
    rmdir("_fsdir_test");

    FSFolder_Initialize(folder);
    FSFolder_MkDir(folder, foo);
    OutStream *outstream = FSFolder_Open_Out(folder, boffo);
    DECREF(outstream);
    outstream = FSFolder_Open_Out(folder, foo_boffo);
    DECREF(outstream);

    FSDirHandle  *dh    = FSDH_open(test_dir);
    CharBuf      *entry = FSDH_Get_Entry(dh);
    while (FSDH_Next(dh)) {
        count++;
        if (CB_Equals(entry, (Obj*)foo)) {
            saw_foo = true;
            foo_was_dir = FSDH_Entry_Is_Dir(dh);
        }
        else if (CB_Equals(entry, (Obj*)boffo)) {
            saw_boffo = true;
            boffo_was_dir = FSDH_Entry_Is_Dir(dh);
        }
    }
    TEST_INT_EQ(batch, 2, count, "correct number of entries");
    TEST_TRUE(batch, saw_foo, "Directory was iterated over");
    TEST_TRUE(batch, foo_was_dir,
              "Dir correctly identified by Entry_Is_Dir");
    TEST_TRUE(batch, saw_boffo, "File was iterated over");
    TEST_FALSE(batch, boffo_was_dir,
               "File correctly identified by Entry_Is_Dir");

    DECREF(dh);
    FSFolder_Delete(folder, foo_boffo);
    FSFolder_Delete(folder, foo);
    FSFolder_Delete(folder, boffo);
    DECREF(folder);
    rmdir("_fsdir_test");
}
Exemple #6
0
static void
test_Keys_Values_Iter(TestBatch *batch) {
    Hash     *hash     = Hash_new(0); // trigger multiple rebuilds.
    VArray   *expected = VA_new(100);
    VArray   *keys;
    VArray   *values;

    for (uint32_t i = 0; i < 500; i++) {
        CharBuf *cb = CB_newf("%u32", i);
        Hash_Store(hash, (Obj*)cb, (Obj*)cb);
        VA_Push(expected, INCREF(cb));
    }

    VA_Sort(expected, NULL, NULL);

    keys   = Hash_Keys(hash);
    values = Hash_Values(hash);
    VA_Sort(keys, NULL, NULL);
    VA_Sort(values, NULL, NULL);
    TEST_TRUE(batch, VA_Equals(keys, (Obj*)expected), "Keys");
    TEST_TRUE(batch, VA_Equals(values, (Obj*)expected), "Values");
    VA_Clear(keys);
    VA_Clear(values);

    {
        Obj *key;
        Obj *value;
        Hash_Iterate(hash);
        while (Hash_Next(hash, &key, &value)) {
            VA_Push(keys, INCREF(key));
            VA_Push(values, INCREF(value));
        }
    }

    VA_Sort(keys, NULL, NULL);
    VA_Sort(values, NULL, NULL);
    TEST_TRUE(batch, VA_Equals(keys, (Obj*)expected), "Keys from Iter");
    TEST_TRUE(batch, VA_Equals(values, (Obj*)expected), "Values from Iter");

    {
        ZombieCharBuf *forty = ZCB_WRAP_STR("40", 2);
        ZombieCharBuf *nope  = ZCB_WRAP_STR("nope", 4);
        Obj *key = Hash_Find_Key(hash, (Obj*)forty, ZCB_Hash_Sum(forty));
        TEST_TRUE(batch, Obj_Equals(key, (Obj*)forty), "Find_Key");
        key = Hash_Find_Key(hash, (Obj*)nope, ZCB_Hash_Sum(nope)),
        TEST_TRUE(batch, key == NULL,
                  "Find_Key returns NULL for non-existent key");
    }

    DECREF(hash);
    DECREF(expected);
    DECREF(keys);
    DECREF(values);
}
Exemple #7
0
static void
test_spew_and_slurp(TestBatch *batch) {
    Obj *dump = S_make_dump();
    Folder *folder = (Folder*)RAMFolder_new(NULL);

    CharBuf *foo = (CharBuf*)ZCB_WRAP_STR("foo", 3);
    bool_t result = Json_spew_json(dump, folder, foo);
    TEST_TRUE(batch, result, "spew_json returns true on success");
    TEST_TRUE(batch, Folder_Exists(folder, foo),
              "spew_json wrote file");

    Obj *got = Json_slurp_json(folder, foo);
    TEST_TRUE(batch, got && Obj_Equals(dump, got),
              "Round trip through spew_json and slurp_json");
    DECREF(got);

    Err_set_error(NULL);
    result = Json_spew_json(dump, folder, foo);
    TEST_FALSE(batch, result, "Can't spew_json when file exists");
    TEST_TRUE(batch, Err_get_error() != NULL,
              "Failed spew_json sets Err_error");

    Err_set_error(NULL);
    CharBuf *bar = (CharBuf*)ZCB_WRAP_STR("bar", 3);
    got = Json_slurp_json(folder, bar);
    TEST_TRUE(batch, got == NULL,
              "slurp_json returns NULL when file doesn't exist");
    TEST_TRUE(batch, Err_get_error() != NULL,
              "Failed slurp_json sets Err_error");

    CharBuf *boffo = (CharBuf*)ZCB_WRAP_STR("boffo", 5);

    FileHandle *fh
        = Folder_Open_FileHandle(folder, boffo, FH_CREATE | FH_WRITE_ONLY);
    FH_Write(fh, "garbage", 7);
    DECREF(fh);

    Err_set_error(NULL);
    got = Json_slurp_json(folder, boffo);
    TEST_TRUE(batch, got == NULL,
              "slurp_json returns NULL when file doesn't contain valid JSON");
    TEST_TRUE(batch, Err_get_error() != NULL,
              "Failed slurp_json sets Err_error");
    DECREF(got);

    DECREF(dump);
    DECREF(folder);
}
Exemple #8
0
Lock*
IxManager_make_merge_lock(IndexManager *self)
{
    ZombieCharBuf *merge_lock_name = ZCB_WRAP_STR("merge", 5);
    LockFactory *lock_factory = S_obtain_lock_factory(self);
    return LockFact_Make_Lock(lock_factory, (CharBuf*)merge_lock_name,
        self->merge_lock_timeout, self->merge_lock_interval);
}
Exemple #9
0
Lock*
IxManager_make_deletion_lock(IndexManager *self)
{
    ZombieCharBuf *lock_name = ZCB_WRAP_STR("deletion", 8);
    LockFactory *lock_factory = S_obtain_lock_factory(self);
    return LockFact_Make_Lock(lock_factory, (CharBuf*)lock_name, 
        self->deletion_lock_timeout, self->deletion_lock_interval);
}
Exemple #10
0
static void
S_verify_bad_syntax(TestBatch *batch, const char *bad, const char *mess) {
    ZombieCharBuf *has_errors = ZCB_WRAP_STR(bad, strlen(bad));
    Err_set_error(NULL);
    Obj *not_json = Json_from_json((CharBuf*)has_errors);
    TEST_TRUE(batch, not_json == NULL, "from_json returns NULL: %s", mess);
    TEST_TRUE(batch, Err_get_error() != NULL, "from_json sets Err_error: %s",
              mess);
}
Exemple #11
0
bool_t
LFLock_maybe_delete_file(LockFileLock *self, const CharBuf *path,
                         bool_t delete_mine, bool_t delete_other) {
    Folder *folder  = self->folder;
    bool_t  success = false;
    ZombieCharBuf *scratch = ZCB_WRAP(path);

    // Only delete locks that start with our lock name.
    CharBuf *lock_dir_name = (CharBuf*)ZCB_WRAP_STR("locks", 5);
    if (!ZCB_Starts_With(scratch, lock_dir_name)) {
        return false;
    }
    ZCB_Nip(scratch, CB_Get_Size(lock_dir_name) + 1);
    if (!ZCB_Starts_With(scratch, self->name)) {
        return false;
    }

    // Attempt to delete dead lock file.
    if (Folder_Exists(folder, path)) {
        Hash *hash = (Hash*)Json_slurp_json(folder, path);
        if (hash != NULL && Obj_Is_A((Obj*)hash, HASH)) {
            CharBuf *pid_buf = (CharBuf*)Hash_Fetch_Str(hash, "pid", 3);
            CharBuf *host    = (CharBuf*)Hash_Fetch_Str(hash, "host", 4);
            CharBuf *name
                = (CharBuf*)Hash_Fetch_Str(hash, "name", 4);

            // Match hostname and lock name.
            if (host != NULL
                && CB_Equals(host, (Obj*)self->host)
                && name != NULL
                && CB_Equals(name, (Obj*)self->name)
                && pid_buf != NULL
               ) {
                // Verify that pid is either mine or dead.
                int pid = (int)CB_To_I64(pid_buf);
                if ((delete_mine && pid == PID_getpid())  // This process.
                    || (delete_other && !PID_active(pid)) // Dead pid.
                   ) {
                    if (Folder_Delete(folder, path)) {
                        success = true;
                    }
                    else {
                        CharBuf *mess
                            = MAKE_MESS("Can't delete '%o'", path);
                        DECREF(hash);
                        Err_throw_mess(ERR, mess);
                    }
                }
            }
        }
        DECREF(hash);
    }

    return success;
}
Exemple #12
0
static Obj*
S_new_testobj() {
    ZombieCharBuf *klass = ZCB_WRAP_STR("TestObj", 7);
    Obj *obj;
    VTable *vtable = VTable_fetch_vtable((CharBuf*)klass);
    if (!vtable) {
        vtable = VTable_singleton((CharBuf*)klass, OBJ);
    }
    obj = VTable_Make_Obj(vtable);
    return Obj_init(obj);
}
Exemple #13
0
void
IxManager_write_merge_data(IndexManager *self, int64_t cutoff)
{
    ZombieCharBuf *merge_json = ZCB_WRAP_STR("merge.json", 10);
    Hash *data = Hash_new(1);
    bool_t success;
    Hash_Store_Str(data, "cutoff", 6, (Obj*)CB_newf("%i64", cutoff));
    success = Json_spew_json((Obj*)data, self->folder, (CharBuf*)merge_json);
    DECREF(data);
    if (!success) {
        THROW(ERR, "Failed to write to %o", merge_json);
    }
}
Exemple #14
0
static void
S_zap_dead_merge(FilePurger *self, Hash *candidates)
{
    IndexManager *manager = self->manager;
    Lock *merge_lock   = IxManager_Make_Merge_Lock(manager);

    Lock_Clear_Stale(merge_lock);
    if (!Lock_Is_Locked(merge_lock)) { 
        Hash *merge_data = IxManager_Read_Merge_Data(manager);
        Obj  *cutoff = merge_data 
                     ? Hash_Fetch_Str(merge_data, "cutoff", 6) 
                     : NULL;

        if (cutoff) {
            CharBuf *cutoff_seg = Seg_num_to_name(Obj_To_I64(cutoff));
            if (Folder_Exists(self->folder, cutoff_seg)) {
                ZombieCharBuf *merge_json = ZCB_WRAP_STR("merge.json", 10);
                DirHandle *dh = Folder_Open_Dir(self->folder, cutoff_seg);
                CharBuf *entry = dh ? DH_Get_Entry(dh) : NULL;
                CharBuf *filepath = CB_new(32);

                if (!dh) {
                    THROW(ERR, "Can't open segment dir '%o'", filepath);
                }

                Hash_Store(candidates, (Obj*)cutoff_seg, INCREF(&EMPTY));
                Hash_Store(candidates, (Obj*)merge_json, INCREF(&EMPTY));
                while (DH_Next(dh)) {
                    // TODO: recursively delete subdirs within seg dir.
                    CB_setf(filepath, "%o/%o", cutoff_seg, entry);
                    Hash_Store(candidates, (Obj*)filepath, INCREF(&EMPTY));
                }
                DECREF(filepath);
                DECREF(dh);
            }
            DECREF(cutoff_seg);
        }

        DECREF(merge_data);
    }

    DECREF(merge_lock);
    return;
}
Exemple #15
0
Hash*
IxManager_read_merge_data(IndexManager *self)
{
    ZombieCharBuf *merge_json = ZCB_WRAP_STR("merge.json", 10);
    if (Folder_Exists(self->folder, (CharBuf*)merge_json)) {
        Hash *stuff 
            = (Hash*)Json_slurp_json(self->folder, (CharBuf*)merge_json);
        if (stuff) {
            CERTIFY(stuff, HASH);
            return stuff;
        }
        else {
            return Hash_new(0);
        }
    }
    else {
        return NULL;
    }
}
Exemple #16
0
static void
S_set_error(CharBuf *mess, char *json, char *limit, int line,
            const char *func) {
    if (func) {
        CB_catf(mess, " at %s %s line %i32 near ", func, __FILE__,
                (int32_t)line);
    }
    else {
        CB_catf(mess, " at %s line %i32 near ", __FILE__, (int32_t)line);
    }

    // Append escaped text.
    int64_t len = limit - json;
    if (len > 32) {
        const char *end = StrHelp_back_utf8_char(json + 32, json);
        len = end - json;
    }
    ZombieCharBuf *snippet = ZCB_WRAP_STR(json, len);
    S_append_json_string((Obj*)snippet, mess);

    // Set Err_error.
    Err_set_error(Err_new(mess));
}
Exemple #17
0
static void
test_Equals(TestBatch *batch) {
    Hash *hash  = Hash_new(0);
    Hash *other = Hash_new(0);
    ZombieCharBuf *stuff = ZCB_WRAP_STR("stuff", 5);

    TEST_TRUE(batch, Hash_Equals(hash, (Obj*)other),
              "Empty hashes are equal");

    Hash_Store_Str(hash, "foo", 3, (Obj*)CFISH_TRUE);
    TEST_FALSE(batch, Hash_Equals(hash, (Obj*)other),
               "Add one pair and Equals returns false");

    Hash_Store_Str(other, "foo", 3, (Obj*)CFISH_TRUE);
    TEST_TRUE(batch, Hash_Equals(hash, (Obj*)other),
              "Add a matching pair and Equals returns true");

    Hash_Store_Str(other, "foo", 3, INCREF(stuff));
    TEST_FALSE(batch, Hash_Equals(hash, (Obj*)other),
               "Non-matching value spoils Equals");

    DECREF(hash);
    DECREF(other);
}
static Folder*
S_create_index() {
    Schema     *schema  = (Schema*)TestSchema_new();
    RAMFolder  *folder  = RAMFolder_new(NULL);
    VArray     *doc_set = TestUtils_doc_set();
    Indexer    *indexer = Indexer_new(schema, (Obj*)folder, NULL, 0);
    uint32_t i, max;

    CharBuf *field = (CharBuf*)ZCB_WRAP_STR("content", 7);
    for (i = 0, max = VA_Get_Size(doc_set); i < max; i++) {
        Doc *doc = Doc_new(NULL, 0);
        Doc_Store(doc, field, VA_Fetch(doc_set, i));
        Indexer_Add_Doc(indexer, doc, 1.0f);
        DECREF(doc);
    }

    Indexer_Commit(indexer);

    DECREF(doc_set);
    DECREF(indexer);
    DECREF(schema);

    return (Folder*)folder;
}
Exemple #19
0
CompoundFileReader*
CFReader_do_open(CompoundFileReader *self, Folder *folder) {
    CharBuf *cfmeta_file = (CharBuf*)ZCB_WRAP_STR("cfmeta.json", 11);
    Hash *metadata = (Hash*)Json_slurp_json((Folder*)folder, cfmeta_file);
    Err *error = NULL;

    Folder_init((Folder*)self, Folder_Get_Path(folder));

    // Parse metadata file.
    if (!metadata || !Hash_Is_A(metadata, HASH)) {
        error = Err_new(CB_newf("Can't read '%o' in '%o'", cfmeta_file,
                                Folder_Get_Path(folder)));
    }
    else {
        Obj *format = Hash_Fetch_Str(metadata, "format", 6);
        self->format = format ? (int32_t)Obj_To_I64(format) : 0;
        self->records = (Hash*)INCREF(Hash_Fetch_Str(metadata, "files", 5));
        if (self->format < 1) {
            error = Err_new(CB_newf("Corrupt %o file: Missing or invalid 'format'",
                                    cfmeta_file));
        }
        else if (self->format > CFWriter_current_file_format) {
            error = Err_new(CB_newf("Unsupported compound file format: %i32 "
                                    "(current = %i32", self->format,
                                    CFWriter_current_file_format));
        }
        else if (!self->records) {
            error = Err_new(CB_newf("Corrupt %o file: missing 'files' key",
                                    cfmeta_file));
        }
    }
    DECREF(metadata);
    if (error) {
        Err_set_error(error);
        DECREF(self);
        return NULL;
    }

    // Open an instream which we'll clone over and over.
    CharBuf *cf_file = (CharBuf*)ZCB_WRAP_STR("cf.dat", 6);
    self->instream = Folder_Open_In(folder, cf_file);
    if (!self->instream) {
        ERR_ADD_FRAME(Err_get_error());
        DECREF(self);
        return NULL;
    }

    // Assign.
    self->real_folder = (Folder*)INCREF(folder);

    // Strip directory name from filepaths for old format.
    if (self->format == 1) {
        VArray *files = Hash_Keys(self->records);
        ZombieCharBuf *filename = ZCB_BLANK();
        ZombieCharBuf *folder_name
            = IxFileNames_local_part(Folder_Get_Path(folder), ZCB_BLANK());
        size_t folder_name_len = ZCB_Length(folder_name);

        for (uint32_t i = 0, max = VA_Get_Size(files); i < max; i++) {
            CharBuf *orig = (CharBuf*)VA_Fetch(files, i);
            if (CB_Starts_With(orig, (CharBuf*)folder_name)) {
                Obj *record = Hash_Delete(self->records, (Obj*)orig);
                ZCB_Assign(filename, orig);
                ZCB_Nip(filename, folder_name_len + sizeof(DIR_SEP) - 1);
                Hash_Store(self->records, (Obj*)filename, (Obj*)record);
            }
        }

        DECREF(files);
    }

    return self;
}
Exemple #20
0
bool_t
LFLock_request(LockFileLock *self) {
    Hash   *file_data;
    bool_t wrote_json;
    bool_t success = false;
    bool_t deletion_failed = false;

    if (Folder_Exists(self->folder, self->lock_path)) {
        Err_set_error((Err*)LockErr_new(CB_newf("Can't obtain lock: '%o' exists",
                                                self->lock_path)));
        return false;
    }

    // Create the "locks" subdirectory if necessary.
    CharBuf *lock_dir_name = (CharBuf*)ZCB_WRAP_STR("locks", 5);
    if (!Folder_Exists(self->folder, lock_dir_name)) {
        if (!Folder_MkDir(self->folder, lock_dir_name)) {
            Err *mkdir_err = (Err*)CERTIFY(Err_get_error(), ERR);
            LockErr *err = LockErr_new(CB_newf("Can't create 'locks' directory: %o",
                                               Err_Get_Mess(mkdir_err)));
            // Maybe our attempt failed because another process succeeded.
            if (Folder_Find_Folder(self->folder, lock_dir_name)) {
                DECREF(err);
            }
            else {
                // Nope, everything failed, so bail out.
                Err_set_error((Err*)err);
                return false;
            }
        }
    }

    // Prepare to write pid, lock name, and host to the lock file as JSON.
    file_data = Hash_new(3);
    Hash_Store_Str(file_data, "pid", 3,
                   (Obj*)CB_newf("%i32", (int32_t)PID_getpid()));
    Hash_Store_Str(file_data, "host", 4, INCREF(self->host));
    Hash_Store_Str(file_data, "name", 4, INCREF(self->name));

    // Write to a temporary file, then use the creation of a hard link to
    // ensure atomic but non-destructive creation of the lockfile with its
    // complete contents.
    wrote_json = Json_spew_json((Obj*)file_data, self->folder, self->link_path);
    if (wrote_json) {
        success = Folder_Hard_Link(self->folder, self->link_path,
                                   self->lock_path);
        if (!success) {
            Err *hard_link_err = (Err*)CERTIFY(Err_get_error(), ERR);
            Err_set_error((Err*)LockErr_new(CB_newf("Failed to obtain lock at '%o': %o",
                                                    self->lock_path,
                                                    Err_Get_Mess(hard_link_err))));
        }
        deletion_failed = !Folder_Delete(self->folder, self->link_path);
    }
    else {
        Err *spew_json_err = (Err*)CERTIFY(Err_get_error(), ERR);
        Err_set_error((Err*)LockErr_new(CB_newf("Failed to obtain lock at '%o': %o",
                                                self->lock_path,
                                                Err_Get_Mess(spew_json_err))));
    }
    DECREF(file_data);

    // Verify that our temporary file got zapped.
    if (wrote_json && deletion_failed) {
        CharBuf *mess = MAKE_MESS("Failed to delete '%o'", self->link_path);
        Err_throw_mess(ERR, mess);
    }

    return success;
}
Exemple #21
0
void
Hash_store_str(Hash *self, const char *key, size_t key_len, Obj *value) {
    ZombieCharBuf *key_buf = ZCB_WRAP_STR((char*)key, key_len);
    Hash_do_store(self, (Obj*)key_buf, value,
                  ZCB_Hash_Sum(key_buf), false);
}
Exemple #22
0
Obj*
Hash_delete_str(Hash *self, const char *key, size_t key_len) {
    ZombieCharBuf *key_buf = ZCB_WRAP_STR(key, key_len);
    return Hash_delete(self, (Obj*)key_buf);
}
Exemple #23
0
bool_t
IxManager_remove_merge_data(IndexManager *self)
{
    ZombieCharBuf *merge_json = ZCB_WRAP_STR("merge.json", 10);
    return Folder_Delete(self->folder, (CharBuf*)merge_json) != 0;
}
Exemple #24
0
void
Seg_store_metadata_str(Segment *self, const char *key, size_t key_len,
                       Obj *value) {
    ZombieCharBuf *k = ZCB_WRAP_STR((char*)key, key_len);
    Seg_Store_Metadata(self, (CharBuf*)k, value);
}
void
TestQPLogic_run_tests() {
    uint32_t i;
    TestBatch     *batch      = TestBatch_new(258);
    Folder        *folder     = S_create_index();
    IndexSearcher *searcher   = IxSearcher_new((Obj*)folder);
    QueryParser   *or_parser  = QParser_new(IxSearcher_Get_Schema(searcher),
                                            NULL, NULL, NULL);
    ZombieCharBuf *AND        = ZCB_WRAP_STR("AND", 3);
    QueryParser   *and_parser = QParser_new(IxSearcher_Get_Schema(searcher),
                                            NULL, (CharBuf*)AND, NULL);
    QParser_Set_Heed_Colons(or_parser, true);
    QParser_Set_Heed_Colons(and_parser, true);

    TestBatch_Plan(batch);

    // Run logical tests with default boolop of OR.
    for (i = 0; logical_test_funcs[i] != NULL; i++) {
        Lucy_TestQPLogic_Logical_Test_t test_func = logical_test_funcs[i];
        TestQueryParser *test_case = test_func(BOOLOP_OR);
        Query *tree     = QParser_Tree(or_parser, test_case->query_string);
        Query *parsed   = QParser_Parse(or_parser, test_case->query_string);
        Hits  *hits     = IxSearcher_Hits(searcher, (Obj*)parsed, 0, 10, NULL);

        TEST_TRUE(batch, Query_Equals(tree, (Obj*)test_case->tree),
                  "tree() OR   %s", (char*)CB_Get_Ptr8(test_case->query_string));
        TEST_INT_EQ(batch, Hits_Total_Hits(hits), test_case->num_hits,
                    "hits: OR   %s", (char*)CB_Get_Ptr8(test_case->query_string));
        DECREF(hits);
        DECREF(parsed);
        DECREF(tree);
        DECREF(test_case);
    }

    // Run logical tests with default boolop of AND.
    for (i = 0; logical_test_funcs[i] != NULL; i++) {
        Lucy_TestQPLogic_Logical_Test_t test_func = logical_test_funcs[i];
        TestQueryParser *test_case = test_func(BOOLOP_AND);
        Query *tree     = QParser_Tree(and_parser, test_case->query_string);
        Query *parsed   = QParser_Parse(and_parser, test_case->query_string);
        Hits  *hits     = IxSearcher_Hits(searcher, (Obj*)parsed, 0, 10, NULL);

        TEST_TRUE(batch, Query_Equals(tree, (Obj*)test_case->tree),
                  "tree() AND   %s", (char*)CB_Get_Ptr8(test_case->query_string));
        TEST_INT_EQ(batch, Hits_Total_Hits(hits), test_case->num_hits,
                    "hits: AND   %s", (char*)CB_Get_Ptr8(test_case->query_string));
        DECREF(hits);
        DECREF(parsed);
        DECREF(tree);
        DECREF(test_case);
    }

    // Run tests for QParser_Prune().
    for (i = 0; prune_test_funcs[i] != NULL; i++) {
        Lucy_TestQPLogic_Prune_Test_t test_func = prune_test_funcs[i];
        TestQueryParser *test_case = test_func();
        CharBuf *qstring = test_case->tree
                           ? Query_To_String(test_case->tree)
                           : CB_new_from_trusted_utf8("(NULL)", 6);
        Query *tree = test_case->tree;
        Query *wanted = test_case->expanded;
        Query *pruned   = QParser_Prune(or_parser, tree);
        Query *expanded;
        Hits  *hits;

        TEST_TRUE(batch, Query_Equals(pruned, (Obj*)wanted),
                  "prune()   %s", (char*)CB_Get_Ptr8(qstring));
        expanded = QParser_Expand(or_parser, pruned);
        hits = IxSearcher_Hits(searcher, (Obj*)expanded, 0, 10, NULL);
        TEST_INT_EQ(batch, Hits_Total_Hits(hits), test_case->num_hits,
                    "hits:    %s", (char*)CB_Get_Ptr8(qstring));

        DECREF(hits);
        DECREF(expanded);
        DECREF(pruned);
        DECREF(qstring);
        DECREF(test_case);
    }

    DECREF(and_parser);
    DECREF(or_parser);
    DECREF(searcher);
    DECREF(folder);
    DECREF(batch);
}
Exemple #26
0
static void
test_Store_and_Fetch(TestBatch *batch) {
    Hash          *hash         = Hash_new(100);
    Hash          *dupe         = Hash_new(100);
    const uint32_t starting_cap = Hash_Get_Capacity(hash);
    VArray        *expected     = VA_new(100);
    VArray        *got          = VA_new(100);
    ZombieCharBuf *twenty       = ZCB_WRAP_STR("20", 2);
    ZombieCharBuf *forty        = ZCB_WRAP_STR("40", 2);
    ZombieCharBuf *foo          = ZCB_WRAP_STR("foo", 3);

    for (int32_t i = 0; i < 100; i++) {
        CharBuf *cb = CB_newf("%i32", i);
        Hash_Store(hash, (Obj*)cb, (Obj*)cb);
        Hash_Store(dupe, (Obj*)cb, INCREF(cb));
        VA_Push(expected, INCREF(cb));
    }
    TEST_TRUE(batch, Hash_Equals(hash, (Obj*)dupe), "Equals");

    TEST_INT_EQ(batch, Hash_Get_Capacity(hash), starting_cap,
                "Initial capacity sufficient (no rebuilds)");

    for (int32_t i = 0; i < 100; i++) {
        Obj *key  = VA_Fetch(expected, i);
        Obj *elem = Hash_Fetch(hash, key);
        VA_Push(got, (Obj*)INCREF(elem));
    }

    TEST_TRUE(batch, VA_Equals(got, (Obj*)expected),
              "basic Store and Fetch");
    TEST_INT_EQ(batch, Hash_Get_Size(hash), 100,
                "size incremented properly by Hash_Store");

    TEST_TRUE(batch, Hash_Fetch(hash, (Obj*)foo) == NULL,
              "Fetch against non-existent key returns NULL");

    Hash_Store(hash, (Obj*)forty, INCREF(foo));
    TEST_TRUE(batch, ZCB_Equals(foo, Hash_Fetch(hash, (Obj*)forty)),
              "Hash_Store replaces existing value");
    TEST_FALSE(batch, Hash_Equals(hash, (Obj*)dupe),
               "replacement value spoils equals");
    TEST_INT_EQ(batch, Hash_Get_Size(hash), 100,
                "size unaffected after value replaced");

    TEST_TRUE(batch, Hash_Delete(hash, (Obj*)forty) == (Obj*)foo,
              "Delete returns value");
    DECREF(foo);
    TEST_INT_EQ(batch, Hash_Get_Size(hash), 99,
                "size decremented by successful Delete");
    TEST_TRUE(batch, Hash_Delete(hash, (Obj*)forty) == NULL,
              "Delete returns NULL when key not found");
    TEST_INT_EQ(batch, Hash_Get_Size(hash), 99,
                "size not decremented by unsuccessful Delete");
    DECREF(Hash_Delete(dupe, (Obj*)forty));
    TEST_TRUE(batch, VA_Equals(got, (Obj*)expected), "Equals after Delete");

    Hash_Clear(hash);
    TEST_TRUE(batch, Hash_Fetch(hash, (Obj*)twenty) == NULL, "Clear");
    TEST_TRUE(batch, Hash_Get_Size(hash) == 0, "size is 0 after Clear");

    DECREF(hash);
    DECREF(dupe);
    DECREF(got);
    DECREF(expected);
}
Exemple #27
0
PolyReader*
PolyReader_do_open(PolyReader *self, Obj *index, Snapshot *snapshot,
                   IndexManager *manager) {
    PolyReaderIVARS *const ivars = PolyReader_IVARS(self);
    Folder   *folder   = S_derive_folder(index);
    uint64_t  last_gen = 0;

    PolyReader_init(self, NULL, folder, snapshot, manager, NULL);
    DECREF(folder);

    if (manager) { 
        if (!S_obtain_deletion_lock(self)) {
            DECREF(self);
            THROW(LOCKERR, "Couldn't get deletion lock");
        }
    }

    while (1) {
        CharBuf *target_snap_file;

        // If a Snapshot was supplied, use its file.
        if (snapshot) {
            target_snap_file = Snapshot_Get_Path(snapshot);
            if (!target_snap_file) {
                THROW(ERR, "Supplied snapshot objects must not be empty");
            }
            else {
                CB_Inc_RefCount(target_snap_file);
            }
        }
        else {
            // Otherwise, pick the most recent snap file.
            target_snap_file = IxFileNames_latest_snapshot(folder);

            // No snap file?  Looks like the index is empty.  We can stop now
            // and return NULL.
            if (!target_snap_file) { break; }
        }

        // Derive "generation" of this snapshot file from its name.
        uint64_t gen = IxFileNames_extract_gen(target_snap_file);

        // Get a read lock on the most recent snapshot file if indicated.
        if (manager) {
            if (!S_obtain_read_lock(self, target_snap_file)) {
                DECREF(self);
                THROW(LOCKERR, "Couldn't get read lock for %o",
                      target_snap_file);
            }
        }

        // Testing only.
        if (PolyReader_race_condition_debug1) {
            ZombieCharBuf *temp = ZCB_WRAP_STR("temp", 4);
            if (Folder_Exists(folder, (CharBuf*)temp)) {
                bool success = Folder_Rename(folder, (CharBuf*)temp,
                                               PolyReader_race_condition_debug1);
                if (!success) { RETHROW(INCREF(Err_get_error())); }
            }
            PolyReader_debug1_num_passes++;
        }

        // If a Snapshot object was passed in, the file has already been read.
        // If that's not the case, we must read the file we just picked.
        if (!snapshot) {
            struct try_read_snapshot_context context;
            context.snapshot = ivars->snapshot;
            context.folder   = folder;
            context.path     = target_snap_file;
            Err *error = Err_trap(S_try_read_snapshot, &context);

            if (error) {
                S_release_read_lock(self);
                DECREF(target_snap_file);
                if (last_gen < gen) { // Index updated, so try again.
                    DECREF(error);
                    last_gen = gen;
                    continue;
                }
                else { // Real error.
                    if (manager) { S_release_deletion_lock(self); }
                    RETHROW(error);
                }
            }
        }

        /* It's possible, though unlikely, for an Indexer to delete files
         * out from underneath us after the snapshot file is read but before
         * we've got SegReaders holding open all the required files.  If we
         * failed to open something, see if we can find a newer snapshot file.
         * If we can, then the exception was due to the race condition.  If
         * not, we have a real exception, so throw an error. */
        struct try_open_elements_context context;
        context.self        = self;
        context.seg_readers = NULL;
        Err *error = Err_trap(S_try_open_elements, &context);
        if (error) {
            S_release_read_lock(self);
            DECREF(target_snap_file);
            if (last_gen < gen) { // Index updated, so try again.
                DECREF(error);
                last_gen = gen;
            }
            else { // Real error.
                if (manager) { S_release_deletion_lock(self); }
                RETHROW(error);
            }
        }
        else { // Succeeded.
            S_init_sub_readers(self, (VArray*)context.seg_readers);
            DECREF(context.seg_readers);
            DECREF(target_snap_file);
            break;
        }
    }

    if (manager) { S_release_deletion_lock(self); }

    return self;
}