예제 #1
0
CharBuf*
IxManager_make_snapshot_filename(IndexManager *self)
{
    VArray *files = Folder_List(self->folder);
    u32_t i, max;
    i32_t max_gen = 0;

    for (i = 0, max = VA_Get_Size(files); i < max; i++) {
        CharBuf *file = (CharBuf*)VA_Fetch(files, i);
        if (    CB_Starts_With_Str(file, "snapshot_", 9)
                && CB_Ends_With_Str(file, ".json", 5)
           ) {
            i32_t gen = IxFileNames_extract_gen(file);
            if (gen > max_gen) {
                max_gen = gen;
            }
        }
    }
    DECREF(files);

    {
        i32_t    new_gen = max_gen + 1;
        CharBuf *base_36 = StrHelp_to_base36(new_gen);
        CharBuf *snapfile = CB_newf("snapshot_%o.json", base_36);
        DECREF(base_36);
        return snapfile;
    }
}
예제 #2
0
CharBuf*
IxManager_make_snapshot_filename(IndexManager *self)
{
    Folder *folder = (Folder*)CERTIFY(self->folder, FOLDER);
    DirHandle *dh = Folder_Open_Dir(folder, NULL);
    CharBuf *entry;
    uint64_t max_gen = 0;

    if (!dh) { RETHROW(INCREF(Err_get_error())); }
    entry = DH_Get_Entry(dh);
    while (DH_Next(dh)) {
        if (    CB_Starts_With_Str(entry, "snapshot_", 9)
            && CB_Ends_With_Str(entry, ".json", 5)
        ) {
            uint64_t gen = IxFileNames_extract_gen(entry);
            if (gen > max_gen) { max_gen = gen; }
        }
    }
    DECREF(dh);

    {
        uint64_t new_gen = max_gen + 1;
        char  base36[StrHelp_MAX_BASE36_BYTES];
        StrHelp_to_base36(new_gen, &base36);
        return CB_newf("snapshot_%s.json", &base36);
    }
}
예제 #3
0
파일: Err.c 프로젝트: pavansondur/lucy
void
Err_add_frame(Err *self, const char *file, int line, const char *func) {
    if (CB_Ends_With_Str(self->mess, "\n", 1)) { CB_Chop(self->mess, 1); }

    if (func != NULL) {
        CB_catf(self->mess, "\n\t%s at %s line %i32\n", func, file,
                (int32_t)line);
    }
    else {
        CB_catf(self->mess, "\n\tat %s line %i32\n", file, (int32_t)line);
    }
}
예제 #4
0
Lock*
IxManager_make_snapshot_read_lock(IndexManager *self, const CharBuf *filename)
{
    ZombieCharBuf *lock_name = ZCB_WRAP(filename);
    LockFactory *lock_factory = S_obtain_lock_factory(self);
    
    if (   !CB_Starts_With_Str(filename, "snapshot_", 9)
        || !CB_Ends_With_Str(filename, ".json", 5)
    ) {
        THROW(ERR, "Not a snapshot filename: %o", filename);
    }
        
    // Truncate ".json" from end of snapshot file name. 
    ZCB_Chop(lock_name, sizeof(".json") - 1);

    return LockFact_Make_Shared_Lock(lock_factory, (CharBuf*)lock_name, 1000, 100);
}
예제 #5
0
파일: Indexer.c 프로젝트: gitpan/KinoSearch
static CharBuf*
S_find_schema_file(Snapshot *snapshot)
{
    VArray *files = Snapshot_List(snapshot);
    uint32_t i, max;
    CharBuf *retval = NULL;
    for (i = 0, max = VA_Get_Size(files); i < max; i++) {
        CharBuf *file = (CharBuf*)VA_Fetch(files, i);
        if (   CB_Starts_With_Str(file, "schema_", 7)
            && CB_Ends_With_Str(file, ".json", 5)
        ) {
            retval = file;
            break;
        }
    }
    DECREF(files);
    return retval;
}
예제 #6
0
파일: Folder.c 프로젝트: pavansondur/lucy
Folder*
Folder_init(Folder *self, const CharBuf *path) {
    // Init.
    self->entries = Hash_new(16);

    // Copy.
    if (path == NULL) {
        self->path = CB_new_from_trusted_utf8("", 0);
    }
    else {
        // Copy path, strip trailing slash or equivalent.
        self->path = CB_Clone(path);
        if (CB_Ends_With_Str(self->path, DIR_SEP, strlen(DIR_SEP))) {
            CB_Chop(self->path, 1);
        }
    }

    ABSTRACT_CLASS_CHECK(self, FOLDER);
    return self;
}
예제 #7
0
파일: PolyReader.c 프로젝트: theory/lucy
void
S_try_open_elements(void *context) {
    struct try_open_elements_context *args
        = (struct try_open_elements_context*)context;
    PolyReader *self              = args->self;
    PolyReaderIVARS *const ivars  = PolyReader_IVARS(self);
    VArray     *files             = Snapshot_List(ivars->snapshot);
    Folder     *folder            = PolyReader_Get_Folder(self);
    uint32_t    num_segs          = 0;
    uint64_t    latest_schema_gen = 0;
    CharBuf    *schema_file       = NULL;

    // Find schema file, count segments.
    for (uint32_t i = 0, max = VA_Get_Size(files); i < max; i++) {
        CharBuf *entry = (CharBuf*)VA_Fetch(files, i);

        if (Seg_valid_seg_name(entry)) {
            num_segs++;
        }
        else if (CB_Starts_With_Str(entry, "schema_", 7)
                 && CB_Ends_With_Str(entry, ".json", 5)
                ) {
            uint64_t gen = IxFileNames_extract_gen(entry);
            if (gen > latest_schema_gen) {
                latest_schema_gen = gen;
                if (!schema_file) { schema_file = CB_Clone(entry); }
                else { CB_Mimic(schema_file, (Obj*)entry); }
            }
        }
    }

    // Read Schema.
    if (!schema_file) {
        DECREF(files);
        THROW(ERR, "Can't find a schema file.");
    }
    else {
        Hash *dump = (Hash*)Json_slurp_json(folder, schema_file);
        if (dump) { // read file successfully
            DECREF(ivars->schema);
            ivars->schema = (Schema*)CERTIFY(
                               VTable_Load_Obj(SCHEMA, (Obj*)dump), SCHEMA);
            DECREF(dump);
            DECREF(schema_file);
            schema_file = NULL;
        }
        else {
            CharBuf *mess = MAKE_MESS("Failed to parse %o", schema_file);
            DECREF(schema_file);
            DECREF(files);
            Err_throw_mess(ERR, mess);
        }
    }

    VArray *segments = VA_new(num_segs);
    for (uint32_t i = 0, max = VA_Get_Size(files); i < max; i++) {
        CharBuf *entry = (CharBuf*)VA_Fetch(files, i);

        // Create a Segment for each segmeta.
        if (Seg_valid_seg_name(entry)) {
            int64_t seg_num = IxFileNames_extract_gen(entry);
            Segment *segment = Seg_new(seg_num);

            // Bail if reading the file fails (probably because it's been
            // deleted and a new snapshot file has been written so we need to
            // retry).
            if (Seg_Read_File(segment, folder)) {
                VA_Push(segments, (Obj*)segment);
            }
            else {
                CharBuf *mess = MAKE_MESS("Failed to read %o", entry);
                DECREF(segment);
                DECREF(segments);
                DECREF(files);
                Err_throw_mess(ERR, mess);
            }
        }
    }

    // Sort the segments by age.
    VA_Sort(segments, NULL, NULL);

    // Open individual SegReaders.
    struct try_open_segreader_context seg_context;
    seg_context.schema   = PolyReader_Get_Schema(self);
    seg_context.folder   = folder;
    seg_context.snapshot = PolyReader_Get_Snapshot(self);
    seg_context.segments = segments;
    seg_context.result   = NULL;
    args->seg_readers = VA_new(num_segs);
    Err *error = NULL;
    for (uint32_t seg_tick = 0; seg_tick < num_segs; seg_tick++) {
        seg_context.seg_tick = seg_tick;
        error = Err_trap(S_try_open_segreader, &seg_context);
        if (error) {
            break;
        }
        VA_Push(args->seg_readers, (Obj*)seg_context.result);
        seg_context.result = NULL;
    }

    DECREF(segments);
    DECREF(files);
    if (error) {
        DECREF(args->seg_readers);
        args->seg_readers = NULL;
        RETHROW(error);
    }
}
예제 #8
0
static void
S_discover_unused(FilePurger *self, VArray **purgables_ptr, 
                  VArray **snapshots_ptr)
{
    Folder      *folder       = self->folder;
    DirHandle   *dh           = Folder_Open_Dir(folder, NULL);
    if (!dh) { RETHROW(INCREF(Err_get_error())); }
    VArray      *spared       = VA_new(1);
    VArray      *snapshots    = VA_new(1);
    CharBuf     *snapfile     = NULL;

    // Start off with the list of files in the current snapshot.
    if (self->snapshot) {
        VArray *entries    = Snapshot_List(self->snapshot);
        VArray *referenced = S_find_all_referenced(folder, entries);
        VA_Push_VArray(spared, referenced);
        DECREF(entries);
        DECREF(referenced);
        snapfile = Snapshot_Get_Path(self->snapshot);
        if (snapfile) { VA_Push(spared, INCREF(snapfile)); }
    }

    CharBuf *entry      = DH_Get_Entry(dh);
    Hash    *candidates = Hash_new(64);
    while (DH_Next(dh)) {
        if      (!CB_Starts_With_Str(entry, "snapshot_", 9))   { continue; }
        else if (!CB_Ends_With_Str(entry, ".json", 5))         { continue; }
        else if (snapfile && CB_Equals(entry, (Obj*)snapfile)) { continue; }
        else {
            Snapshot *snapshot 
                = Snapshot_Read_File(Snapshot_new(), folder, entry);
            Lock *lock
                = IxManager_Make_Snapshot_Read_Lock(self->manager, entry);
            VArray *snap_list  = Snapshot_List(snapshot);
            VArray *referenced = S_find_all_referenced(folder, snap_list);

            // DON'T obtain the lock -- only see whether another
            // entity holds a lock on the snapshot file.
            if (lock) {
                Lock_Clear_Stale(lock);
            }
            if (lock && Lock_Is_Locked(lock)) {
                // The snapshot file is locked, which means someone's using
                // that version of the index -- protect all of its entries.
                uint32_t new_size = VA_Get_Size(spared) 
                                  + VA_Get_Size(referenced)  + 1;
                VA_Grow(spared, new_size);
                VA_Push(spared, (Obj*)CB_Clone(entry));
                VA_Push_VArray(spared, referenced);
            }
            else {
                // No one's using this snapshot, so all of its entries are
                // candidates for deletion.
                for (uint32_t i = 0, max = VA_Get_Size(referenced); i < max; i++) {
                    CharBuf *file = (CharBuf*)VA_Fetch(referenced, i);
                    Hash_Store(candidates, (Obj*)file, INCREF(&EMPTY));
                }
                VA_Push(snapshots, INCREF(snapshot));
            }

            DECREF(referenced);
            DECREF(snap_list);
            DECREF(snapshot);
            DECREF(lock);
        }
    }
    DECREF(dh);

    // Clean up after a dead segment consolidation.
    S_zap_dead_merge(self, candidates);

    // Eliminate any current files from the list of files to be purged.
    for (uint32_t i = 0, max = VA_Get_Size(spared); i < max; i++) {
        CharBuf *filename = (CharBuf*)VA_Fetch(spared, i);
        DECREF(Hash_Delete(candidates, (Obj*)filename));
    }

    // Pass back purgables and Snapshots.
    *purgables_ptr = Hash_Keys(candidates);
    *snapshots_ptr = snapshots;

    DECREF(candidates);
    DECREF(spared);
}