Example #1
0
static void
test_Open_Dir(TestBatch *batch)
{
    Folder     *folder = (Folder*)RAMFolder_new(NULL);
    DirHandle  *dh;

    Folder_MkDir(folder, &foo);
    Folder_MkDir(folder, &foo_bar);

    dh = Folder_Open_Dir(folder, &foo);
    TEST_TRUE(batch, dh && DH_Is_A(dh, DIRHANDLE), "Open_Dir");
    DECREF(dh);
    dh = Folder_Open_Dir(folder, &foo_bar);
    TEST_TRUE(batch, dh && DH_Is_A(dh, DIRHANDLE), "Open_Dir nested dir");
    DECREF(dh);

    Err_set_error(NULL);
    dh = Folder_Open_Dir(folder, &bar);
    TEST_TRUE(batch, dh == NULL,
        "Open_Dir on non-existent entry fails");
    TEST_TRUE(batch, Err_get_error() != NULL,
        "Open_Dir on non-existent entry sets Err_error");

    Err_set_error(NULL);
    dh = Folder_Open_Dir(folder, &foo_foo);
    TEST_TRUE(batch, dh == NULL,
        "Open_Dir on non-existent nested entry fails");
    TEST_TRUE(batch, Err_get_error() != NULL,
        "Open_Dir on non-existent nested entry sets Err_error");

    DECREF(folder);
}
Example #2
0
String*
IxFileNames_latest_snapshot(Folder *folder) {
    DirHandle *dh = Folder_Open_Dir(folder, NULL);
    String    *retval   = NULL;
    uint64_t   latest_gen = 0;

    if (!dh) { RETHROW(INCREF(Err_get_error())); }

    while (DH_Next(dh)) {
        String *entry = DH_Get_Entry(dh);
        if (Str_Starts_With_Utf8(entry, "snapshot_", 9)
            && Str_Ends_With_Utf8(entry, ".json", 5)
           ) {
            uint64_t gen = IxFileNames_extract_gen(entry);
            if (gen > latest_gen) {
                latest_gen = gen;
                DECREF(retval);
                retval = Str_Clone(entry);
            }
        }
        DECREF(entry);
    }

    DECREF(dh);
    return retval;
}
Example #3
0
String*
IxManager_Make_Snapshot_Filename_IMP(IndexManager *self) {
    IndexManagerIVARS *const ivars = IxManager_IVARS(self);
    Folder *folder = (Folder*)CERTIFY(ivars->folder, FOLDER);
    DirHandle *dh = Folder_Open_Dir(folder, NULL);
    uint64_t max_gen = 0;

    if (!dh) { RETHROW(INCREF(Err_get_error())); }
    while (DH_Next(dh)) {
        String *entry = DH_Get_Entry(dh);
        if (Str_Starts_With_Utf8(entry, "snapshot_", 9)
            && Str_Ends_With_Utf8(entry, ".json", 5)
           ) {
            uint64_t gen = IxFileNames_extract_gen(entry);
            if (gen > max_gen) { max_gen = gen; }
        }
        DECREF(entry);
    }
    DECREF(dh);

    uint64_t new_gen = max_gen + 1;
    char  base36[StrHelp_MAX_BASE36_BYTES];
    StrHelp_to_base36(new_gen, &base36);
    return Str_newf("snapshot_%s.json", &base36);
}
Example #4
0
CharBuf*
IxManager_make_snapshot_filename(IndexManager *self)
{
    Folder *folder = (Folder*)CERTIFY(self->folder, FOLDER);
    DirHandle *dh = Folder_Open_Dir(folder, NULL);
    CharBuf *entry;
    uint64_t max_gen = 0;

    if (!dh) { RETHROW(INCREF(Err_get_error())); }
    entry = DH_Get_Entry(dh);
    while (DH_Next(dh)) {
        if (    CB_Starts_With_Str(entry, "snapshot_", 9)
            && CB_Ends_With_Str(entry, ".json", 5)
        ) {
            uint64_t gen = IxFileNames_extract_gen(entry);
            if (gen > max_gen) { max_gen = gen; }
        }
    }
    DECREF(dh);

    {
        uint64_t new_gen = max_gen + 1;
        char  base36[StrHelp_MAX_BASE36_BYTES];
        StrHelp_to_base36(new_gen, &base36);
        return CB_newf("snapshot_%s.json", &base36);
    }
}
Example #5
0
static void
S_zap_dead_merge(FilePurger *self, Hash *candidates)
{
    IndexManager *manager = self->manager;
    Lock *merge_lock   = IxManager_Make_Merge_Lock(manager);

    Lock_Clear_Stale(merge_lock);
    if (!Lock_Is_Locked(merge_lock)) { 
        Hash *merge_data = IxManager_Read_Merge_Data(manager);
        Obj  *cutoff = merge_data 
                     ? Hash_Fetch_Str(merge_data, "cutoff", 6) 
                     : NULL;

        if (cutoff) {
            CharBuf *cutoff_seg = Seg_num_to_name(Obj_To_I64(cutoff));
            if (Folder_Exists(self->folder, cutoff_seg)) {
                ZombieCharBuf *merge_json = ZCB_WRAP_STR("merge.json", 10);
                DirHandle *dh = Folder_Open_Dir(self->folder, cutoff_seg);
                CharBuf *entry = dh ? DH_Get_Entry(dh) : NULL;
                CharBuf *filepath = CB_new(32);

                if (!dh) {
                    THROW(ERR, "Can't open segment dir '%o'", filepath);
                }

                Hash_Store(candidates, (Obj*)cutoff_seg, INCREF(&EMPTY));
                Hash_Store(candidates, (Obj*)merge_json, INCREF(&EMPTY));
                while (DH_Next(dh)) {
                    // TODO: recursively delete subdirs within seg dir.
                    CB_setf(filepath, "%o/%o", cutoff_seg, entry);
                    Hash_Store(candidates, (Obj*)filepath, INCREF(&EMPTY));
                }
                DECREF(filepath);
                DECREF(dh);
            }
            DECREF(cutoff_seg);
        }

        DECREF(merge_data);
    }

    DECREF(merge_lock);
    return;
}
Example #6
0
static void
S_zap_dead_merge(FilePurger *self, Hash *candidates) {
    FilePurgerIVARS *const ivars = FilePurger_IVARS(self);
    IndexManager *manager    = ivars->manager;
    Lock         *merge_lock = IxManager_Make_Merge_Lock(manager);

    Lock_Clear_Stale(merge_lock);
    if (!Lock_Is_Locked(merge_lock)) {
        Hash *merge_data = IxManager_Read_Merge_Data(manager);
        Obj  *cutoff = merge_data
                       ? Hash_Fetch_Utf8(merge_data, "cutoff", 6)
                       : NULL;

        if (cutoff) {
            String *cutoff_seg = Seg_num_to_name(Json_obj_to_i64(cutoff));
            if (Folder_Exists(ivars->folder, cutoff_seg)) {
                String *merge_json = SSTR_WRAP_UTF8("merge.json", 10);
                DirHandle *dh = Folder_Open_Dir(ivars->folder, cutoff_seg);

                if (!dh) {
                    THROW(ERR, "Can't open segment dir '%o'", cutoff_seg);
                }

                Hash_Store(candidates, cutoff_seg, (Obj*)CFISH_TRUE);
                Hash_Store(candidates, merge_json, (Obj*)CFISH_TRUE);
                while (DH_Next(dh)) {
                    // TODO: recursively delete subdirs within seg dir.
                    String *entry = DH_Get_Entry(dh);
                    String *filepath = Str_newf("%o/%o", cutoff_seg, entry);
                    Hash_Store(candidates, filepath, (Obj*)CFISH_TRUE);
                    DECREF(filepath);
                    DECREF(entry);
                }
                DECREF(dh);
            }
            DECREF(cutoff_seg);
        }

        DECREF(merge_data);
    }

    DECREF(merge_lock);
    return;
}
Example #7
0
File: Folder.c Project: theory/lucy
static void
S_add_to_file_list(Folder *self, VArray *list, CharBuf *dir, CharBuf *prefix) {
    size_t     orig_prefix_size = CB_Get_Size(prefix);
    DirHandle *dh = Folder_Open_Dir(self, dir);
    CharBuf   *entry;

    if (!dh) {
        RETHROW(INCREF(Err_get_error()));
    }

    entry = DH_Get_Entry(dh);
    while (DH_Next(dh)) { // Updates entry
        if (!S_is_updir(entry)) {
            CharBuf *relpath = CB_newf("%o%o", prefix, entry);
            if (VA_Get_Size(list) == VA_Get_Capacity(list)) {
                VA_Grow(list, VA_Get_Size(list) * 2);
            }
            VA_Push(list, (Obj*)relpath);

            if (DH_Entry_Is_Dir(dh) && !DH_Entry_Is_Symlink(dh)) {
                CharBuf *subdir = CB_Get_Size(dir)
                                  ? CB_newf("%o/%o", dir, entry)
                                  : CB_Clone(entry);
                CB_catf(prefix, "%o/", entry);
                S_add_to_file_list(self, list, subdir, prefix); // recurse
                CB_Set_Size(prefix, orig_prefix_size);
                DECREF(subdir);
            }
        }
    }

    if (!DH_Close(dh)) {
        RETHROW(INCREF(Err_get_error()));
    }
    DECREF(dh);
}
Example #8
0
static void
S_discover_unused(FilePurger *self, VArray **purgables_ptr, 
                  VArray **snapshots_ptr)
{
    Folder      *folder       = self->folder;
    DirHandle   *dh           = Folder_Open_Dir(folder, NULL);
    if (!dh) { RETHROW(INCREF(Err_get_error())); }
    VArray      *spared       = VA_new(1);
    VArray      *snapshots    = VA_new(1);
    CharBuf     *snapfile     = NULL;

    // Start off with the list of files in the current snapshot.
    if (self->snapshot) {
        VArray *entries    = Snapshot_List(self->snapshot);
        VArray *referenced = S_find_all_referenced(folder, entries);
        VA_Push_VArray(spared, referenced);
        DECREF(entries);
        DECREF(referenced);
        snapfile = Snapshot_Get_Path(self->snapshot);
        if (snapfile) { VA_Push(spared, INCREF(snapfile)); }
    }

    CharBuf *entry      = DH_Get_Entry(dh);
    Hash    *candidates = Hash_new(64);
    while (DH_Next(dh)) {
        if      (!CB_Starts_With_Str(entry, "snapshot_", 9))   { continue; }
        else if (!CB_Ends_With_Str(entry, ".json", 5))         { continue; }
        else if (snapfile && CB_Equals(entry, (Obj*)snapfile)) { continue; }
        else {
            Snapshot *snapshot 
                = Snapshot_Read_File(Snapshot_new(), folder, entry);
            Lock *lock
                = IxManager_Make_Snapshot_Read_Lock(self->manager, entry);
            VArray *snap_list  = Snapshot_List(snapshot);
            VArray *referenced = S_find_all_referenced(folder, snap_list);

            // DON'T obtain the lock -- only see whether another
            // entity holds a lock on the snapshot file.
            if (lock) {
                Lock_Clear_Stale(lock);
            }
            if (lock && Lock_Is_Locked(lock)) {
                // The snapshot file is locked, which means someone's using
                // that version of the index -- protect all of its entries.
                uint32_t new_size = VA_Get_Size(spared) 
                                  + VA_Get_Size(referenced)  + 1;
                VA_Grow(spared, new_size);
                VA_Push(spared, (Obj*)CB_Clone(entry));
                VA_Push_VArray(spared, referenced);
            }
            else {
                // No one's using this snapshot, so all of its entries are
                // candidates for deletion.
                for (uint32_t i = 0, max = VA_Get_Size(referenced); i < max; i++) {
                    CharBuf *file = (CharBuf*)VA_Fetch(referenced, i);
                    Hash_Store(candidates, (Obj*)file, INCREF(&EMPTY));
                }
                VA_Push(snapshots, INCREF(snapshot));
            }

            DECREF(referenced);
            DECREF(snap_list);
            DECREF(snapshot);
            DECREF(lock);
        }
    }
    DECREF(dh);

    // Clean up after a dead segment consolidation.
    S_zap_dead_merge(self, candidates);

    // Eliminate any current files from the list of files to be purged.
    for (uint32_t i = 0, max = VA_Get_Size(spared); i < max; i++) {
        CharBuf *filename = (CharBuf*)VA_Fetch(spared, i);
        DECREF(Hash_Delete(candidates, (Obj*)filename));
    }

    // Pass back purgables and Snapshots.
    *purgables_ptr = Hash_Keys(candidates);
    *snapshots_ptr = snapshots;

    DECREF(candidates);
    DECREF(spared);
}