Exemplo n.º 1
0
String*
IxFileNames_latest_snapshot(Folder *folder) {
    DirHandle *dh = Folder_Open_Dir(folder, NULL);
    String    *retval   = NULL;
    uint64_t   latest_gen = 0;

    if (!dh) { RETHROW(INCREF(Err_get_error())); }

    while (DH_Next(dh)) {
        String *entry = DH_Get_Entry(dh);
        if (Str_Starts_With_Utf8(entry, "snapshot_", 9)
            && Str_Ends_With_Utf8(entry, ".json", 5)
           ) {
            uint64_t gen = IxFileNames_extract_gen(entry);
            if (gen > latest_gen) {
                latest_gen = gen;
                DECREF(retval);
                retval = Str_Clone(entry);
            }
        }
        DECREF(entry);
    }

    DECREF(dh);
    return retval;
}
Exemplo n.º 2
0
static void
test_Local_Open_Dir(TestBatchRunner *runner) {

    Folder *real_folder = S_folder_with_contents();
    CompoundFileReader *cf_reader = CFReader_open(real_folder);
    bool saw_foo       = false;
    bool saw_stuff     = false;
    bool stuff_was_dir = false;

    CFReader_MkDir(cf_reader, stuff);

    DirHandle *dh = CFReader_Local_Open_Dir(cf_reader);
    while (DH_Next(dh)) {
        String *entry = DH_Get_Entry(dh);
        if (Str_Equals(entry, (Obj*)foo)) {
            saw_foo = true;
        }
        else if (Str_Equals(entry, (Obj*)stuff)) {
            saw_stuff = true;
            stuff_was_dir = DH_Entry_Is_Dir(dh);
        }
        DECREF(entry);
    }

    TEST_TRUE(runner, saw_foo, "DirHandle iterated over virtual file");
    TEST_TRUE(runner, saw_stuff, "DirHandle iterated over real directory");
    TEST_TRUE(runner, stuff_was_dir,
              "DirHandle knew that real entry was dir");

    DECREF(dh);
    DECREF(cf_reader);
    DECREF(real_folder);
}
Exemplo n.º 3
0
String*
IxManager_Make_Snapshot_Filename_IMP(IndexManager *self) {
    IndexManagerIVARS *const ivars = IxManager_IVARS(self);
    Folder *folder = (Folder*)CERTIFY(ivars->folder, FOLDER);
    DirHandle *dh = Folder_Open_Dir(folder, NULL);
    uint64_t max_gen = 0;

    if (!dh) { RETHROW(INCREF(Err_get_error())); }
    while (DH_Next(dh)) {
        String *entry = DH_Get_Entry(dh);
        if (Str_Starts_With_Utf8(entry, "snapshot_", 9)
            && Str_Ends_With_Utf8(entry, ".json", 5)
           ) {
            uint64_t gen = IxFileNames_extract_gen(entry);
            if (gen > max_gen) { max_gen = gen; }
        }
        DECREF(entry);
    }
    DECREF(dh);

    uint64_t new_gen = max_gen + 1;
    char  base36[StrHelp_MAX_BASE36_BYTES];
    StrHelp_to_base36(new_gen, &base36);
    return Str_newf("snapshot_%s.json", &base36);
}
Exemplo n.º 4
0
CharBuf*
IxManager_make_snapshot_filename(IndexManager *self)
{
    Folder *folder = (Folder*)CERTIFY(self->folder, FOLDER);
    DirHandle *dh = Folder_Open_Dir(folder, NULL);
    CharBuf *entry;
    uint64_t max_gen = 0;

    if (!dh) { RETHROW(INCREF(Err_get_error())); }
    entry = DH_Get_Entry(dh);
    while (DH_Next(dh)) {
        if (    CB_Starts_With_Str(entry, "snapshot_", 9)
            && CB_Ends_With_Str(entry, ".json", 5)
        ) {
            uint64_t gen = IxFileNames_extract_gen(entry);
            if (gen > max_gen) { max_gen = gen; }
        }
    }
    DECREF(dh);

    {
        uint64_t new_gen = max_gen + 1;
        char  base36[StrHelp_MAX_BASE36_BYTES];
        StrHelp_to_base36(new_gen, &base36);
        return CB_newf("snapshot_%s.json", &base36);
    }
}
Exemplo n.º 5
0
bool_t
Folder_delete_tree(Folder *self, const CharBuf *path) {
    Folder *enclosing_folder = Folder_Enclosing_Folder(self, path);

    // Don't allow Folder to delete itself.
    if (!path || !CB_Get_Size(path)) {
        return false;
    }

    if (enclosing_folder) {
        ZombieCharBuf *local = IxFileNames_local_part(path, ZCB_BLANK());
        if (Folder_Local_Is_Directory(enclosing_folder, (CharBuf*)local)) {
            Folder *inner_folder
                = Folder_Local_Find_Folder(enclosing_folder, (CharBuf*)local);
            DirHandle *dh = Folder_Local_Open_Dir(inner_folder);
            if (dh) {
                VArray *files = VA_new(20);
                VArray *dirs  = VA_new(20);
                CharBuf *entry = DH_Get_Entry(dh);
                while (DH_Next(dh)) {
                    VA_Push(files, (Obj*)CB_Clone(entry));
                    if (DH_Entry_Is_Dir(dh) && !DH_Entry_Is_Symlink(dh)) {
                        VA_Push(dirs, (Obj*)CB_Clone(entry));
                    }
                }
                for (uint32_t i = 0, max = VA_Get_Size(dirs); i < max; i++) {
                    CharBuf *name = (CharBuf*)VA_Fetch(files, i);
                    bool_t success = Folder_Delete_Tree(inner_folder, name);
                    if (!success && Folder_Local_Exists(inner_folder, name)) {
                        break;
                    }
                }
                for (uint32_t i = 0, max = VA_Get_Size(files); i < max; i++) {
                    CharBuf *name = (CharBuf*)VA_Fetch(files, i);
                    bool_t success = Folder_Local_Delete(inner_folder, name);
                    if (!success && Folder_Local_Exists(inner_folder, name)) {
                        break;
                    }
                }
                DECREF(dirs);
                DECREF(files);
                DECREF(dh);
            }
        }
        return Folder_Local_Delete(enclosing_folder, (CharBuf*)local);
    }
    else {
        // Return failure if the entry wasn't there in the first place.
        return false;
    }
}
Exemplo n.º 6
0
CFReaderDirHandle*
CFReaderDH_init(CFReaderDirHandle *self, CompoundFileReader *cf_reader) {
    DH_init((DirHandle*)self, CFReader_Get_Path(cf_reader));
    self->cf_reader = (CompoundFileReader*)INCREF(cf_reader);
    self->elems  = Hash_Keys(self->cf_reader->records);
    self->tick   = -1;
    // Accumulate entries from real Folder.
    DirHandle *dh = Folder_Local_Open_Dir(self->cf_reader->real_folder);
    CharBuf *entry = DH_Get_Entry(dh);
    while (DH_Next(dh)) {
        VA_Push(self->elems, (Obj*)CB_Clone(entry));
    }
    DECREF(dh);
    return self;
}
Exemplo n.º 7
0
Arquivo: Folder.c Projeto: theory/lucy
VArray*
Folder_list(Folder *self, const CharBuf *path) {
    Folder *local_folder = Folder_Find_Folder(self, path);
    VArray *list = NULL;
    DirHandle *dh = Folder_Local_Open_Dir(local_folder);
    if (dh) {
        CharBuf *entry = DH_Get_Entry(dh);
        list = VA_new(32);
        while (DH_Next(dh)) { VA_Push(list, (Obj*)CB_Clone(entry)); }
        DECREF(dh);
    }
    else {
        ERR_ADD_FRAME(Err_get_error());
    }
    return list;
}
Exemplo n.º 8
0
static void
S_zap_dead_merge(FilePurger *self, Hash *candidates)
{
    IndexManager *manager = self->manager;
    Lock *merge_lock   = IxManager_Make_Merge_Lock(manager);

    Lock_Clear_Stale(merge_lock);
    if (!Lock_Is_Locked(merge_lock)) { 
        Hash *merge_data = IxManager_Read_Merge_Data(manager);
        Obj  *cutoff = merge_data 
                     ? Hash_Fetch_Str(merge_data, "cutoff", 6) 
                     : NULL;

        if (cutoff) {
            CharBuf *cutoff_seg = Seg_num_to_name(Obj_To_I64(cutoff));
            if (Folder_Exists(self->folder, cutoff_seg)) {
                ZombieCharBuf *merge_json = ZCB_WRAP_STR("merge.json", 10);
                DirHandle *dh = Folder_Open_Dir(self->folder, cutoff_seg);
                CharBuf *entry = dh ? DH_Get_Entry(dh) : NULL;
                CharBuf *filepath = CB_new(32);

                if (!dh) {
                    THROW(ERR, "Can't open segment dir '%o'", filepath);
                }

                Hash_Store(candidates, (Obj*)cutoff_seg, INCREF(&EMPTY));
                Hash_Store(candidates, (Obj*)merge_json, INCREF(&EMPTY));
                while (DH_Next(dh)) {
                    // TODO: recursively delete subdirs within seg dir.
                    CB_setf(filepath, "%o/%o", cutoff_seg, entry);
                    Hash_Store(candidates, (Obj*)filepath, INCREF(&EMPTY));
                }
                DECREF(filepath);
                DECREF(dh);
            }
            DECREF(cutoff_seg);
        }

        DECREF(merge_data);
    }

    DECREF(merge_lock);
    return;
}
Exemplo n.º 9
0
static void
S_zap_dead_merge(FilePurger *self, Hash *candidates) {
    FilePurgerIVARS *const ivars = FilePurger_IVARS(self);
    IndexManager *manager    = ivars->manager;
    Lock         *merge_lock = IxManager_Make_Merge_Lock(manager);

    Lock_Clear_Stale(merge_lock);
    if (!Lock_Is_Locked(merge_lock)) {
        Hash *merge_data = IxManager_Read_Merge_Data(manager);
        Obj  *cutoff = merge_data
                       ? Hash_Fetch_Utf8(merge_data, "cutoff", 6)
                       : NULL;

        if (cutoff) {
            String *cutoff_seg = Seg_num_to_name(Json_obj_to_i64(cutoff));
            if (Folder_Exists(ivars->folder, cutoff_seg)) {
                String *merge_json = SSTR_WRAP_UTF8("merge.json", 10);
                DirHandle *dh = Folder_Open_Dir(ivars->folder, cutoff_seg);

                if (!dh) {
                    THROW(ERR, "Can't open segment dir '%o'", cutoff_seg);
                }

                Hash_Store(candidates, cutoff_seg, (Obj*)CFISH_TRUE);
                Hash_Store(candidates, merge_json, (Obj*)CFISH_TRUE);
                while (DH_Next(dh)) {
                    // TODO: recursively delete subdirs within seg dir.
                    String *entry = DH_Get_Entry(dh);
                    String *filepath = Str_newf("%o/%o", cutoff_seg, entry);
                    Hash_Store(candidates, filepath, (Obj*)CFISH_TRUE);
                    DECREF(filepath);
                    DECREF(entry);
                }
                DECREF(dh);
            }
            DECREF(cutoff_seg);
        }

        DECREF(merge_data);
    }

    DECREF(merge_lock);
    return;
}
Exemplo n.º 10
0
CFReaderDirHandle*
CFReaderDH_init(CFReaderDirHandle *self, CompoundFileReader *cf_reader) {
    DH_init((DirHandle*)self, CFReader_Get_Path(cf_reader));
    CFReaderDirHandleIVARS *const ivars = CFReaderDH_IVARS(self);
    ivars->cf_reader = (CompoundFileReader*)INCREF(cf_reader);

    Hash *cf_records = CFReader_IVARS(ivars->cf_reader)->records;
    ivars->elems  = Hash_Keys(cf_records);
    ivars->tick   = -1;
    // Accumulate entries from real Folder.
    Folder *real_folder = CFReader_Get_Real_Folder(ivars->cf_reader);
    DirHandle *dh = Folder_Local_Open_Dir(real_folder);
    while (DH_Next(dh)) {
        String *entry = DH_Get_Entry(dh);
        Vec_Push(ivars->elems, (Obj*)Str_Clone(entry));
        DECREF(entry);
    }
    DECREF(dh);
    return self;
}
Exemplo n.º 11
0
Arquivo: Folder.c Projeto: theory/lucy
static void
S_add_to_file_list(Folder *self, VArray *list, CharBuf *dir, CharBuf *prefix) {
    size_t     orig_prefix_size = CB_Get_Size(prefix);
    DirHandle *dh = Folder_Open_Dir(self, dir);
    CharBuf   *entry;

    if (!dh) {
        RETHROW(INCREF(Err_get_error()));
    }

    entry = DH_Get_Entry(dh);
    while (DH_Next(dh)) { // Updates entry
        if (!S_is_updir(entry)) {
            CharBuf *relpath = CB_newf("%o%o", prefix, entry);
            if (VA_Get_Size(list) == VA_Get_Capacity(list)) {
                VA_Grow(list, VA_Get_Size(list) * 2);
            }
            VA_Push(list, (Obj*)relpath);

            if (DH_Entry_Is_Dir(dh) && !DH_Entry_Is_Symlink(dh)) {
                CharBuf *subdir = CB_Get_Size(dir)
                                  ? CB_newf("%o/%o", dir, entry)
                                  : CB_Clone(entry);
                CB_catf(prefix, "%o/", entry);
                S_add_to_file_list(self, list, subdir, prefix); // recurse
                CB_Set_Size(prefix, orig_prefix_size);
                DECREF(subdir);
            }
        }
    }

    if (!DH_Close(dh)) {
        RETHROW(INCREF(Err_get_error()));
    }
    DECREF(dh);
}
Exemplo n.º 12
0
static void
S_discover_unused(FilePurger *self, VArray **purgables_ptr, 
                  VArray **snapshots_ptr)
{
    Folder      *folder       = self->folder;
    DirHandle   *dh           = Folder_Open_Dir(folder, NULL);
    if (!dh) { RETHROW(INCREF(Err_get_error())); }
    VArray      *spared       = VA_new(1);
    VArray      *snapshots    = VA_new(1);
    CharBuf     *snapfile     = NULL;

    // Start off with the list of files in the current snapshot.
    if (self->snapshot) {
        VArray *entries    = Snapshot_List(self->snapshot);
        VArray *referenced = S_find_all_referenced(folder, entries);
        VA_Push_VArray(spared, referenced);
        DECREF(entries);
        DECREF(referenced);
        snapfile = Snapshot_Get_Path(self->snapshot);
        if (snapfile) { VA_Push(spared, INCREF(snapfile)); }
    }

    CharBuf *entry      = DH_Get_Entry(dh);
    Hash    *candidates = Hash_new(64);
    while (DH_Next(dh)) {
        if      (!CB_Starts_With_Str(entry, "snapshot_", 9))   { continue; }
        else if (!CB_Ends_With_Str(entry, ".json", 5))         { continue; }
        else if (snapfile && CB_Equals(entry, (Obj*)snapfile)) { continue; }
        else {
            Snapshot *snapshot 
                = Snapshot_Read_File(Snapshot_new(), folder, entry);
            Lock *lock
                = IxManager_Make_Snapshot_Read_Lock(self->manager, entry);
            VArray *snap_list  = Snapshot_List(snapshot);
            VArray *referenced = S_find_all_referenced(folder, snap_list);

            // DON'T obtain the lock -- only see whether another
            // entity holds a lock on the snapshot file.
            if (lock) {
                Lock_Clear_Stale(lock);
            }
            if (lock && Lock_Is_Locked(lock)) {
                // The snapshot file is locked, which means someone's using
                // that version of the index -- protect all of its entries.
                uint32_t new_size = VA_Get_Size(spared) 
                                  + VA_Get_Size(referenced)  + 1;
                VA_Grow(spared, new_size);
                VA_Push(spared, (Obj*)CB_Clone(entry));
                VA_Push_VArray(spared, referenced);
            }
            else {
                // No one's using this snapshot, so all of its entries are
                // candidates for deletion.
                for (uint32_t i = 0, max = VA_Get_Size(referenced); i < max; i++) {
                    CharBuf *file = (CharBuf*)VA_Fetch(referenced, i);
                    Hash_Store(candidates, (Obj*)file, INCREF(&EMPTY));
                }
                VA_Push(snapshots, INCREF(snapshot));
            }

            DECREF(referenced);
            DECREF(snap_list);
            DECREF(snapshot);
            DECREF(lock);
        }
    }
    DECREF(dh);

    // Clean up after a dead segment consolidation.
    S_zap_dead_merge(self, candidates);

    // Eliminate any current files from the list of files to be purged.
    for (uint32_t i = 0, max = VA_Get_Size(spared); i < max; i++) {
        CharBuf *filename = (CharBuf*)VA_Fetch(spared, i);
        DECREF(Hash_Delete(candidates, (Obj*)filename));
    }

    // Pass back purgables and Snapshots.
    *purgables_ptr = Hash_Keys(candidates);
    *snapshots_ptr = snapshots;

    DECREF(candidates);
    DECREF(spared);
}