예제 #1
0
void
LexWriter_start_field(LexiconWriter *self, i32_t field_num)
{
    CharBuf  *const seg_name = Seg_Get_Name(self->segment);
    Folder   *const folder   = self->folder;
    Snapshot *const snapshot = LexWriter_Get_Snapshot(self);

    /* Open outstreams. */
    CB_setf(self->dat_file,  "%o/lexicon-%i32.dat",  seg_name, field_num);
    CB_setf(self->ix_file,   "%o/lexicon-%i32.ix",   seg_name, field_num);
    CB_setf(self->ixix_file, "%o/lexicon-%i32.ixix", seg_name, field_num);
    Snapshot_Add_Entry(snapshot, self->dat_file);
    Snapshot_Add_Entry(snapshot, self->ix_file);
    Snapshot_Add_Entry(snapshot, self->ixix_file);
    self->dat_out  = Folder_Open_Out(folder, self->dat_file);
    self->ix_out   = Folder_Open_Out(folder, self->ix_file);
    self->ixix_out = Folder_Open_Out(folder, self->ixix_file);
    if (!self->dat_out)  { THROW("Can't open %o", self->dat_file); }
    if (!self->ix_out)   { THROW("Can't open %o", self->ix_file); }
    if (!self->ixix_out) { THROW("Can't open %o", self->ixix_file); }

    /* Initialize count and ix_count, TermInfo and last term text. */
    self->count    = 0;
    self->ix_count = 0;
    TInfo_Reset(self->last_tinfo);
    CB_Set_Size(self->last_text, 0);
}
예제 #2
0
static OutStream*
S_lazy_init(HighlightWriter *self)
{
    if (!self->dat_out) {
        Segment  *segment  = self->segment;
        Folder   *folder   = self->folder;
        Snapshot *snapshot = HLWriter_Get_Snapshot(self);
        CharBuf  *seg_name = Seg_Get_Name(segment);
        CharBuf  *ix_file  = CB_newf("%o/highlight.ix", seg_name);
        CharBuf  *dat_file = CB_newf("%o/highlight.dat", seg_name);

        /* Open outstreams. */
        Snapshot_Add_Entry(snapshot, ix_file);
        Snapshot_Add_Entry(snapshot, dat_file);
        self->ix_out  = Folder_Open_Out(folder, ix_file);
        self->dat_out = Folder_Open_Out(folder, dat_file);
        if (!self->ix_out)  { THROW("Can't open %o", ix_file); }
        if (!self->dat_out) { THROW("Can't open %o", dat_file); }
        DECREF(ix_file);
        DECREF(dat_file);

        /* Go past invalid doc 0. */
        OutStream_Write_U64(self->ix_out, 0);
    }

    return self->dat_out;
}
예제 #3
0
static void
test_Read_File_and_Write_File(TestBatchRunner *runner) {
    Snapshot *snapshot = Snapshot_new();
    Folder   *folder   = (Folder*)RAMFolder_new(NULL);
    String   *snap     = (String*)SSTR_WRAP_UTF8("snap", 4);
    String   *foo      = (String*)SSTR_WRAP_UTF8("foo", 3);

    Snapshot_Add_Entry(snapshot, foo);
    Snapshot_Write_File(snapshot, folder, snap);

    Snapshot *dupe = Snapshot_new();
    Snapshot *read_retval = Snapshot_Read_File(dupe, folder, snap);
    TEST_TRUE(runner, dupe == read_retval, "Read_File() returns the object");

    Vector *orig_list = Snapshot_List(snapshot);
    Vector *dupe_list = Snapshot_List(dupe);
    TEST_TRUE(runner, Vec_Equals(orig_list, (Obj*)dupe_list),
              "Round trip through Write_File() and Read_File()");

    DECREF(orig_list);
    DECREF(dupe_list);
    DECREF(dupe);
    DECREF(snapshot);
    DECREF(folder);
}
예제 #4
0
파일: Indexer.c 프로젝트: gitpan/KinoSearch
void
Indexer_prepare_commit(Indexer *self)
{
    VArray   *seg_readers     = PolyReader_Get_Seg_Readers(self->polyreader);
    uint32_t  num_seg_readers = VA_Get_Size(seg_readers);
    bool_t    merge_happened  = false;

    if ( !self->write_lock || self->prepared ) {
        THROW(ERR, "Can't call Prepare_Commit() more than once");
    }

    // Merge existing index data. 
    if (num_seg_readers) { 
        merge_happened = S_maybe_merge(self, seg_readers);
    }

    // Add a new segment and write a new snapshot file if... 
    if (   Seg_Get_Count(self->segment)      // Docs/segs added. 
        || merge_happened                        // Some segs merged. 
        || !Snapshot_Num_Entries(self->snapshot) // Initializing index. 
        || DelWriter_Updated(self->del_writer) 
    ) {
        Folder   *folder   = self->folder;
        Schema   *schema   = self->schema;
        Snapshot *snapshot = self->snapshot;
        CharBuf  *old_schema_name = S_find_schema_file(snapshot);
        uint64_t  schema_gen = old_schema_name
                             ? IxFileNames_extract_gen(old_schema_name) + 1
                             : 1;
        char      base36[StrHelp_MAX_BASE36_BYTES];
        CharBuf  *new_schema_name;
        
        StrHelp_to_base36(schema_gen, &base36);
        new_schema_name = CB_newf("schema_%s.json", base36);

        // Finish the segment, write schema file. 
        SegWriter_Finish(self->seg_writer);
        Schema_Write(schema, folder, new_schema_name);
        if (old_schema_name) {
            Snapshot_Delete_Entry(snapshot, old_schema_name);
        }
        Snapshot_Add_Entry(snapshot, new_schema_name);
        DECREF(new_schema_name);

        // Write temporary snapshot file. 
        DECREF(self->snapfile);
        self->snapfile = IxManager_Make_Snapshot_Filename(self->manager);
        CB_Cat_Trusted_Str(self->snapfile, ".temp", 5);
        Folder_Delete(folder, self->snapfile);
        Snapshot_Write_File(snapshot, folder, self->snapfile);

        self->needs_commit = true;
    }

    // Close reader, so that we can delete its files if appropriate. 
    PolyReader_Close(self->polyreader);

    self->prepared = true;
}
예제 #5
0
파일: Indexer.c 프로젝트: rectang/lucy
void
Indexer_Prepare_Commit_IMP(Indexer *self) {
    IndexerIVARS *const ivars = Indexer_IVARS(self);
    Vector   *seg_readers     = PolyReader_Get_Seg_Readers(ivars->polyreader);
    size_t    num_seg_readers = Vec_Get_Size(seg_readers);
    bool      merge_happened  = false;

    if (!ivars->write_lock || ivars->prepared) {
        THROW(ERR, "Can't call Prepare_Commit() more than once");
    }

    // Merge existing index data.
    if (num_seg_readers) {
        merge_happened = S_maybe_merge(self, seg_readers);
    }

    // Add a new segment and write a new snapshot file if...
    if (Seg_Get_Count(ivars->segment)             // Docs/segs added.
        || merge_happened                        // Some segs merged.
        || !Snapshot_Num_Entries(ivars->snapshot) // Initializing index.
        || DelWriter_Updated(ivars->del_writer)
       ) {
        Folder   *folder   = ivars->folder;
        Schema   *schema   = ivars->schema;
        Snapshot *snapshot = ivars->snapshot;

        // Derive snapshot and schema file names.
        DECREF(ivars->snapfile);
        String *snapfile = IxManager_Make_Snapshot_Filename(ivars->manager);
        ivars->snapfile = Str_Cat_Trusted_Utf8(snapfile, ".temp", 5);
        DECREF(snapfile);
        uint64_t schema_gen = IxFileNames_extract_gen(ivars->snapfile);
        char base36[StrHelp_MAX_BASE36_BYTES];
        StrHelp_to_base36(schema_gen, &base36);
        String *new_schema_name = Str_newf("schema_%s.json", base36);

        // Finish the segment, write schema file.
        SegWriter_Finish(ivars->seg_writer);
        Schema_Write(schema, folder, new_schema_name);
        String *old_schema_name = S_find_schema_file(snapshot);
        if (old_schema_name) {
            Snapshot_Delete_Entry(snapshot, old_schema_name);
        }
        Snapshot_Add_Entry(snapshot, new_schema_name);
        DECREF(new_schema_name);

        // Write temporary snapshot file.
        Folder_Delete(folder, ivars->snapfile);
        Snapshot_Write_File(snapshot, folder, ivars->snapfile);

        ivars->needs_commit = true;
    }

    // Close reader, so that we can delete its files if appropriate.
    PolyReader_Close(ivars->polyreader);

    ivars->prepared = true;
}
예제 #6
0
static void
test_Add_and_Delete(TestBatchRunner *runner) {
    Snapshot *snapshot = Snapshot_new();
    String *foo = (String*)SSTR_WRAP_UTF8("foo", 3);
    String *bar = (String*)SSTR_WRAP_UTF8("bar", 3);

    Snapshot_Add_Entry(snapshot, foo);
    Snapshot_Add_Entry(snapshot, foo); // redundant
    Vector *entries = Snapshot_List(snapshot);
    TEST_INT_EQ(runner, Snapshot_Num_Entries(snapshot), 1,
                "One entry added");
    TEST_TRUE(runner, Str_Equals(foo, Vec_Fetch(entries, 0)), "correct entry");
    DECREF(entries);

    Snapshot_Add_Entry(snapshot, bar);
    TEST_INT_EQ(runner, Snapshot_Num_Entries(snapshot), 2,
                "second entry added");
    Snapshot_Delete_Entry(snapshot, foo);
    TEST_INT_EQ(runner, Snapshot_Num_Entries(snapshot), 1, "Delete_Entry");

    DECREF(snapshot);
}
예제 #7
0
void
SegWriter_Finish_IMP(SegWriter *self) {
    SegWriterIVARS *const ivars = SegWriter_IVARS(self);
    String *seg_name = Seg_Get_Name(ivars->segment);

    // Finish off children.
    for (size_t i = 0, max = Vec_Get_Size(ivars->writers); i < max; i++) {
        DataWriter *writer = (DataWriter*)Vec_Fetch(ivars->writers, i);
        DataWriter_Finish(writer);
    }

    // Write segment metadata and add the segment directory to the snapshot.
    Snapshot *snapshot = SegWriter_Get_Snapshot(self);
    String *segmeta_filename = Str_newf("%o/segmeta.json", seg_name);
    Seg_Write_File(ivars->segment, ivars->folder);
    Snapshot_Add_Entry(snapshot, seg_name);
    DECREF(segmeta_filename);

    // Collapse segment files into compound file.
    Folder_Consolidate(ivars->folder, seg_name);
}
예제 #8
0
void
BGMerger_Prepare_Commit_IMP(BackgroundMerger *self) {
    BackgroundMergerIVARS *const ivars = BGMerger_IVARS(self);
    Vector   *seg_readers     = PolyReader_Get_Seg_Readers(ivars->polyreader);
    uint32_t  num_seg_readers = Vec_Get_Size(seg_readers);
    uint32_t  segs_merged     = 0;

    if (ivars->prepared) {
        THROW(ERR, "Can't call Prepare_Commit() more than once");
    }

    // Maybe merge existing index data.
    if (num_seg_readers) {
        segs_merged = S_maybe_merge(self);
    }

    if (!segs_merged) {
        // Nothing merged.  Leave `needs_commit` false and bail out.
        ivars->prepared = true;
        return;
    }
    // Finish the segment and write a new snapshot file.
    else {
        Folder   *folder   = ivars->folder;
        Snapshot *snapshot = ivars->snapshot;

        // Write out new deletions.
        if (DelWriter_Updated(ivars->del_writer)) {
            // Only write out if they haven't all been applied.
            if (segs_merged != num_seg_readers) {
                DelWriter_Finish(ivars->del_writer);
            }
        }

        // Finish the segment.
        SegWriter_Finish(ivars->seg_writer);

        // Grab the write lock.
        S_obtain_write_lock(self);
        if (!ivars->write_lock) {
            RETHROW(INCREF(Err_get_error()));
        }

        // Write temporary snapshot file.
        DECREF(ivars->snapfile);
        String *snapfile = IxManager_Make_Snapshot_Filename(ivars->manager);
        ivars->snapfile = Str_Cat_Trusted_Utf8(snapfile, ".temp", 5);
        DECREF(snapfile);
        Folder_Delete(folder, ivars->snapfile);
        Snapshot_Write_File(snapshot, folder, ivars->snapfile);

        // Determine whether the index has been updated while this background
        // merge process was running.

        String *start_snapfile
            = Snapshot_Get_Path(PolyReader_Get_Snapshot(ivars->polyreader));
        Snapshot *latest_snapshot
            = Snapshot_Read_File(Snapshot_new(), ivars->folder, NULL);
        String *latest_snapfile = Snapshot_Get_Path(latest_snapshot);
        bool index_updated
            = !Str_Equals(start_snapfile, (Obj*)latest_snapfile);

        if (index_updated) {
            /* See if new deletions have been applied since this
             * background merge process started against any of the
             * segments we just merged away.  If that's true, we need to
             * write another segment which applies the deletions against
             * the new composite segment.
             */
            S_merge_updated_deletions(self);

            // Add the fresh content to our snapshot. (It's important to
            // run this AFTER S_merge_updated_deletions, because otherwise
            // we couldn't tell whether the deletion counts changed.)
            Vector *files = Snapshot_List(latest_snapshot);
            for (uint32_t i = 0, max = Vec_Get_Size(files); i < max; i++) {
                String *file = (String*)Vec_Fetch(files, i);
                if (Str_Starts_With_Utf8(file, "seg_", 4)) {
                    int64_t gen = (int64_t)IxFileNames_extract_gen(file);
                    if (gen > ivars->cutoff) {
                        Snapshot_Add_Entry(ivars->snapshot, file);
                    }
                }
            }
            DECREF(files);

            // Since the snapshot content has changed, we need to rewrite it.
            Folder_Delete(folder, ivars->snapfile);
            Snapshot_Write_File(snapshot, folder, ivars->snapfile);
        }

        DECREF(latest_snapshot);

        ivars->needs_commit = true;
    }

    // Close reader, so that we can delete its files if appropriate.
    PolyReader_Close(ivars->polyreader);

    ivars->prepared = true;
}