예제 #1
0
파일: QueryParser.c 프로젝트: kidaa/lucy
QueryParser*
QParser_init(QueryParser *self, Schema *schema, Analyzer *analyzer,
             String *default_boolop, Vector *fields) {
    QueryParserIVARS *const ivars = QParser_IVARS(self);
    // Init.
    ivars->heed_colons = false;
    ivars->lexer       = QueryLexer_new();

    // Assign.
    ivars->schema         = (Schema*)INCREF(schema);
    ivars->analyzer       = (Analyzer*)INCREF(analyzer);
    ivars->default_boolop = default_boolop
                           ? Str_Clone(default_boolop)
                           : Str_new_from_trusted_utf8("OR", 2);

    if (fields) {
        ivars->fields = Vec_Clone(fields);
        for (uint32_t i = 0, max = Vec_Get_Size(fields); i < max; i++) {
            CERTIFY(Vec_Fetch(fields, i), STRING);
        }
        Vec_Sort(ivars->fields);
    }
    else {
        Vector *all_fields = Schema_All_Fields(schema);
        uint32_t num_fields = Vec_Get_Size(all_fields);
        ivars->fields = Vec_new(num_fields);
        for (uint32_t i = 0; i < num_fields; i++) {
            String *field = (String*)Vec_Fetch(all_fields, i);
            FieldType *type = Schema_Fetch_Type(schema, field);
            if (type && FType_Indexed(type)) {
                Vec_Push(ivars->fields, INCREF(field));
            }
        }
        DECREF(all_fields);
    }
    Vec_Sort(ivars->fields);

    // Derive default "occur" from default boolean operator.
    if (Str_Equals_Utf8(ivars->default_boolop, "OR", 2)) {
        ivars->default_occur = SHOULD;
    }
    else if (Str_Equals_Utf8(ivars->default_boolop, "AND", 3)) {
        ivars->default_occur = MUST;
    }
    else {
        THROW(ERR, "Invalid value for default_boolop: %o", ivars->default_boolop);
    }

    return self;
}
예제 #2
0
파일: HeatMap.c 프로젝트: rectang/lucy
HeatMap*
HeatMap_init(HeatMap *self, Vector *spans, uint32_t window) {
    HeatMapIVARS *const ivars = HeatMap_IVARS(self);
    Vector *spans_copy = Vec_Clone(spans);
    Vector *spans_plus_boosts;

    ivars->spans  = NULL;
    ivars->window = window;

    Vec_Sort(spans_copy);
    spans_plus_boosts = HeatMap_Generate_Proximity_Boosts(self, spans_copy);
    Vec_Push_All(spans_plus_boosts, spans_copy);
    Vec_Sort(spans_plus_boosts);
    ivars->spans = HeatMap_Flatten_Spans(self, spans_plus_boosts);

    DECREF(spans_plus_boosts);
    DECREF(spans_copy);

    return self;
}
예제 #3
0
void
Snapshot_Write_File_IMP(Snapshot *self, Folder *folder, String *path) {
    SnapshotIVARS *const ivars = Snapshot_IVARS(self);
    Hash   *all_data = Hash_new(0);
    Vector *list     = Snapshot_List(self);

    // Update path.
    DECREF(ivars->path);
    if (path != NULL && Str_Get_Size(path) != 0) {
        ivars->path = Str_Clone(path);
    }
    else {
        String *latest = IxFileNames_latest_snapshot(folder);
        uint64_t gen = latest ? IxFileNames_extract_gen(latest) + 1 : 1;
        char base36[StrHelp_MAX_BASE36_BYTES];
        StrHelp_to_base36(gen, &base36);
        ivars->path = Str_newf("snapshot_%s.json", &base36);
        DECREF(latest);
    }

    // Don't overwrite.
    if (Folder_Exists(folder, ivars->path)) {
        THROW(ERR, "Snapshot file '%o' already exists", ivars->path);
    }

    // Sort, then store file names.
    Vec_Sort(list);
    Hash_Store_Utf8(all_data, "entries", 7, (Obj*)list);

    // Create a JSON-izable data structure.
    Hash_Store_Utf8(all_data, "format", 6,
                    (Obj*)Str_newf("%i32", (int32_t)Snapshot_current_file_format));
    Hash_Store_Utf8(all_data, "subformat", 9,
                    (Obj*)Str_newf("%i32", (int32_t)Snapshot_current_file_subformat));

    // Write out JSON-ized data to the new file.
    Json_spew_json((Obj*)all_data, folder, ivars->path);

    DECREF(all_data);
}
예제 #4
0
static void
S_do_consolidate(CompoundFileWriter *self, CompoundFileWriterIVARS *ivars) {
    UNUSED_VAR(self);
    Folder    *folder       = ivars->folder;
    Hash      *metadata     = Hash_new(0);
    Hash      *sub_files    = Hash_new(0);
    Vector    *files        = Folder_List(folder, NULL);
    Vector    *merged       = Vec_new(Vec_Get_Size(files));
    String    *cf_file      = (String*)SSTR_WRAP_UTF8("cf.dat", 6);
    OutStream *outstream    = Folder_Open_Out(folder, (String*)cf_file);
    bool       rename_success;

    if (!outstream) { RETHROW(INCREF(Err_get_error())); }

    // Start metadata.
    Hash_Store_Utf8(metadata, "files", 5, INCREF(sub_files));
    Hash_Store_Utf8(metadata, "format", 6,
                    (Obj*)Str_newf("%i32", CFWriter_current_file_format));

    Vec_Sort(files);
    for (uint32_t i = 0, max = Vec_Get_Size(files); i < max; i++) {
        String *infilename = (String*)Vec_Fetch(files, i);

        if (!Str_Ends_With_Utf8(infilename, ".json", 5)) {
            InStream *instream   = Folder_Open_In(folder, infilename);
            Hash     *file_data  = Hash_new(2);
            int64_t   offset, len;

            if (!instream) { RETHROW(INCREF(Err_get_error())); }

            // Absorb the file.
            offset = OutStream_Tell(outstream);
            OutStream_Absorb(outstream, instream);
            len = OutStream_Tell(outstream) - offset;

            // Record offset and length.
            Hash_Store_Utf8(file_data, "offset", 6,
                            (Obj*)Str_newf("%i64", offset));
            Hash_Store_Utf8(file_data, "length", 6,
                            (Obj*)Str_newf("%i64", len));
            Hash_Store(sub_files, infilename, (Obj*)file_data);
            Vec_Push(merged, INCREF(infilename));

            // Add filler NULL bytes so that every sub-file begins on a file
            // position multiple of 8.
            OutStream_Align(outstream, 8);

            InStream_Close(instream);
            DECREF(instream);
        }
    }

    // Write metadata to cfmeta file.
    String *cfmeta_temp = (String*)SSTR_WRAP_UTF8("cfmeta.json.temp", 16);
    String *cfmeta_file = (String*)SSTR_WRAP_UTF8("cfmeta.json", 11);
    Json_spew_json((Obj*)metadata, (Folder*)ivars->folder, cfmeta_temp);
    rename_success = Folder_Rename(ivars->folder, cfmeta_temp, cfmeta_file);
    if (!rename_success) { RETHROW(INCREF(Err_get_error())); }

    // Clean up.
    OutStream_Close(outstream);
    DECREF(outstream);
    DECREF(files);
    DECREF(metadata);
    /*
    HashIterator *iter = HashIter_new(sub_files);
    while (HashIter_Next(iter)) {
        String *merged_file = HashIter_Get_Key(iter);
        if (!Folder_Delete(folder, merged_file)) {
            String *mess = MAKE_MESS("Can't delete '%o'", merged_file);
            DECREF(sub_files);
            Err_throw_mess(ERR, mess);
        }
    }
    DECREF(iter);
    */
    DECREF(sub_files);
    for (uint32_t i = 0, max = Vec_Get_Size(merged); i < max; i++) {
        String *merged_file = (String*)Vec_Fetch(merged, i);
        if (!Folder_Delete(folder, merged_file)) {
            String *mess = MAKE_MESS("Can't delete '%o'", merged_file);
            DECREF(merged);
            Err_throw_mess(ERR, mess);
        }
    }
    DECREF(merged);
}
예제 #5
0
파일: FilePurger.c 프로젝트: kidaa/lucy
void
FilePurger_Purge_IMP(FilePurger *self) {
    FilePurgerIVARS *const ivars = FilePurger_IVARS(self);
    Lock *deletion_lock = IxManager_Make_Deletion_Lock(ivars->manager);

    // Obtain deletion lock, purge files, release deletion lock.
    Lock_Clear_Stale(deletion_lock);
    if (Lock_Obtain(deletion_lock)) {
        Folder *folder   = ivars->folder;
        Hash   *failures = Hash_new(0);
        Vector *purgables;
        Vector *snapshots;

        S_discover_unused(self, &purgables, &snapshots);

        // Attempt to delete entries -- if failure, no big deal, just try
        // again later.  Proceed in reverse lexical order so that directories
        // get deleted after they've been emptied.
        Vec_Sort(purgables);
        for (uint32_t i = Vec_Get_Size(purgables); i--;) {
            String *entry = (String*)Vec_Fetch(purgables, i);
            if (Hash_Fetch(ivars->disallowed, entry)) { continue; }
            if (!Folder_Delete(folder, entry)) {
                if (Folder_Exists(folder, entry)) {
                    Hash_Store(failures, entry, (Obj*)CFISH_TRUE);
                }
            }
        }

        for (uint32_t i = 0, max = Vec_Get_Size(snapshots); i < max; i++) {
            Snapshot *snapshot = (Snapshot*)Vec_Fetch(snapshots, i);
            bool snapshot_has_failures = false;
            if (Hash_Get_Size(failures)) {
                // Only delete snapshot files if all of their entries were
                // successfully deleted.
                Vector *entries = Snapshot_List(snapshot);
                for (uint32_t j = Vec_Get_Size(entries); j--;) {
                    String *entry = (String*)Vec_Fetch(entries, j);
                    if (Hash_Fetch(failures, entry)) {
                        snapshot_has_failures = true;
                        break;
                    }
                }
                DECREF(entries);
            }
            if (!snapshot_has_failures) {
                String *snapfile = Snapshot_Get_Path(snapshot);
                Folder_Delete(folder, snapfile);
            }
        }

        DECREF(failures);
        DECREF(purgables);
        DECREF(snapshots);
        Lock_Release(deletion_lock);
    }
    else {
        WARN("Can't obtain deletion lock, skipping deletion of "
             "obsolete files");
    }

    DECREF(deletion_lock);
}