Obj* RangeQuery_Load_IMP(RangeQuery *self, Obj *dump) { Hash *source = (Hash*)CERTIFY(dump, HASH); RangeQuery_Load_t super_load = SUPER_METHOD_PTR(RANGEQUERY, LUCY_RangeQuery_Load); RangeQuery *loaded = (RangeQuery*)super_load(self, dump); RangeQueryIVARS *loaded_ivars = RangeQuery_IVARS(loaded); Obj *field = CERTIFY(Hash_Fetch_Utf8(source, "field", 5), OBJ); loaded_ivars->field = (String*)CERTIFY(Freezer_load(field), STRING); Obj *lower_term = Hash_Fetch_Utf8(source, "lower_term", 10); if (lower_term) { loaded_ivars->lower_term = (Obj*)CERTIFY(Freezer_load(lower_term), OBJ); } Obj *upper_term = Hash_Fetch_Utf8(source, "upper_term", 10); if (upper_term) { loaded_ivars->upper_term = (Obj*)CERTIFY(Freezer_load(upper_term), OBJ); } Obj *include_lower = CERTIFY(Hash_Fetch_Utf8(source, "include_lower", 13), OBJ); loaded_ivars->include_lower = Json_obj_to_bool(include_lower); Obj *include_upper = CERTIFY(Hash_Fetch_Utf8(source, "include_upper", 13), OBJ); loaded_ivars->include_upper = Json_obj_to_bool(include_upper); return (Obj*)loaded; }
Obj* TermQuery_Load_IMP(TermQuery *self, Obj *dump) { Hash *source = (Hash*)CERTIFY(dump, HASH); TermQuery_Load_t super_load = SUPER_METHOD_PTR(TERMQUERY, LUCY_TermQuery_Load); TermQuery *loaded = (TermQuery*)super_load(self, dump); TermQueryIVARS *loaded_ivars = TermQuery_IVARS(loaded); Obj *field = CERTIFY(Hash_Fetch_Utf8(source, "field", 5), OBJ); loaded_ivars->field = (String*)CERTIFY(Freezer_load(field), STRING); Obj *term = CERTIFY(Hash_Fetch_Utf8(source, "term", 4), OBJ); loaded_ivars->term = (Obj*)CERTIFY(Freezer_load(term), OBJ); return (Obj*)loaded; }
Obj* PhraseQuery_Load_IMP(PhraseQuery *self, Obj *dump) { Hash *source = (Hash*)CERTIFY(dump, HASH); PhraseQuery_Load_t super_load = SUPER_METHOD_PTR(PHRASEQUERY, LUCY_PhraseQuery_Load); PhraseQuery *loaded = (PhraseQuery*)super_load(self, dump); PhraseQueryIVARS *loaded_ivars = PhraseQuery_IVARS(loaded); Obj *field = CERTIFY(Hash_Fetch_Utf8(source, "field", 5), OBJ); loaded_ivars->field = (String*)CERTIFY(Freezer_load(field), STRING); Obj *terms = CERTIFY(Hash_Fetch_Utf8(source, "terms", 5), OBJ); loaded_ivars->terms = (Vector*)CERTIFY(Freezer_load(terms), VECTOR); return (Obj*)loaded; }
Obj* ProximityQuery_Load_IMP(ProximityQuery *self, Obj *dump) { Hash *source = (Hash*)CERTIFY(dump, HASH); ProximityQuery_Load_t super_load = SUPER_METHOD_PTR(PROXIMITYQUERY, LUCY_ProximityQuery_Load); ProximityQuery *loaded = (ProximityQuery*)super_load(self, dump); ProximityQueryIVARS *loaded_ivars = ProximityQuery_IVARS(loaded); Obj *field = CERTIFY(Hash_Fetch_Utf8(source, "field", 5), OBJ); loaded_ivars->field = (String*)CERTIFY(Freezer_load(field), STRING); Obj *terms = CERTIFY(Hash_Fetch_Utf8(source, "terms", 5), OBJ); loaded_ivars->terms = (VArray*)CERTIFY(Freezer_load(terms), VARRAY); Obj *within = CERTIFY(Hash_Fetch_Utf8(source, "within", 6), OBJ); loaded_ivars->within = (uint32_t)Obj_To_I64(within); return (Obj*)loaded; }
static void test_Dump_Load_and_Equals(TestBatchRunner *runner) { Query *a_leaf = (Query*)TestUtils_make_leaf_query(NULL, "a"); Query *b_leaf = (Query*)TestUtils_make_leaf_query(NULL, "b"); Query *c_leaf = (Query*)TestUtils_make_leaf_query(NULL, "c"); RequiredOptionalQuery *query = ReqOptQuery_new(a_leaf, b_leaf); RequiredOptionalQuery *kids_differ = ReqOptQuery_new(a_leaf, c_leaf); RequiredOptionalQuery *boost_differs = ReqOptQuery_new(a_leaf, b_leaf); Obj *dump = (Obj*)ReqOptQuery_Dump(query); RequiredOptionalQuery *clone = (RequiredOptionalQuery*)Freezer_load(dump); TEST_FALSE(runner, ReqOptQuery_Equals(query, (Obj*)kids_differ), "Different kids spoil Equals"); TEST_TRUE(runner, ReqOptQuery_Equals(query, (Obj*)boost_differs), "Equals with identical boosts"); ReqOptQuery_Set_Boost(boost_differs, 1.5); TEST_FALSE(runner, ReqOptQuery_Equals(query, (Obj*)boost_differs), "Different boost spoils Equals"); TEST_TRUE(runner, ReqOptQuery_Equals(query, (Obj*)clone), "Dump => Load round trip"); DECREF(a_leaf); DECREF(b_leaf); DECREF(c_leaf); DECREF(query); DECREF(kids_differ); DECREF(boost_differs); DECREF(dump); DECREF(clone); }
Obj* LeafQuery_Load_IMP(LeafQuery *self, Obj *dump) { Hash *source = (Hash*)CERTIFY(dump, HASH); LeafQuery_Load_t super_load = SUPER_METHOD_PTR(LEAFQUERY, LUCY_LeafQuery_Load); LeafQuery *loaded = (LeafQuery*)super_load(self, dump); LeafQueryIVARS *loaded_ivars = LeafQuery_IVARS(loaded); Obj *field = Hash_Fetch_Utf8(source, "field", 5); if (field) { loaded_ivars->field = (String*)CERTIFY(Freezer_load(field), STRING); } Obj *text = CERTIFY(Hash_Fetch_Utf8(source, "text", 4), OBJ); loaded_ivars->text = (String*)CERTIFY(Freezer_load(text), STRING); return (Obj*)loaded; }
Obj* PolyQuery_Load_IMP(PolyQuery *self, Obj *dump) { Hash *source = (Hash*)CERTIFY(dump, HASH); PolyQuery_Load_t super_load = SUPER_METHOD_PTR(POLYQUERY, LUCY_PolyQuery_Load); PolyQuery *loaded = (PolyQuery*)super_load(self, dump); Obj *children = CERTIFY(Hash_Fetch_Utf8(source, "children", 8), OBJ); PolyQuery_IVARS(loaded)->children = (Vector*)CERTIFY(Freezer_load(children), VECTOR); return (Obj*)loaded; }
Obj* S_load_from_array(Vector *dump) { Vector *loaded = Vec_new(Vec_Get_Size(dump)); for (size_t i = 0, max = Vec_Get_Size(dump); i < max; i++) { Obj *elem_dump = Vec_Fetch(dump, i); if (elem_dump) { Vec_Store(loaded, i, Freezer_load(elem_dump)); } } return (Obj*)loaded; }
Obj* SnowStop_Load_IMP(SnowballStopFilter *self, Obj *dump) { Hash *source = (Hash*)CERTIFY(dump, HASH); SnowStop_Load_t super_load = SUPER_METHOD_PTR(SNOWBALLSTOPFILTER, LUCY_SnowStop_Load); SnowballStopFilter *loaded = (SnowballStopFilter*)super_load(self, dump); Obj *stoplist = Hash_Fetch_Utf8(source, "stoplist", 8); if (stoplist) { SnowStop_IVARS(loaded)->stoplist = (Hash*)CERTIFY(Freezer_load(stoplist), HASH); } return (Obj*)loaded; }
static void test_Dump_Load_and_Equals(TestBatchRunner *runner) { StandardTokenizer *tokenizer = StandardTokenizer_new(); Normalizer *normalizer = Normalizer_new(NULL, true, false); FullTextType *type = FullTextType_new((Analyzer*)tokenizer); FullTextType *other = FullTextType_new((Analyzer*)normalizer); FullTextType *boost_differs = FullTextType_new((Analyzer*)tokenizer); FullTextType *not_indexed = FullTextType_new((Analyzer*)tokenizer); FullTextType *not_stored = FullTextType_new((Analyzer*)tokenizer); FullTextType *highlightable = FullTextType_new((Analyzer*)tokenizer); Obj *dump = (Obj*)FullTextType_Dump(type); Obj *clone = Freezer_load(dump); Obj *another_dump = (Obj*)FullTextType_Dump_For_Schema(type); FullTextType_Set_Boost(boost_differs, 1.5); FullTextType_Set_Indexed(not_indexed, false); FullTextType_Set_Stored(not_stored, false); FullTextType_Set_Highlightable(highlightable, true); // (This step is normally performed by Schema_Load() internally.) Hash_Store_Utf8((Hash*)another_dump, "analyzer", 8, INCREF(tokenizer)); FullTextType *another_clone = FullTextType_Load(type, another_dump); TEST_FALSE(runner, FullTextType_Equals(type, (Obj*)boost_differs), "Equals() false with different boost"); TEST_FALSE(runner, FullTextType_Equals(type, (Obj*)other), "Equals() false with different Analyzer"); TEST_FALSE(runner, FullTextType_Equals(type, (Obj*)not_indexed), "Equals() false with indexed => false"); TEST_FALSE(runner, FullTextType_Equals(type, (Obj*)not_stored), "Equals() false with stored => false"); TEST_FALSE(runner, FullTextType_Equals(type, (Obj*)highlightable), "Equals() false with highlightable => true"); TEST_TRUE(runner, FullTextType_Equals(type, (Obj*)clone), "Dump => Load round trip"); TEST_TRUE(runner, FullTextType_Equals(type, (Obj*)another_clone), "Dump_For_Schema => Load round trip"); DECREF(another_clone); DECREF(dump); DECREF(clone); DECREF(another_dump); DECREF(highlightable); DECREF(not_stored); DECREF(not_indexed); DECREF(boost_differs); DECREF(other); DECREF(type); DECREF(normalizer); DECREF(tokenizer); }
PolyAnalyzer* PolyAnalyzer_Load_IMP(PolyAnalyzer *self, Obj *dump) { Hash *source = (Hash*)CERTIFY(dump, HASH); PolyAnalyzer_Load_t super_load = SUPER_METHOD_PTR(POLYANALYZER, LUCY_PolyAnalyzer_Load); PolyAnalyzer *loaded = super_load(self, dump); VArray *analyzer_dumps = (VArray*)CERTIFY(Hash_Fetch_Utf8(source, "analyzers", 9), VARRAY); VArray *analyzers = (VArray*)CERTIFY(Freezer_load((Obj*)analyzer_dumps), VARRAY); PolyAnalyzer_init(loaded, NULL, analyzers); DECREF(analyzers); return loaded; }
static Obj* S_load_from_hash(Hash *dump) { String *class_name = (String*)Hash_Fetch_Utf8(dump, "_class", 6); // Assume that the presence of the "_class" key paired with a valid class // name indicates the output of a dump() rather than an ordinary Hash. if (class_name && Str_is_a(class_name, STRING)) { Class *klass = Class_fetch_class(class_name); if (!klass) { String *parent_class_name = Class_find_parent_class(class_name); if (parent_class_name) { Class *parent = Class_singleton(parent_class_name, NULL); klass = Class_singleton(class_name, parent); DECREF(parent_class_name); } else { // TODO: Fix load() so that it works with ordinary hash keys // named "_class". THROW(ERR, "Can't find class '%o'", class_name); } } // Dispatch to an alternate Load() method. if (klass) { return S_load_via_load_method(klass, (Obj*)dump); } } // It's an ordinary Hash. Hash *loaded = Hash_new(Hash_Get_Size(dump)); HashIterator *iter = HashIter_new(dump); while (HashIter_Next(iter)) { String *key = HashIter_Get_Key(iter); Obj *value = HashIter_Get_Value(iter); Hash_Store(loaded, key, Freezer_load(value)); } DECREF(iter); return (Obj*)loaded; }
FullTextType* FullTextType_Load_IMP(FullTextType *self, Obj *dump) { UNUSED_VAR(self); Hash *source = (Hash*)CERTIFY(dump, HASH); String *class_name = (String*)Hash_Fetch_Utf8(source, "_class", 6); Class *klass = (class_name != NULL && Obj_is_a((Obj*)class_name, STRING)) ? Class_singleton(class_name, NULL) : FULLTEXTTYPE; FullTextType *loaded = (FullTextType*)Class_Make_Obj(klass); // Extract boost. Obj *boost_dump = Hash_Fetch_Utf8(source, "boost", 5); float boost = boost_dump ? (float)Json_obj_to_f64(boost_dump) : 1.0f; // Find boolean properties. Obj *indexed_dump = Hash_Fetch_Utf8(source, "indexed", 7); Obj *stored_dump = Hash_Fetch_Utf8(source, "stored", 6); Obj *sort_dump = Hash_Fetch_Utf8(source, "sortable", 8); Obj *hl_dump = Hash_Fetch_Utf8(source, "highlightable", 13); bool indexed = indexed_dump ? Json_obj_to_bool(indexed_dump) : true; bool stored = stored_dump ? Json_obj_to_bool(stored_dump) : true; bool sortable = sort_dump ? Json_obj_to_bool(sort_dump) : false; bool hl = hl_dump ? Json_obj_to_bool(hl_dump) : false; // Extract an Analyzer. Obj *analyzer_dump = Hash_Fetch_Utf8(source, "analyzer", 8); Analyzer *analyzer = NULL; if (analyzer_dump) { if (Obj_is_a(analyzer_dump, ANALYZER)) { // Schema munged the dump and installed a shared analyzer. analyzer = (Analyzer*)INCREF(analyzer_dump); } else if (Obj_is_a((Obj*)analyzer_dump, HASH)) { analyzer = (Analyzer*)Freezer_load(analyzer_dump); } } CERTIFY(analyzer, ANALYZER); FullTextType_init2(loaded, analyzer, boost, indexed, stored, sortable, hl); DECREF(analyzer); return loaded; }
static void test_Dump_Load_and_Equals(TestBatchRunner *runner) { Int32Type *i32 = Int32Type_new(); Int64Type *i64 = Int64Type_new(); Float32Type *f32 = Float32Type_new(); Float64Type *f64 = Float64Type_new(); TEST_FALSE(runner, Int32Type_Equals(i32, (Obj*)i64), "Int32Type_Equals() false for different type"); TEST_FALSE(runner, Int32Type_Equals(i32, NULL), "Int32Type_Equals() false for NULL"); TEST_FALSE(runner, Int64Type_Equals(i64, (Obj*)i32), "Int64Type_Equals() false for different type"); TEST_FALSE(runner, Int64Type_Equals(i64, NULL), "Int64Type_Equals() false for NULL"); TEST_FALSE(runner, Float32Type_Equals(f32, (Obj*)f64), "Float32Type_Equals() false for different type"); TEST_FALSE(runner, Float32Type_Equals(f32, NULL), "Float32Type_Equals() false for NULL"); TEST_FALSE(runner, Float64Type_Equals(f64, (Obj*)f32), "Float64Type_Equals() false for different type"); TEST_FALSE(runner, Float64Type_Equals(f64, NULL), "Float64Type_Equals() false for NULL"); { Obj *dump = (Obj*)Int32Type_Dump(i32); Obj *other = Freezer_load(dump); TEST_TRUE(runner, Int32Type_Equals(i32, other), "Dump => Load round trip for Int32Type"); DECREF(dump); DECREF(other); } { Obj *dump = (Obj*)Int64Type_Dump(i64); Obj *other = Freezer_load(dump); TEST_TRUE(runner, Int64Type_Equals(i64, other), "Dump => Load round trip for Int64Type"); DECREF(dump); DECREF(other); } { Obj *dump = (Obj*)Float32Type_Dump(f32); Obj *other = Freezer_load(dump); TEST_TRUE(runner, Float32Type_Equals(f32, other), "Dump => Load round trip for Float32Type"); DECREF(dump); DECREF(other); } { Obj *dump = (Obj*)Float64Type_Dump(f64); Obj *other = Freezer_load(dump); TEST_TRUE(runner, Float64Type_Equals(f64, other), "Dump => Load round trip for Float64Type"); DECREF(dump); DECREF(other); } DECREF(i32); DECREF(i64); DECREF(f32); DECREF(f64); }
BackgroundMerger* BGMerger_init(BackgroundMerger *self, Obj *index, IndexManager *manager) { BackgroundMergerIVARS *const ivars = BGMerger_IVARS(self); Folder *folder = S_init_folder(index); // Init. ivars->optimize = false; ivars->prepared = false; ivars->needs_commit = false; ivars->snapfile = NULL; ivars->doc_maps = Hash_new(0); // Assign. ivars->folder = folder; if (manager) { ivars->manager = (IndexManager*)INCREF(manager); } else { ivars->manager = IxManager_new(NULL, NULL); IxManager_Set_Write_Lock_Timeout(ivars->manager, 10000); } IxManager_Set_Folder(ivars->manager, folder); // Obtain write lock (which we'll only hold briefly), then merge lock. S_obtain_write_lock(self); if (!ivars->write_lock) { DECREF(self); RETHROW(INCREF(Err_get_error())); } S_obtain_merge_lock(self); if (!ivars->merge_lock) { DECREF(self); RETHROW(INCREF(Err_get_error())); } // Find the latest snapshot. If there's no index content, bail early. ivars->snapshot = Snapshot_Read_File(Snapshot_new(), folder, NULL); if (!Snapshot_Get_Path(ivars->snapshot)) { S_release_write_lock(self); S_release_merge_lock(self); return self; } // Create FilePurger. Zap detritus from previous sessions. ivars->file_purger = FilePurger_new(folder, ivars->snapshot, ivars->manager); FilePurger_Purge(ivars->file_purger); // Open a PolyReader, passing in the IndexManager so we get a read lock on // the Snapshot's files -- so that Indexers don't zap our files while // we're operating in the background. ivars->polyreader = PolyReader_open((Obj*)folder, NULL, ivars->manager); // Clone the PolyReader's schema. Obj *dump = (Obj*)Schema_Dump(PolyReader_Get_Schema(ivars->polyreader)); ivars->schema = (Schema*)CERTIFY(Freezer_load(dump), SCHEMA); DECREF(dump); // Create new Segment. int64_t new_seg_num = IxManager_Highest_Seg_Num(ivars->manager, ivars->snapshot) + 1; Vector *fields = Schema_All_Fields(ivars->schema); ivars->segment = Seg_new(new_seg_num); for (uint32_t i = 0, max = Vec_Get_Size(fields); i < max; i++) { Seg_Add_Field(ivars->segment, (String*)Vec_Fetch(fields, i)); } DECREF(fields); // Our "cutoff" is the segment this BackgroundMerger will write. Now that // we've determined the cutoff, write the merge data file. ivars->cutoff = Seg_Get_Number(ivars->segment); IxManager_Write_Merge_Data(ivars->manager, ivars->cutoff); /* Create the SegWriter but hold off on preparing the new segment * directory -- because if we don't need to merge any segments we don't * need it. (We've reserved the dir by plopping down the merge.json * file.) */ ivars->seg_writer = SegWriter_new(ivars->schema, ivars->snapshot, ivars->segment, ivars->polyreader); // Grab a local ref to the DeletionsWriter. ivars->del_writer = (DeletionsWriter*)INCREF(SegWriter_Get_Del_Writer(ivars->seg_writer)); // Release the write lock. Now new Indexers can start while we work in // the background. S_release_write_lock(self); return self; }
void S_try_open_elements(void *context) { struct try_open_elements_context *args = (struct try_open_elements_context*)context; PolyReader *self = args->self; PolyReaderIVARS *const ivars = PolyReader_IVARS(self); VArray *files = Snapshot_List(ivars->snapshot); Folder *folder = PolyReader_Get_Folder(self); uint32_t num_segs = 0; uint64_t latest_schema_gen = 0; String *schema_file = NULL; // Find schema file, count segments. for (uint32_t i = 0, max = VA_Get_Size(files); i < max; i++) { String *entry = (String*)VA_Fetch(files, i); if (Seg_valid_seg_name(entry)) { num_segs++; } else if (Str_Starts_With_Utf8(entry, "schema_", 7) && Str_Ends_With_Utf8(entry, ".json", 5) ) { uint64_t gen = IxFileNames_extract_gen(entry); if (gen > latest_schema_gen) { latest_schema_gen = gen; schema_file = entry; } } } // Read Schema. if (!schema_file) { DECREF(files); THROW(ERR, "Can't find a schema file."); } else { Obj *dump = Json_slurp_json(folder, schema_file); if (dump) { // read file successfully DECREF(ivars->schema); ivars->schema = (Schema*)CERTIFY(Freezer_load(dump), SCHEMA); DECREF(dump); schema_file = NULL; } else { String *mess = MAKE_MESS("Failed to parse %o", schema_file); DECREF(files); Err_throw_mess(ERR, mess); } } VArray *segments = VA_new(num_segs); for (uint32_t i = 0, max = VA_Get_Size(files); i < max; i++) { String *entry = (String*)VA_Fetch(files, i); // Create a Segment for each segmeta. if (Seg_valid_seg_name(entry)) { int64_t seg_num = IxFileNames_extract_gen(entry); Segment *segment = Seg_new(seg_num); // Bail if reading the file fails (probably because it's been // deleted and a new snapshot file has been written so we need to // retry). if (Seg_Read_File(segment, folder)) { VA_Push(segments, (Obj*)segment); } else { String *mess = MAKE_MESS("Failed to read %o", entry); DECREF(segment); DECREF(segments); DECREF(files); Err_throw_mess(ERR, mess); } } } // Sort the segments by age. VA_Sort(segments, NULL, NULL); // Open individual SegReaders. struct try_open_segreader_context seg_context; seg_context.schema = PolyReader_Get_Schema(self); seg_context.folder = folder; seg_context.snapshot = PolyReader_Get_Snapshot(self); seg_context.segments = segments; seg_context.result = NULL; args->seg_readers = VA_new(num_segs); Err *error = NULL; for (uint32_t seg_tick = 0; seg_tick < num_segs; seg_tick++) { seg_context.seg_tick = seg_tick; error = Err_trap(S_try_open_segreader, &seg_context); if (error) { break; } VA_Push(args->seg_readers, (Obj*)seg_context.result); seg_context.result = NULL; } DECREF(segments); DECREF(files); if (error) { DECREF(args->seg_readers); args->seg_readers = NULL; RETHROW(error); } }
Schema* Schema_Load_IMP(Schema *self, Obj *dump) { Hash *source = (Hash*)CERTIFY(dump, HASH); String *class_name = (String*)CERTIFY(Hash_Fetch_Utf8(source, "_class", 6), STRING); Class *klass = Class_singleton(class_name, NULL); Schema *loaded = (Schema*)Class_Make_Obj(klass); Hash *type_dumps = (Hash*)CERTIFY(Hash_Fetch_Utf8(source, "fields", 6), HASH); Vector *analyzer_dumps = (Vector*)CERTIFY(Hash_Fetch_Utf8(source, "analyzers", 9), VECTOR); Vector *analyzers = (Vector*)Freezer_load((Obj*)analyzer_dumps); UNUSED_VAR(self); // Start with a blank Schema. Schema_init(loaded); SchemaIVARS *const loaded_ivars = Schema_IVARS(loaded); Vec_Grow(loaded_ivars->uniq_analyzers, Vec_Get_Size(analyzers)); HashIterator *iter = HashIter_new(type_dumps); while (HashIter_Next(iter)) { String *field = HashIter_Get_Key(iter); Hash *type_dump = (Hash*)CERTIFY(HashIter_Get_Value(iter), HASH); String *type_str = (String*)Hash_Fetch_Utf8(type_dump, "type", 4); if (type_str) { if (Str_Equals_Utf8(type_str, "fulltext", 8)) { // Replace the "analyzer" tick with the real thing. Obj *tick = CERTIFY(Hash_Fetch_Utf8(type_dump, "analyzer", 8), OBJ); Analyzer *analyzer = (Analyzer*)Vec_Fetch(analyzers, (uint32_t)Json_obj_to_i64(tick)); if (!analyzer) { THROW(ERR, "Can't find analyzer for '%o'", field); } Hash_Store_Utf8(type_dump, "analyzer", 8, INCREF(analyzer)); FullTextType *type = (FullTextType*)S_load_type(FULLTEXTTYPE, (Obj*)type_dump); Schema_Spec_Field(loaded, field, (FieldType*)type); DECREF(type); } else if (Str_Equals_Utf8(type_str, "string", 6)) { StringType *type = (StringType*)S_load_type(STRINGTYPE, (Obj*)type_dump); Schema_Spec_Field(loaded, field, (FieldType*)type); DECREF(type); } else if (Str_Equals_Utf8(type_str, "blob", 4)) { BlobType *type = (BlobType*)S_load_type(BLOBTYPE, (Obj*)type_dump); Schema_Spec_Field(loaded, field, (FieldType*)type); DECREF(type); } else if (Str_Equals_Utf8(type_str, "i32_t", 5)) { Int32Type *type = (Int32Type*)S_load_type(INT32TYPE, (Obj*)type_dump); Schema_Spec_Field(loaded, field, (FieldType*)type); DECREF(type); } else if (Str_Equals_Utf8(type_str, "i64_t", 5)) { Int64Type *type = (Int64Type*)S_load_type(INT64TYPE, (Obj*)type_dump); Schema_Spec_Field(loaded, field, (FieldType*)type); DECREF(type); } else if (Str_Equals_Utf8(type_str, "f32_t", 5)) { Float32Type *type = (Float32Type*)S_load_type(FLOAT32TYPE, (Obj*)type_dump); Schema_Spec_Field(loaded, field, (FieldType*)type); DECREF(type); } else if (Str_Equals_Utf8(type_str, "f64_t", 5)) { Float64Type *type = (Float64Type*)S_load_type(FLOAT64TYPE, (Obj*)type_dump); Schema_Spec_Field(loaded, field, (FieldType*)type); DECREF(type); } else { THROW(ERR, "Unknown type '%o' for field '%o'", type_str, field); } } else { FieldType *type = (FieldType*)CERTIFY(Freezer_load((Obj*)type_dump), FIELDTYPE); Schema_Spec_Field(loaded, field, type); DECREF(type); } } DECREF(iter); DECREF(analyzers); return loaded; }
Indexer* Indexer_init(Indexer *self, Schema *schema, Obj *index, IndexManager *manager, int32_t flags) { IndexerIVARS *const ivars = Indexer_IVARS(self); bool create = (flags & Indexer_CREATE) ? true : false; bool truncate = (flags & Indexer_TRUNCATE) ? true : false; Folder *folder = S_init_folder(index, create); Snapshot *latest_snapshot = Snapshot_new(); // Init. ivars->stock_doc = Doc_new(NULL, 0); ivars->truncate = false; ivars->optimize = false; ivars->prepared = false; ivars->needs_commit = false; ivars->snapfile = NULL; ivars->merge_lock = NULL; // Assign. ivars->folder = folder; ivars->manager = manager ? (IndexManager*)INCREF(manager) : IxManager_new(NULL, NULL); IxManager_Set_Folder(ivars->manager, folder); // Get a write lock for this folder. Lock *write_lock = IxManager_Make_Write_Lock(ivars->manager); Lock_Clear_Stale(write_lock); if (Lock_Obtain(write_lock)) { // Only assign if successful, otherwise DESTROY unlocks -- bad! ivars->write_lock = write_lock; } else { DECREF(write_lock); DECREF(self); RETHROW(INCREF(Err_get_error())); } // Find the latest snapshot or create a new one. String *latest_snapfile = IxFileNames_latest_snapshot(folder); if (latest_snapfile) { Snapshot_Read_File(latest_snapshot, folder, latest_snapfile); } // Look for an existing Schema if one wasn't supplied. if (schema) { ivars->schema = (Schema*)INCREF(schema); } else { if (!latest_snapfile) { S_release_write_lock(self); THROW(ERR, "No Schema supplied, and can't find one in the index"); } else { String *schema_file = S_find_schema_file(latest_snapshot); Obj *dump = Json_slurp_json(folder, schema_file); if (dump) { // read file successfully ivars->schema = (Schema*)CERTIFY(Freezer_load(dump), SCHEMA); schema = ivars->schema; DECREF(dump); schema_file = NULL; } else { THROW(ERR, "Failed to parse %o", schema_file); } } } // If we're clobbering, start with an empty Snapshot and an empty // PolyReader. Otherwise, start with the most recent Snapshot and an // up-to-date PolyReader. if (truncate) { ivars->snapshot = Snapshot_new(); ivars->polyreader = PolyReader_new(schema, folder, NULL, NULL, NULL); ivars->truncate = true; } else { // TODO: clone most recent snapshot rather than read it twice. ivars->snapshot = (Snapshot*)INCREF(latest_snapshot); ivars->polyreader = latest_snapfile ? PolyReader_open((Obj*)folder, NULL, NULL) : PolyReader_new(schema, folder, NULL, NULL, NULL); if (latest_snapfile) { // Make sure than any existing fields which may have been // dynamically added during past indexing sessions get added. Schema *old_schema = PolyReader_Get_Schema(ivars->polyreader); Schema_Eat(schema, old_schema); } } // Zap detritus from previous sessions. // Note: we have to feed FilePurger with the most recent snapshot file // now, but with the Indexer's snapshot later. FilePurger *file_purger = FilePurger_new(folder, latest_snapshot, ivars->manager); FilePurger_Purge(file_purger); DECREF(file_purger); // Create a new segment. int64_t new_seg_num = IxManager_Highest_Seg_Num(ivars->manager, latest_snapshot) + 1; Lock *merge_lock = IxManager_Make_Merge_Lock(ivars->manager); if (Lock_Is_Locked(merge_lock)) { // If there's a background merge process going on, stay out of its // way. Hash *merge_data = IxManager_Read_Merge_Data(ivars->manager); Obj *cutoff_obj = merge_data ? Hash_Fetch_Utf8(merge_data, "cutoff", 6) : NULL; if (!cutoff_obj) { DECREF(merge_lock); DECREF(merge_data); THROW(ERR, "Background merge detected, but can't read merge data"); } else { int64_t cutoff = Json_obj_to_i64(cutoff_obj); if (cutoff >= new_seg_num) { new_seg_num = cutoff + 1; } } DECREF(merge_data); } ivars->segment = Seg_new(new_seg_num); // Add all known fields to Segment. Vector *fields = Schema_All_Fields(schema); for (size_t i = 0, max = Vec_Get_Size(fields); i < max; i++) { Seg_Add_Field(ivars->segment, (String*)Vec_Fetch(fields, i)); } DECREF(fields); DECREF(merge_lock); // Create new SegWriter and FilePurger. ivars->file_purger = FilePurger_new(folder, ivars->snapshot, ivars->manager); ivars->seg_writer = SegWriter_new(ivars->schema, ivars->snapshot, ivars->segment, ivars->polyreader); SegWriter_Prep_Seg_Dir(ivars->seg_writer); // Grab a local ref to the DeletionsWriter. ivars->del_writer = (DeletionsWriter*)INCREF( SegWriter_Get_Del_Writer(ivars->seg_writer)); DECREF(latest_snapfile); DECREF(latest_snapshot); return self; }