Obj*
RangeQuery_Load_IMP(RangeQuery *self, Obj *dump) {
Hash *source = (Hash*)CERTIFY(dump, HASH);
RangeQuery_Load_t super_load
= SUPER_METHOD_PTR(RANGEQUERY, LUCY_RangeQuery_Load);
RangeQuery *loaded = (RangeQuery*)super_load(self, dump);
RangeQueryIVARS *loaded_ivars = RangeQuery_IVARS(loaded);
Obj *field = CERTIFY(Hash_Fetch_Utf8(source, "field", 5), OBJ);
loaded_ivars->field = (String*)CERTIFY(Freezer_load(field), STRING);
Obj *lower_term = Hash_Fetch_Utf8(source, "lower_term", 10);
if (lower_term) {
loaded_ivars->lower_term
= (Obj*)CERTIFY(Freezer_load(lower_term), OBJ);
}
Obj *upper_term = Hash_Fetch_Utf8(source, "upper_term", 10);
if (upper_term) {
loaded_ivars->upper_term
= (Obj*)CERTIFY(Freezer_load(upper_term), OBJ);
}
Obj *include_lower
= CERTIFY(Hash_Fetch_Utf8(source, "include_lower", 13), OBJ);
loaded_ivars->include_lower = Json_obj_to_bool(include_lower);
Obj *include_upper
= CERTIFY(Hash_Fetch_Utf8(source, "include_upper", 13), OBJ);
loaded_ivars->include_upper = Json_obj_to_bool(include_upper);
return (Obj*)loaded;
}
Obj*
TermQuery_Load_IMP(TermQuery *self, Obj *dump) {
Hash *source = (Hash*)CERTIFY(dump, HASH);
TermQuery_Load_t super_load
= SUPER_METHOD_PTR(TERMQUERY, LUCY_TermQuery_Load);
TermQuery *loaded = (TermQuery*)super_load(self, dump);
TermQueryIVARS *loaded_ivars = TermQuery_IVARS(loaded);
Obj *field = CERTIFY(Hash_Fetch_Utf8(source, "field", 5), OBJ);
loaded_ivars->field = (String*)CERTIFY(Freezer_load(field), STRING);
Obj *term = CERTIFY(Hash_Fetch_Utf8(source, "term", 4), OBJ);
loaded_ivars->term = (Obj*)CERTIFY(Freezer_load(term), OBJ);
return (Obj*)loaded;
}
Obj*
PhraseQuery_Load_IMP(PhraseQuery *self, Obj *dump) {
Hash *source = (Hash*)CERTIFY(dump, HASH);
PhraseQuery_Load_t super_load
= SUPER_METHOD_PTR(PHRASEQUERY, LUCY_PhraseQuery_Load);
PhraseQuery *loaded = (PhraseQuery*)super_load(self, dump);
PhraseQueryIVARS *loaded_ivars = PhraseQuery_IVARS(loaded);
Obj *field = CERTIFY(Hash_Fetch_Utf8(source, "field", 5), OBJ);
loaded_ivars->field
= (String*)CERTIFY(Freezer_load(field), STRING);
Obj *terms = CERTIFY(Hash_Fetch_Utf8(source, "terms", 5), OBJ);
loaded_ivars->terms
= (Vector*)CERTIFY(Freezer_load(terms), VECTOR);
return (Obj*)loaded;
}
Obj*
ProximityQuery_Load_IMP(ProximityQuery *self, Obj *dump) {
Hash *source = (Hash*)CERTIFY(dump, HASH);
ProximityQuery_Load_t super_load
= SUPER_METHOD_PTR(PROXIMITYQUERY, LUCY_ProximityQuery_Load);
ProximityQuery *loaded = (ProximityQuery*)super_load(self, dump);
ProximityQueryIVARS *loaded_ivars = ProximityQuery_IVARS(loaded);
Obj *field = CERTIFY(Hash_Fetch_Utf8(source, "field", 5), OBJ);
loaded_ivars->field = (String*)CERTIFY(Freezer_load(field), STRING);
Obj *terms = CERTIFY(Hash_Fetch_Utf8(source, "terms", 5), OBJ);
loaded_ivars->terms = (VArray*)CERTIFY(Freezer_load(terms), VARRAY);
Obj *within = CERTIFY(Hash_Fetch_Utf8(source, "within", 6), OBJ);
loaded_ivars->within = (uint32_t)Obj_To_I64(within);
return (Obj*)loaded;
}
static void
test_Dump_Load_and_Equals(TestBatchRunner *runner) {
Query *a_leaf = (Query*)TestUtils_make_leaf_query(NULL, "a");
Query *b_leaf = (Query*)TestUtils_make_leaf_query(NULL, "b");
Query *c_leaf = (Query*)TestUtils_make_leaf_query(NULL, "c");
RequiredOptionalQuery *query = ReqOptQuery_new(a_leaf, b_leaf);
RequiredOptionalQuery *kids_differ = ReqOptQuery_new(a_leaf, c_leaf);
RequiredOptionalQuery *boost_differs = ReqOptQuery_new(a_leaf, b_leaf);
Obj *dump = (Obj*)ReqOptQuery_Dump(query);
RequiredOptionalQuery *clone
= (RequiredOptionalQuery*)Freezer_load(dump);
TEST_FALSE(runner, ReqOptQuery_Equals(query, (Obj*)kids_differ),
"Different kids spoil Equals");
TEST_TRUE(runner, ReqOptQuery_Equals(query, (Obj*)boost_differs),
"Equals with identical boosts");
ReqOptQuery_Set_Boost(boost_differs, 1.5);
TEST_FALSE(runner, ReqOptQuery_Equals(query, (Obj*)boost_differs),
"Different boost spoils Equals");
TEST_TRUE(runner, ReqOptQuery_Equals(query, (Obj*)clone),
"Dump => Load round trip");
DECREF(a_leaf);
DECREF(b_leaf);
DECREF(c_leaf);
DECREF(query);
DECREF(kids_differ);
DECREF(boost_differs);
DECREF(dump);
DECREF(clone);
}
Obj*
LeafQuery_Load_IMP(LeafQuery *self, Obj *dump) {
Hash *source = (Hash*)CERTIFY(dump, HASH);
LeafQuery_Load_t super_load
= SUPER_METHOD_PTR(LEAFQUERY, LUCY_LeafQuery_Load);
LeafQuery *loaded = (LeafQuery*)super_load(self, dump);
LeafQueryIVARS *loaded_ivars = LeafQuery_IVARS(loaded);
Obj *field = Hash_Fetch_Utf8(source, "field", 5);
if (field) {
loaded_ivars->field
= (String*)CERTIFY(Freezer_load(field), STRING);
}
Obj *text = CERTIFY(Hash_Fetch_Utf8(source, "text", 4), OBJ);
loaded_ivars->text = (String*)CERTIFY(Freezer_load(text), STRING);
return (Obj*)loaded;
}
Obj*
PolyQuery_Load_IMP(PolyQuery *self, Obj *dump) {
Hash *source = (Hash*)CERTIFY(dump, HASH);
PolyQuery_Load_t super_load
= SUPER_METHOD_PTR(POLYQUERY, LUCY_PolyQuery_Load);
PolyQuery *loaded = (PolyQuery*)super_load(self, dump);
Obj *children = CERTIFY(Hash_Fetch_Utf8(source, "children", 8), OBJ);
PolyQuery_IVARS(loaded)->children
= (Vector*)CERTIFY(Freezer_load(children), VECTOR);
return (Obj*)loaded;
}
Obj*
S_load_from_array(Vector *dump) {
Vector *loaded = Vec_new(Vec_Get_Size(dump));
for (size_t i = 0, max = Vec_Get_Size(dump); i < max; i++) {
Obj *elem_dump = Vec_Fetch(dump, i);
if (elem_dump) {
Vec_Store(loaded, i, Freezer_load(elem_dump));
}
}
return (Obj*)loaded;
}
Obj*
SnowStop_Load_IMP(SnowballStopFilter *self, Obj *dump) {
Hash *source = (Hash*)CERTIFY(dump, HASH);
SnowStop_Load_t super_load
= SUPER_METHOD_PTR(SNOWBALLSTOPFILTER, LUCY_SnowStop_Load);
SnowballStopFilter *loaded = (SnowballStopFilter*)super_load(self, dump);
Obj *stoplist = Hash_Fetch_Utf8(source, "stoplist", 8);
if (stoplist) {
SnowStop_IVARS(loaded)->stoplist
= (Hash*)CERTIFY(Freezer_load(stoplist), HASH);
}
return (Obj*)loaded;
}
static void
test_Dump_Load_and_Equals(TestBatchRunner *runner) {
StandardTokenizer *tokenizer = StandardTokenizer_new();
Normalizer *normalizer = Normalizer_new(NULL, true, false);
FullTextType *type = FullTextType_new((Analyzer*)tokenizer);
FullTextType *other = FullTextType_new((Analyzer*)normalizer);
FullTextType *boost_differs = FullTextType_new((Analyzer*)tokenizer);
FullTextType *not_indexed = FullTextType_new((Analyzer*)tokenizer);
FullTextType *not_stored = FullTextType_new((Analyzer*)tokenizer);
FullTextType *highlightable = FullTextType_new((Analyzer*)tokenizer);
Obj *dump = (Obj*)FullTextType_Dump(type);
Obj *clone = Freezer_load(dump);
Obj *another_dump = (Obj*)FullTextType_Dump_For_Schema(type);
FullTextType_Set_Boost(boost_differs, 1.5);
FullTextType_Set_Indexed(not_indexed, false);
FullTextType_Set_Stored(not_stored, false);
FullTextType_Set_Highlightable(highlightable, true);
// (This step is normally performed by Schema_Load() internally.)
Hash_Store_Utf8((Hash*)another_dump, "analyzer", 8, INCREF(tokenizer));
FullTextType *another_clone = FullTextType_Load(type, another_dump);
TEST_FALSE(runner, FullTextType_Equals(type, (Obj*)boost_differs),
"Equals() false with different boost");
TEST_FALSE(runner, FullTextType_Equals(type, (Obj*)other),
"Equals() false with different Analyzer");
TEST_FALSE(runner, FullTextType_Equals(type, (Obj*)not_indexed),
"Equals() false with indexed => false");
TEST_FALSE(runner, FullTextType_Equals(type, (Obj*)not_stored),
"Equals() false with stored => false");
TEST_FALSE(runner, FullTextType_Equals(type, (Obj*)highlightable),
"Equals() false with highlightable => true");
TEST_TRUE(runner, FullTextType_Equals(type, (Obj*)clone),
"Dump => Load round trip");
TEST_TRUE(runner, FullTextType_Equals(type, (Obj*)another_clone),
"Dump_For_Schema => Load round trip");
DECREF(another_clone);
DECREF(dump);
DECREF(clone);
DECREF(another_dump);
DECREF(highlightable);
DECREF(not_stored);
DECREF(not_indexed);
DECREF(boost_differs);
DECREF(other);
DECREF(type);
DECREF(normalizer);
DECREF(tokenizer);
}
PolyAnalyzer*
PolyAnalyzer_Load_IMP(PolyAnalyzer *self, Obj *dump) {
Hash *source = (Hash*)CERTIFY(dump, HASH);
PolyAnalyzer_Load_t super_load
= SUPER_METHOD_PTR(POLYANALYZER, LUCY_PolyAnalyzer_Load);
PolyAnalyzer *loaded = super_load(self, dump);
VArray *analyzer_dumps
= (VArray*)CERTIFY(Hash_Fetch_Utf8(source, "analyzers", 9), VARRAY);
VArray *analyzers
= (VArray*)CERTIFY(Freezer_load((Obj*)analyzer_dumps), VARRAY);
PolyAnalyzer_init(loaded, NULL, analyzers);
DECREF(analyzers);
return loaded;
}
static Obj*
S_load_from_hash(Hash *dump) {
String *class_name = (String*)Hash_Fetch_Utf8(dump, "_class", 6);
// Assume that the presence of the "_class" key paired with a valid class
// name indicates the output of a dump() rather than an ordinary Hash.
if (class_name && Str_is_a(class_name, STRING)) {
Class *klass = Class_fetch_class(class_name);
if (!klass) {
String *parent_class_name = Class_find_parent_class(class_name);
if (parent_class_name) {
Class *parent = Class_singleton(parent_class_name, NULL);
klass = Class_singleton(class_name, parent);
DECREF(parent_class_name);
}
else {
// TODO: Fix load() so that it works with ordinary hash keys
// named "_class".
THROW(ERR, "Can't find class '%o'", class_name);
}
}
// Dispatch to an alternate Load() method.
if (klass) {
return S_load_via_load_method(klass, (Obj*)dump);
}
}
// It's an ordinary Hash.
Hash *loaded = Hash_new(Hash_Get_Size(dump));
HashIterator *iter = HashIter_new(dump);
while (HashIter_Next(iter)) {
String *key = HashIter_Get_Key(iter);
Obj *value = HashIter_Get_Value(iter);
Hash_Store(loaded, key, Freezer_load(value));
}
DECREF(iter);
return (Obj*)loaded;
}
FullTextType*
FullTextType_Load_IMP(FullTextType *self, Obj *dump) {
UNUSED_VAR(self);
Hash *source = (Hash*)CERTIFY(dump, HASH);
String *class_name = (String*)Hash_Fetch_Utf8(source, "_class", 6);
Class *klass
= (class_name != NULL && Obj_is_a((Obj*)class_name, STRING))
? Class_singleton(class_name, NULL)
: FULLTEXTTYPE;
FullTextType *loaded = (FullTextType*)Class_Make_Obj(klass);
// Extract boost.
Obj *boost_dump = Hash_Fetch_Utf8(source, "boost", 5);
float boost = boost_dump ? (float)Json_obj_to_f64(boost_dump) : 1.0f;
// Find boolean properties.
Obj *indexed_dump = Hash_Fetch_Utf8(source, "indexed", 7);
Obj *stored_dump = Hash_Fetch_Utf8(source, "stored", 6);
Obj *sort_dump = Hash_Fetch_Utf8(source, "sortable", 8);
Obj *hl_dump = Hash_Fetch_Utf8(source, "highlightable", 13);
bool indexed = indexed_dump ? Json_obj_to_bool(indexed_dump) : true;
bool stored = stored_dump ? Json_obj_to_bool(stored_dump) : true;
bool sortable = sort_dump ? Json_obj_to_bool(sort_dump) : false;
bool hl = hl_dump ? Json_obj_to_bool(hl_dump) : false;
// Extract an Analyzer.
Obj *analyzer_dump = Hash_Fetch_Utf8(source, "analyzer", 8);
Analyzer *analyzer = NULL;
if (analyzer_dump) {
if (Obj_is_a(analyzer_dump, ANALYZER)) {
// Schema munged the dump and installed a shared analyzer.
analyzer = (Analyzer*)INCREF(analyzer_dump);
}
else if (Obj_is_a((Obj*)analyzer_dump, HASH)) {
analyzer = (Analyzer*)Freezer_load(analyzer_dump);
}
}
CERTIFY(analyzer, ANALYZER);
FullTextType_init2(loaded, analyzer, boost, indexed, stored,
sortable, hl);
DECREF(analyzer);
return loaded;
}
static void
test_Dump_Load_and_Equals(TestBatchRunner *runner) {
Int32Type *i32 = Int32Type_new();
Int64Type *i64 = Int64Type_new();
Float32Type *f32 = Float32Type_new();
Float64Type *f64 = Float64Type_new();
TEST_FALSE(runner, Int32Type_Equals(i32, (Obj*)i64),
"Int32Type_Equals() false for different type");
TEST_FALSE(runner, Int32Type_Equals(i32, NULL),
"Int32Type_Equals() false for NULL");
TEST_FALSE(runner, Int64Type_Equals(i64, (Obj*)i32),
"Int64Type_Equals() false for different type");
TEST_FALSE(runner, Int64Type_Equals(i64, NULL),
"Int64Type_Equals() false for NULL");
TEST_FALSE(runner, Float32Type_Equals(f32, (Obj*)f64),
"Float32Type_Equals() false for different type");
TEST_FALSE(runner, Float32Type_Equals(f32, NULL),
"Float32Type_Equals() false for NULL");
TEST_FALSE(runner, Float64Type_Equals(f64, (Obj*)f32),
"Float64Type_Equals() false for different type");
TEST_FALSE(runner, Float64Type_Equals(f64, NULL),
"Float64Type_Equals() false for NULL");
{
Obj *dump = (Obj*)Int32Type_Dump(i32);
Obj *other = Freezer_load(dump);
TEST_TRUE(runner, Int32Type_Equals(i32, other),
"Dump => Load round trip for Int32Type");
DECREF(dump);
DECREF(other);
}
{
Obj *dump = (Obj*)Int64Type_Dump(i64);
Obj *other = Freezer_load(dump);
TEST_TRUE(runner, Int64Type_Equals(i64, other),
"Dump => Load round trip for Int64Type");
DECREF(dump);
DECREF(other);
}
{
Obj *dump = (Obj*)Float32Type_Dump(f32);
Obj *other = Freezer_load(dump);
TEST_TRUE(runner, Float32Type_Equals(f32, other),
"Dump => Load round trip for Float32Type");
DECREF(dump);
DECREF(other);
}
{
Obj *dump = (Obj*)Float64Type_Dump(f64);
Obj *other = Freezer_load(dump);
TEST_TRUE(runner, Float64Type_Equals(f64, other),
"Dump => Load round trip for Float64Type");
DECREF(dump);
DECREF(other);
}
DECREF(i32);
DECREF(i64);
DECREF(f32);
DECREF(f64);
}
BackgroundMerger*
BGMerger_init(BackgroundMerger *self, Obj *index, IndexManager *manager) {
BackgroundMergerIVARS *const ivars = BGMerger_IVARS(self);
Folder *folder = S_init_folder(index);
// Init.
ivars->optimize = false;
ivars->prepared = false;
ivars->needs_commit = false;
ivars->snapfile = NULL;
ivars->doc_maps = Hash_new(0);
// Assign.
ivars->folder = folder;
if (manager) {
ivars->manager = (IndexManager*)INCREF(manager);
}
else {
ivars->manager = IxManager_new(NULL, NULL);
IxManager_Set_Write_Lock_Timeout(ivars->manager, 10000);
}
IxManager_Set_Folder(ivars->manager, folder);
// Obtain write lock (which we'll only hold briefly), then merge lock.
S_obtain_write_lock(self);
if (!ivars->write_lock) {
DECREF(self);
RETHROW(INCREF(Err_get_error()));
}
S_obtain_merge_lock(self);
if (!ivars->merge_lock) {
DECREF(self);
RETHROW(INCREF(Err_get_error()));
}
// Find the latest snapshot. If there's no index content, bail early.
ivars->snapshot = Snapshot_Read_File(Snapshot_new(), folder, NULL);
if (!Snapshot_Get_Path(ivars->snapshot)) {
S_release_write_lock(self);
S_release_merge_lock(self);
return self;
}
// Create FilePurger. Zap detritus from previous sessions.
ivars->file_purger = FilePurger_new(folder, ivars->snapshot, ivars->manager);
FilePurger_Purge(ivars->file_purger);
// Open a PolyReader, passing in the IndexManager so we get a read lock on
// the Snapshot's files -- so that Indexers don't zap our files while
// we're operating in the background.
ivars->polyreader = PolyReader_open((Obj*)folder, NULL, ivars->manager);
// Clone the PolyReader's schema.
Obj *dump = (Obj*)Schema_Dump(PolyReader_Get_Schema(ivars->polyreader));
ivars->schema = (Schema*)CERTIFY(Freezer_load(dump), SCHEMA);
DECREF(dump);
// Create new Segment.
int64_t new_seg_num
= IxManager_Highest_Seg_Num(ivars->manager, ivars->snapshot) + 1;
Vector *fields = Schema_All_Fields(ivars->schema);
ivars->segment = Seg_new(new_seg_num);
for (uint32_t i = 0, max = Vec_Get_Size(fields); i < max; i++) {
Seg_Add_Field(ivars->segment, (String*)Vec_Fetch(fields, i));
}
DECREF(fields);
// Our "cutoff" is the segment this BackgroundMerger will write. Now that
// we've determined the cutoff, write the merge data file.
ivars->cutoff = Seg_Get_Number(ivars->segment);
IxManager_Write_Merge_Data(ivars->manager, ivars->cutoff);
/* Create the SegWriter but hold off on preparing the new segment
* directory -- because if we don't need to merge any segments we don't
* need it. (We've reserved the dir by plopping down the merge.json
* file.) */
ivars->seg_writer = SegWriter_new(ivars->schema, ivars->snapshot,
ivars->segment, ivars->polyreader);
// Grab a local ref to the DeletionsWriter.
ivars->del_writer
= (DeletionsWriter*)INCREF(SegWriter_Get_Del_Writer(ivars->seg_writer));
// Release the write lock. Now new Indexers can start while we work in
// the background.
S_release_write_lock(self);
return self;
}
void
S_try_open_elements(void *context) {
struct try_open_elements_context *args
= (struct try_open_elements_context*)context;
PolyReader *self = args->self;
PolyReaderIVARS *const ivars = PolyReader_IVARS(self);
VArray *files = Snapshot_List(ivars->snapshot);
Folder *folder = PolyReader_Get_Folder(self);
uint32_t num_segs = 0;
uint64_t latest_schema_gen = 0;
String *schema_file = NULL;
// Find schema file, count segments.
for (uint32_t i = 0, max = VA_Get_Size(files); i < max; i++) {
String *entry = (String*)VA_Fetch(files, i);
if (Seg_valid_seg_name(entry)) {
num_segs++;
}
else if (Str_Starts_With_Utf8(entry, "schema_", 7)
&& Str_Ends_With_Utf8(entry, ".json", 5)
) {
uint64_t gen = IxFileNames_extract_gen(entry);
if (gen > latest_schema_gen) {
latest_schema_gen = gen;
schema_file = entry;
}
}
}
// Read Schema.
if (!schema_file) {
DECREF(files);
THROW(ERR, "Can't find a schema file.");
}
else {
Obj *dump = Json_slurp_json(folder, schema_file);
if (dump) { // read file successfully
DECREF(ivars->schema);
ivars->schema = (Schema*)CERTIFY(Freezer_load(dump), SCHEMA);
DECREF(dump);
schema_file = NULL;
}
else {
String *mess = MAKE_MESS("Failed to parse %o", schema_file);
DECREF(files);
Err_throw_mess(ERR, mess);
}
}
VArray *segments = VA_new(num_segs);
for (uint32_t i = 0, max = VA_Get_Size(files); i < max; i++) {
String *entry = (String*)VA_Fetch(files, i);
// Create a Segment for each segmeta.
if (Seg_valid_seg_name(entry)) {
int64_t seg_num = IxFileNames_extract_gen(entry);
Segment *segment = Seg_new(seg_num);
// Bail if reading the file fails (probably because it's been
// deleted and a new snapshot file has been written so we need to
// retry).
if (Seg_Read_File(segment, folder)) {
VA_Push(segments, (Obj*)segment);
}
else {
String *mess = MAKE_MESS("Failed to read %o", entry);
DECREF(segment);
DECREF(segments);
DECREF(files);
Err_throw_mess(ERR, mess);
}
}
}
// Sort the segments by age.
VA_Sort(segments, NULL, NULL);
// Open individual SegReaders.
struct try_open_segreader_context seg_context;
seg_context.schema = PolyReader_Get_Schema(self);
seg_context.folder = folder;
seg_context.snapshot = PolyReader_Get_Snapshot(self);
seg_context.segments = segments;
seg_context.result = NULL;
args->seg_readers = VA_new(num_segs);
Err *error = NULL;
for (uint32_t seg_tick = 0; seg_tick < num_segs; seg_tick++) {
seg_context.seg_tick = seg_tick;
error = Err_trap(S_try_open_segreader, &seg_context);
if (error) {
break;
}
VA_Push(args->seg_readers, (Obj*)seg_context.result);
seg_context.result = NULL;
}
DECREF(segments);
DECREF(files);
if (error) {
DECREF(args->seg_readers);
args->seg_readers = NULL;
RETHROW(error);
}
}
Schema*
Schema_Load_IMP(Schema *self, Obj *dump) {
Hash *source = (Hash*)CERTIFY(dump, HASH);
String *class_name
= (String*)CERTIFY(Hash_Fetch_Utf8(source, "_class", 6), STRING);
Class *klass = Class_singleton(class_name, NULL);
Schema *loaded = (Schema*)Class_Make_Obj(klass);
Hash *type_dumps
= (Hash*)CERTIFY(Hash_Fetch_Utf8(source, "fields", 6), HASH);
Vector *analyzer_dumps
= (Vector*)CERTIFY(Hash_Fetch_Utf8(source, "analyzers", 9), VECTOR);
Vector *analyzers
= (Vector*)Freezer_load((Obj*)analyzer_dumps);
UNUSED_VAR(self);
// Start with a blank Schema.
Schema_init(loaded);
SchemaIVARS *const loaded_ivars = Schema_IVARS(loaded);
Vec_Grow(loaded_ivars->uniq_analyzers, Vec_Get_Size(analyzers));
HashIterator *iter = HashIter_new(type_dumps);
while (HashIter_Next(iter)) {
String *field = HashIter_Get_Key(iter);
Hash *type_dump = (Hash*)CERTIFY(HashIter_Get_Value(iter), HASH);
String *type_str = (String*)Hash_Fetch_Utf8(type_dump, "type", 4);
if (type_str) {
if (Str_Equals_Utf8(type_str, "fulltext", 8)) {
// Replace the "analyzer" tick with the real thing.
Obj *tick
= CERTIFY(Hash_Fetch_Utf8(type_dump, "analyzer", 8), OBJ);
Analyzer *analyzer
= (Analyzer*)Vec_Fetch(analyzers,
(uint32_t)Json_obj_to_i64(tick));
if (!analyzer) {
THROW(ERR, "Can't find analyzer for '%o'", field);
}
Hash_Store_Utf8(type_dump, "analyzer", 8, INCREF(analyzer));
FullTextType *type
= (FullTextType*)S_load_type(FULLTEXTTYPE,
(Obj*)type_dump);
Schema_Spec_Field(loaded, field, (FieldType*)type);
DECREF(type);
}
else if (Str_Equals_Utf8(type_str, "string", 6)) {
StringType *type
= (StringType*)S_load_type(STRINGTYPE, (Obj*)type_dump);
Schema_Spec_Field(loaded, field, (FieldType*)type);
DECREF(type);
}
else if (Str_Equals_Utf8(type_str, "blob", 4)) {
BlobType *type
= (BlobType*)S_load_type(BLOBTYPE, (Obj*)type_dump);
Schema_Spec_Field(loaded, field, (FieldType*)type);
DECREF(type);
}
else if (Str_Equals_Utf8(type_str, "i32_t", 5)) {
Int32Type *type
= (Int32Type*)S_load_type(INT32TYPE, (Obj*)type_dump);
Schema_Spec_Field(loaded, field, (FieldType*)type);
DECREF(type);
}
else if (Str_Equals_Utf8(type_str, "i64_t", 5)) {
Int64Type *type
= (Int64Type*)S_load_type(INT64TYPE, (Obj*)type_dump);
Schema_Spec_Field(loaded, field, (FieldType*)type);
DECREF(type);
}
else if (Str_Equals_Utf8(type_str, "f32_t", 5)) {
Float32Type *type
= (Float32Type*)S_load_type(FLOAT32TYPE, (Obj*)type_dump);
Schema_Spec_Field(loaded, field, (FieldType*)type);
DECREF(type);
}
else if (Str_Equals_Utf8(type_str, "f64_t", 5)) {
Float64Type *type
= (Float64Type*)S_load_type(FLOAT64TYPE, (Obj*)type_dump);
Schema_Spec_Field(loaded, field, (FieldType*)type);
DECREF(type);
}
else {
THROW(ERR, "Unknown type '%o' for field '%o'", type_str, field);
}
}
else {
FieldType *type
= (FieldType*)CERTIFY(Freezer_load((Obj*)type_dump),
FIELDTYPE);
Schema_Spec_Field(loaded, field, type);
DECREF(type);
}
}
DECREF(iter);
DECREF(analyzers);
return loaded;
}
Indexer*
Indexer_init(Indexer *self, Schema *schema, Obj *index,
IndexManager *manager, int32_t flags) {
IndexerIVARS *const ivars = Indexer_IVARS(self);
bool create = (flags & Indexer_CREATE) ? true : false;
bool truncate = (flags & Indexer_TRUNCATE) ? true : false;
Folder *folder = S_init_folder(index, create);
Snapshot *latest_snapshot = Snapshot_new();
// Init.
ivars->stock_doc = Doc_new(NULL, 0);
ivars->truncate = false;
ivars->optimize = false;
ivars->prepared = false;
ivars->needs_commit = false;
ivars->snapfile = NULL;
ivars->merge_lock = NULL;
// Assign.
ivars->folder = folder;
ivars->manager = manager
? (IndexManager*)INCREF(manager)
: IxManager_new(NULL, NULL);
IxManager_Set_Folder(ivars->manager, folder);
// Get a write lock for this folder.
Lock *write_lock = IxManager_Make_Write_Lock(ivars->manager);
Lock_Clear_Stale(write_lock);
if (Lock_Obtain(write_lock)) {
// Only assign if successful, otherwise DESTROY unlocks -- bad!
ivars->write_lock = write_lock;
}
else {
DECREF(write_lock);
DECREF(self);
RETHROW(INCREF(Err_get_error()));
}
// Find the latest snapshot or create a new one.
String *latest_snapfile = IxFileNames_latest_snapshot(folder);
if (latest_snapfile) {
Snapshot_Read_File(latest_snapshot, folder, latest_snapfile);
}
// Look for an existing Schema if one wasn't supplied.
if (schema) {
ivars->schema = (Schema*)INCREF(schema);
}
else {
if (!latest_snapfile) {
S_release_write_lock(self);
THROW(ERR, "No Schema supplied, and can't find one in the index");
}
else {
String *schema_file = S_find_schema_file(latest_snapshot);
Obj *dump = Json_slurp_json(folder, schema_file);
if (dump) { // read file successfully
ivars->schema = (Schema*)CERTIFY(Freezer_load(dump), SCHEMA);
schema = ivars->schema;
DECREF(dump);
schema_file = NULL;
}
else {
THROW(ERR, "Failed to parse %o", schema_file);
}
}
}
// If we're clobbering, start with an empty Snapshot and an empty
// PolyReader. Otherwise, start with the most recent Snapshot and an
// up-to-date PolyReader.
if (truncate) {
ivars->snapshot = Snapshot_new();
ivars->polyreader = PolyReader_new(schema, folder, NULL, NULL, NULL);
ivars->truncate = true;
}
else {
// TODO: clone most recent snapshot rather than read it twice.
ivars->snapshot = (Snapshot*)INCREF(latest_snapshot);
ivars->polyreader = latest_snapfile
? PolyReader_open((Obj*)folder, NULL, NULL)
: PolyReader_new(schema, folder, NULL, NULL, NULL);
if (latest_snapfile) {
// Make sure than any existing fields which may have been
// dynamically added during past indexing sessions get added.
Schema *old_schema = PolyReader_Get_Schema(ivars->polyreader);
Schema_Eat(schema, old_schema);
}
}
// Zap detritus from previous sessions.
// Note: we have to feed FilePurger with the most recent snapshot file
// now, but with the Indexer's snapshot later.
FilePurger *file_purger
= FilePurger_new(folder, latest_snapshot, ivars->manager);
FilePurger_Purge(file_purger);
DECREF(file_purger);
// Create a new segment.
int64_t new_seg_num
= IxManager_Highest_Seg_Num(ivars->manager, latest_snapshot) + 1;
Lock *merge_lock = IxManager_Make_Merge_Lock(ivars->manager);
if (Lock_Is_Locked(merge_lock)) {
// If there's a background merge process going on, stay out of its
// way.
Hash *merge_data = IxManager_Read_Merge_Data(ivars->manager);
Obj *cutoff_obj = merge_data
? Hash_Fetch_Utf8(merge_data, "cutoff", 6)
: NULL;
if (!cutoff_obj) {
DECREF(merge_lock);
DECREF(merge_data);
THROW(ERR, "Background merge detected, but can't read merge data");
}
else {
int64_t cutoff = Json_obj_to_i64(cutoff_obj);
if (cutoff >= new_seg_num) {
new_seg_num = cutoff + 1;
}
}
DECREF(merge_data);
}
ivars->segment = Seg_new(new_seg_num);
// Add all known fields to Segment.
Vector *fields = Schema_All_Fields(schema);
for (size_t i = 0, max = Vec_Get_Size(fields); i < max; i++) {
Seg_Add_Field(ivars->segment, (String*)Vec_Fetch(fields, i));
}
DECREF(fields);
DECREF(merge_lock);
// Create new SegWriter and FilePurger.
ivars->file_purger
= FilePurger_new(folder, ivars->snapshot, ivars->manager);
ivars->seg_writer = SegWriter_new(ivars->schema, ivars->snapshot,
ivars->segment, ivars->polyreader);
SegWriter_Prep_Seg_Dir(ivars->seg_writer);
// Grab a local ref to the DeletionsWriter.
ivars->del_writer = (DeletionsWriter*)INCREF(
SegWriter_Get_Del_Writer(ivars->seg_writer));
DECREF(latest_snapfile);
DECREF(latest_snapshot);
return self;
}
