void LexWriter_finish_field(LexiconWriter *self, int32_t field_num) { CharBuf *field = Seg_Field_Name(self->segment, field_num); // Store count of terms for this field as metadata. Hash_Store(self->counts, (Obj*)field, (Obj*)CB_newf("%i32", self->count)); Hash_Store(self->ix_counts, (Obj*)field, (Obj*)CB_newf("%i32", self->ix_count)); // Close streams. OutStream_Close(self->dat_out); OutStream_Close(self->ix_out); OutStream_Close(self->ixix_out); DECREF(self->dat_out); DECREF(self->ix_out); DECREF(self->ixix_out); self->dat_out = NULL; self->ix_out = NULL; self->ixix_out = NULL; // Close term stepper. DECREF(self->term_stepper); self->term_stepper = NULL; }
Hash* Hash_deserialize(Hash *self, InStream *instream) { uint32_t size = InStream_Read_C32(instream); uint32_t num_charbufs = InStream_Read_C32(instream); uint32_t num_other = size - num_charbufs; CharBuf *key = num_charbufs ? CB_new(0) : NULL; Hash_init(self, size); // Read key-value pairs with CharBuf keys. while (num_charbufs--) { uint32_t len = InStream_Read_C32(instream); char *key_buf = CB_Grow(key, len); InStream_Read_Bytes(instream, key_buf, len); key_buf[len] = '\0'; CB_Set_Size(key, len); Hash_Store(self, (Obj*)key, THAW(instream)); } DECREF(key); // Read remaining key/value pairs. while (num_other--) { Obj *k = THAW(instream); Hash_Store(self, k, THAW(instream)); DECREF(k); } return self; }
static void S_add_string_field(Schema *self, String *field, FieldType *type) { SchemaIVARS *const ivars = Schema_IVARS(self); StringType *string_type = (StringType*)CERTIFY(type, STRINGTYPE); Similarity *sim = StringType_Make_Similarity(string_type); // Cache helpers. Hash_Store(ivars->sims, field, (Obj*)sim); // Store FieldType. Hash_Store(ivars->types, field, INCREF(type)); }
static void S_init_sub_readers(PolyReader *self, VArray *sub_readers) { PolyReaderIVARS *const ivars = PolyReader_IVARS(self); uint32_t num_sub_readers = VA_Get_Size(sub_readers); int32_t *starts = (int32_t*)MALLOCATE(num_sub_readers * sizeof(int32_t)); Hash *data_readers = Hash_new(0); DECREF(ivars->sub_readers); DECREF(ivars->offsets); ivars->sub_readers = (VArray*)INCREF(sub_readers); // Accumulate doc_max, subreader start offsets, and DataReaders. ivars->doc_max = 0; for (uint32_t i = 0; i < num_sub_readers; i++) { SegReader *seg_reader = (SegReader*)VA_Fetch(sub_readers, i); Hash *components = SegReader_Get_Components(seg_reader); CharBuf *api; DataReader *component; starts[i] = ivars->doc_max; ivars->doc_max += SegReader_Doc_Max(seg_reader); Hash_Iterate(components); while (Hash_Next(components, (Obj**)&api, (Obj**)&component)) { VArray *readers = (VArray*)Hash_Fetch(data_readers, (Obj*)api); if (!readers) { readers = VA_new(num_sub_readers); Hash_Store(data_readers, (Obj*)api, (Obj*)readers); } VA_Store(readers, i, INCREF(component)); } } ivars->offsets = I32Arr_new_steal(starts, num_sub_readers); CharBuf *api; VArray *readers; Hash_Iterate(data_readers); while (Hash_Next(data_readers, (Obj**)&api, (Obj**)&readers)) { DataReader *datareader = (DataReader*)CERTIFY(S_first_non_null(readers), DATAREADER); DataReader *aggregator = DataReader_Aggregator(datareader, readers, ivars->offsets); if (aggregator) { CERTIFY(aggregator, DATAREADER); Hash_Store(ivars->components, (Obj*)api, (Obj*)aggregator); } } DECREF(data_readers); DeletionsReader *del_reader = (DeletionsReader*)Hash_Fetch( ivars->components, (Obj*)VTable_Get_Name(DELETIONSREADER)); ivars->del_count = del_reader ? DelReader_Del_Count(del_reader) : 0; }
static void S_add_text_field(Schema *self, String *field, FieldType *type) { SchemaIVARS *const ivars = Schema_IVARS(self); FullTextType *fttype = (FullTextType*)CERTIFY(type, FULLTEXTTYPE); Similarity *sim = FullTextType_Make_Similarity(fttype); Analyzer *analyzer = FullTextType_Get_Analyzer(fttype); // Cache helpers. Hash_Store(ivars->sims, field, (Obj*)sim); Hash_Store(ivars->analyzers, field, INCREF(analyzer)); S_add_unique(ivars->uniq_analyzers, (Obj*)analyzer); // Store FieldType. Hash_Store(ivars->types, field, INCREF(type)); }
Hash* DefDelWriter_Metadata_IMP(DefaultDeletionsWriter *self) { DefaultDeletionsWriterIVARS *const ivars = DefDelWriter_IVARS(self); DefDelWriter_Metadata_t super_meta = (DefDelWriter_Metadata_t)SUPER_METHOD_PTR(DEFAULTDELETIONSWRITER, LUCY_DefDelWriter_Metadata); Hash *const metadata = super_meta(self); Hash *const files = Hash_new(0); for (uint32_t i = 0, max = VA_Get_Size(ivars->seg_readers); i < max; i++) { SegReader *seg_reader = (SegReader*)VA_Fetch(ivars->seg_readers, i); if (ivars->updated[i]) { BitVector *deldocs = (BitVector*)VA_Fetch(ivars->bit_vecs, i); Segment *segment = SegReader_Get_Segment(seg_reader); Hash *mini_meta = Hash_new(2); Hash_Store_Utf8(mini_meta, "count", 5, (Obj*)Str_newf("%u32", (uint32_t)BitVec_Count(deldocs))); Hash_Store_Utf8(mini_meta, "filename", 8, (Obj*)S_del_filename(self, seg_reader)); Hash_Store(files, (Obj*)Seg_Get_Name(segment), (Obj*)mini_meta); } } Hash_Store_Utf8(metadata, "files", 5, (Obj*)files); return metadata; }
void Seg_store_metadata(Segment *self, const CharBuf *key, Obj *value) { if (Hash_Fetch(self->metadata, (Obj*)key)) { THROW(ERR, "Metadata key '%o' already registered", key); } Hash_Store(self->metadata, (Obj*)key, value); }
TermVector* DocVec_Term_Vector_IMP(DocVector *self, String *field, String *term_text) { DocVectorIVARS *const ivars = DocVec_IVARS(self); Hash *field_vector = (Hash*)Hash_Fetch(ivars->field_vectors, (Obj*)field); // If no cache hit, try to fill cache. if (field_vector == NULL) { ByteBuf *field_buf = (ByteBuf*)Hash_Fetch(ivars->field_bufs, (Obj*)field); // Bail if there's no content or the field isn't highlightable. if (field_buf == NULL) { return NULL; } field_vector = S_extract_tv_cache(field_buf); Hash_Store(ivars->field_vectors, (Obj*)field, (Obj*)field_vector); } // Get a buf for the term text or bail. ByteBuf *tv_buf = (ByteBuf*)Hash_Fetch(field_vector, (Obj*)term_text); if (tv_buf == NULL) { return NULL; } return S_extract_tv_from_tv_buf(field, term_text, tv_buf); }
TermVector* DocVec_term_vector(DocVector *self, const CharBuf *field, const CharBuf *term_text) { Hash *field_vector = (Hash*)Hash_Fetch(self->field_vectors, (Obj*)field); // If no cache hit, try to fill cache. if (field_vector == NULL) { ByteBuf *field_buf = (ByteBuf*)Hash_Fetch(self->field_bufs, (Obj*)field); // Bail if there's no content or the field isn't highlightable. if (field_buf == NULL) { return NULL; } field_vector = S_extract_tv_cache(field_buf); Hash_Store(self->field_vectors, (Obj*)field, (Obj*)field_vector); } // Get a buf for the term text or bail. ByteBuf *tv_buf = (ByteBuf*)Hash_Fetch(field_vector, (Obj*)term_text); if (tv_buf == NULL) { return NULL; } return S_extract_tv_from_tv_buf(field, term_text, tv_buf); }
void Folder_consolidate(Folder *self, const CharBuf *path) { Folder *folder = Folder_Find_Folder(self, path); Folder *enclosing_folder = Folder_Enclosing_Folder(self, path); if (!folder) { THROW(ERR, "Can't consolidate %o", path); } else if (Folder_Is_A(folder, COMPOUNDFILEREADER)) { THROW(ERR, "Can't consolidate %o twice", path); } else { CompoundFileWriter *cf_writer = CFWriter_new(folder); CFWriter_Consolidate(cf_writer); DECREF(cf_writer); if (CB_Get_Size(path)) { ZombieCharBuf *name = IxFileNames_local_part(path, ZCB_BLANK()); CompoundFileReader *cf_reader = CFReader_open(folder); if (!cf_reader) { RETHROW(INCREF(Err_get_error())); } Hash_Store(enclosing_folder->entries, (Obj*)name, (Obj*)cf_reader); } } }
Hash* SnowStop_gen_stoplist(String *language) { char lang[2]; lang[0] = tolower(Str_Code_Point_At(language, 0)); lang[1] = tolower(Str_Code_Point_At(language, 1)); const uint8_t **words = NULL; if (memcmp(lang, "da", 2) == 0) { words = SnowStop_snow_da; } else if (memcmp(lang, "de", 2) == 0) { words = SnowStop_snow_de; } else if (memcmp(lang, "en", 2) == 0) { words = SnowStop_snow_en; } else if (memcmp(lang, "es", 2) == 0) { words = SnowStop_snow_es; } else if (memcmp(lang, "fi", 2) == 0) { words = SnowStop_snow_fi; } else if (memcmp(lang, "fr", 2) == 0) { words = SnowStop_snow_fr; } else if (memcmp(lang, "hu", 2) == 0) { words = SnowStop_snow_hu; } else if (memcmp(lang, "it", 2) == 0) { words = SnowStop_snow_it; } else if (memcmp(lang, "nl", 2) == 0) { words = SnowStop_snow_nl; } else if (memcmp(lang, "no", 2) == 0) { words = SnowStop_snow_no; } else if (memcmp(lang, "pt", 2) == 0) { words = SnowStop_snow_pt; } else if (memcmp(lang, "ru", 2) == 0) { words = SnowStop_snow_ru; } else if (memcmp(lang, "sv", 2) == 0) { words = SnowStop_snow_sv; } else { return NULL; } size_t num_stopwords = 0; for (uint32_t i = 0; words[i] != NULL; i++) { num_stopwords++; } Hash *stoplist = Hash_new(num_stopwords); for (uint32_t i = 0; words[i] != NULL; i++) { char *word = (char*)words[i]; String *stop = Str_new_wrap_trusted_utf8(word, strlen(word)); Hash_Store(stoplist, stop, (Obj*)CFISH_TRUE); DECREF(stop); } return (Hash*)stoplist; }
Hash* Schema_Dump_IMP(Schema *self) { SchemaIVARS *const ivars = Schema_IVARS(self); Hash *dump = Hash_new(0); Hash *type_dumps = Hash_new(Hash_Get_Size(ivars->types)); // Record class name, store dumps of unique Analyzers. Hash_Store_Utf8(dump, "_class", 6, (Obj*)Str_Clone(Schema_get_class_name(self))); Hash_Store_Utf8(dump, "analyzers", 9, Freezer_dump((Obj*)ivars->uniq_analyzers)); // Dump FieldTypes. Hash_Store_Utf8(dump, "fields", 6, (Obj*)type_dumps); HashIterator *iter = HashIter_new(ivars->types); while (HashIter_Next(iter)) { String *field = HashIter_Get_Key(iter); FieldType *type = (FieldType*)HashIter_Get_Value(iter); Class *type_class = FType_get_class(type); // Dump known types to simplified format. if (type_class == FULLTEXTTYPE) { FullTextType *fttype = (FullTextType*)type; Hash *type_dump = FullTextType_Dump_For_Schema(fttype); Analyzer *analyzer = FullTextType_Get_Analyzer(fttype); uint32_t tick = S_find_in_array(ivars->uniq_analyzers, (Obj*)analyzer); // Store the tick which references a unique analyzer. Hash_Store_Utf8(type_dump, "analyzer", 8, (Obj*)Str_newf("%u32", tick)); Hash_Store(type_dumps, field, (Obj*)type_dump); } else if (type_class == STRINGTYPE || type_class == BLOBTYPE) { Hash *type_dump = FType_Dump_For_Schema(type); Hash_Store(type_dumps, field, (Obj*)type_dump); } // Unknown FieldType type, so punt. else { Hash_Store(type_dumps, field, FType_Dump(type)); } } DECREF(iter); return dump; }
void Seg_Store_Metadata_IMP(Segment *self, String *key, Obj *value) { SegmentIVARS *const ivars = Seg_IVARS(self); if (Hash_Fetch(ivars->metadata, key)) { THROW(ERR, "Metadata key '%o' already registered", key); } Hash_Store(ivars->metadata, key, value); }
void SegReader_register(SegReader *self, const CharBuf *api, DataReader *component) { if (Hash_Fetch(self->components, api)) { THROW("Interface '%o' already registered"); } ASSERT_IS_A(component, DATAREADER); Hash_Store(self->components, api, (Obj*)component); }
static void S_zap_dead_merge(FilePurger *self, Hash *candidates) { FilePurgerIVARS *const ivars = FilePurger_IVARS(self); IndexManager *manager = ivars->manager; Lock *merge_lock = IxManager_Make_Merge_Lock(manager); Lock_Clear_Stale(merge_lock); if (!Lock_Is_Locked(merge_lock)) { Hash *merge_data = IxManager_Read_Merge_Data(manager); Obj *cutoff = merge_data ? Hash_Fetch_Utf8(merge_data, "cutoff", 6) : NULL; if (cutoff) { String *cutoff_seg = Seg_num_to_name(Json_obj_to_i64(cutoff)); if (Folder_Exists(ivars->folder, cutoff_seg)) { String *merge_json = SSTR_WRAP_UTF8("merge.json", 10); DirHandle *dh = Folder_Open_Dir(ivars->folder, cutoff_seg); if (!dh) { THROW(ERR, "Can't open segment dir '%o'", cutoff_seg); } Hash_Store(candidates, cutoff_seg, (Obj*)CFISH_TRUE); Hash_Store(candidates, merge_json, (Obj*)CFISH_TRUE); while (DH_Next(dh)) { // TODO: recursively delete subdirs within seg dir. String *entry = DH_Get_Entry(dh); String *filepath = Str_newf("%o/%o", cutoff_seg, entry); Hash_Store(candidates, filepath, (Obj*)CFISH_TRUE); DECREF(filepath); DECREF(entry); } DECREF(dh); } DECREF(cutoff_seg); } DECREF(merge_data); } DECREF(merge_lock); return; }
static void S_zap_dead_merge(FilePurger *self, Hash *candidates) { IndexManager *manager = self->manager; Lock *merge_lock = IxManager_Make_Merge_Lock(manager); Lock_Clear_Stale(merge_lock); if (!Lock_Is_Locked(merge_lock)) { Hash *merge_data = IxManager_Read_Merge_Data(manager); Obj *cutoff = merge_data ? Hash_Fetch_Str(merge_data, "cutoff", 6) : NULL; if (cutoff) { CharBuf *cutoff_seg = Seg_num_to_name(Obj_To_I64(cutoff)); if (Folder_Exists(self->folder, cutoff_seg)) { ZombieCharBuf *merge_json = ZCB_WRAP_STR("merge.json", 10); DirHandle *dh = Folder_Open_Dir(self->folder, cutoff_seg); CharBuf *entry = dh ? DH_Get_Entry(dh) : NULL; CharBuf *filepath = CB_new(32); if (!dh) { THROW(ERR, "Can't open segment dir '%o'", filepath); } Hash_Store(candidates, (Obj*)cutoff_seg, INCREF(&EMPTY)); Hash_Store(candidates, (Obj*)merge_json, INCREF(&EMPTY)); while (DH_Next(dh)) { // TODO: recursively delete subdirs within seg dir. CB_setf(filepath, "%o/%o", cutoff_seg, entry); Hash_Store(candidates, (Obj*)filepath, INCREF(&EMPTY)); } DECREF(filepath); DECREF(dh); } DECREF(cutoff_seg); } DECREF(merge_data); } DECREF(merge_lock); return; }
void SegReader_Register_IMP(SegReader *self, String *api, DataReader *component) { SegReaderIVARS *const ivars = SegReader_IVARS(self); if (Hash_Fetch(ivars->components, api)) { THROW(ERR, "Interface '%o' already registered"); } CERTIFY(component, DATAREADER); Hash_Store(ivars->components, api, (Obj*)component); }
void SegWriter_Register_IMP(SegWriter *self, String *api, DataWriter *component) { SegWriterIVARS *const ivars = SegWriter_IVARS(self); CERTIFY(component, DATAWRITER); if (Hash_Fetch(ivars->by_api, api)) { THROW(ERR, "API %o already registered", api); } Hash_Store(ivars->by_api, api, (Obj*)component); }
static Vector* S_find_all_referenced(Folder *folder, Vector *entries) { Hash *uniqued = Hash_new(Vec_Get_Size(entries)); for (uint32_t i = 0, max = Vec_Get_Size(entries); i < max; i++) { String *entry = (String*)Vec_Fetch(entries, i); Hash_Store(uniqued, entry, (Obj*)CFISH_TRUE); if (Folder_Is_Directory(folder, entry)) { Vector *contents = Folder_List_R(folder, entry); for (uint32_t j = Vec_Get_Size(contents); j--;) { String *sub_entry = (String*)Vec_Fetch(contents, j); Hash_Store(uniqued, sub_entry, (Obj*)CFISH_TRUE); } DECREF(contents); } } Vector *referenced = Hash_Keys(uniqued); DECREF(uniqued); return referenced; }
static void test_Keys_Values_Iter(TestBatch *batch) { Hash *hash = Hash_new(0); // trigger multiple rebuilds. VArray *expected = VA_new(100); VArray *keys; VArray *values; for (uint32_t i = 0; i < 500; i++) { CharBuf *cb = CB_newf("%u32", i); Hash_Store(hash, (Obj*)cb, (Obj*)cb); VA_Push(expected, INCREF(cb)); } VA_Sort(expected, NULL, NULL); keys = Hash_Keys(hash); values = Hash_Values(hash); VA_Sort(keys, NULL, NULL); VA_Sort(values, NULL, NULL); TEST_TRUE(batch, VA_Equals(keys, (Obj*)expected), "Keys"); TEST_TRUE(batch, VA_Equals(values, (Obj*)expected), "Values"); VA_Clear(keys); VA_Clear(values); { Obj *key; Obj *value; Hash_Iterate(hash); while (Hash_Next(hash, &key, &value)) { VA_Push(keys, INCREF(key)); VA_Push(values, INCREF(value)); } } VA_Sort(keys, NULL, NULL); VA_Sort(values, NULL, NULL); TEST_TRUE(batch, VA_Equals(keys, (Obj*)expected), "Keys from Iter"); TEST_TRUE(batch, VA_Equals(values, (Obj*)expected), "Values from Iter"); { ZombieCharBuf *forty = ZCB_WRAP_STR("40", 2); ZombieCharBuf *nope = ZCB_WRAP_STR("nope", 4); Obj *key = Hash_Find_Key(hash, (Obj*)forty, ZCB_Hash_Sum(forty)); TEST_TRUE(batch, Obj_Equals(key, (Obj*)forty), "Find_Key"); key = Hash_Find_Key(hash, (Obj*)nope, ZCB_Hash_Sum(nope)), TEST_TRUE(batch, key == NULL, "Find_Key returns NULL for non-existent key"); } DECREF(hash); DECREF(expected); DECREF(keys); DECREF(values); }
static VArray* S_find_all_referenced(Folder *folder, VArray *entries) { Hash *uniqued = Hash_new(VA_Get_Size(entries)); for (uint32_t i = 0, max = VA_Get_Size(entries); i < max; i++) { CharBuf *entry = (CharBuf*)VA_Fetch(entries, i); Hash_Store(uniqued, (Obj*)entry, INCREF(&EMPTY)); if (Folder_Is_Directory(folder, entry)) { VArray *contents = Folder_List_R(folder, entry); for (uint32_t j = VA_Get_Size(contents); j--; ) { CharBuf *sub_entry = (CharBuf*)VA_Fetch(contents, j); Hash_Store(uniqued, (Obj*)sub_entry, INCREF(&EMPTY)); } DECREF(contents); } } VArray *referenced = Hash_Keys(uniqued); DECREF(uniqued); return referenced; }
void RAMFolder_rename(RAMFolder *self, const CharBuf* from, const CharBuf *to) { RAMFileDes *file_des = (RAMFileDes*)Hash_Delete(self->elems, from); if (file_des == NULL) { THROW("File '%o' not loaded into RAM", from); } Hash_Store(self->elems, to, (Obj*)file_des); FileDes_Set_Path(file_des, to); }
OutStream* RAMFolder_open_out(RAMFolder *self, const CharBuf *filepath) { if (Hash_Fetch(self->elems, filepath)) { return NULL; } else { RAMFileDes *file_des = RAMFileDes_new(filepath); Hash_Store(self->elems, filepath, (Obj*)file_des); return OutStream_new((FileDes*)file_des); } }
int32_t Seg_add_field(Segment *self, const CharBuf *field) { Integer32 *num = (Integer32*)Hash_Fetch(self->by_name, (Obj*)field); if (num) { return Int32_Get_Value(num); } else { int32_t field_num = VA_Get_Size(self->by_num); Hash_Store(self->by_name, (Obj*)field, (Obj*)Int32_new(field_num)); VA_Push(self->by_num, (Obj*)CB_Clone(field)); return field_num; } }
Obj* Hash_load(Hash *self, Obj *dump) { Hash *source = (Hash*)CERTIFY(dump, HASH); CharBuf *class_name = (CharBuf*)Hash_Fetch_Str(source, "_class", 6); UNUSED_VAR(self); // Assume that the presence of the "_class" key paired with a valid class // name indicates the output of a Dump rather than an ordinary Hash. */ if (class_name && CB_Is_A(class_name, CHARBUF)) { VTable *vtable = VTable_fetch_vtable(class_name); if (!vtable) { CharBuf *parent_class = VTable_find_parent_class(class_name); if (parent_class) { VTable *parent = VTable_singleton(parent_class, NULL); vtable = VTable_singleton(class_name, parent); DECREF(parent_class); } else { // TODO: Fix Hash_Load() so that it works with ordinary hash // keys named "_class". THROW(ERR, "Can't find class '%o'", class_name); } } // Dispatch to an alternate Load() method. if (vtable) { Obj_Load_t load = METHOD_PTR(vtable, Lucy_Obj_Load); if (load == Obj_load) { THROW(ERR, "Abstract method Load() not defined for %o", VTable_Get_Name(vtable)); } else if (load != (Obj_Load_t)Hash_load) { // stop inf loop return VTable_Load_Obj(vtable, dump); } } } // It's an ordinary Hash. Hash *loaded = Hash_new(source->size); Obj *key; Obj *value; Hash_Iterate(source); while (Hash_Next(source, &key, &value)) { Hash_Store(loaded, key, Obj_Load(value, value)); } return (Obj*)loaded; }
Obj* S_dump_hash(Hash *hash) { Hash *dump = Hash_new(Hash_Get_Size(hash)); HashIterator *iter = HashIter_new(hash); while (HashIter_Next(iter)) { String *key = HashIter_Get_Key(iter); Obj *value = HashIter_Get_Value(iter); Hash_Store(dump, key, Freezer_dump(value)); } DECREF(iter); return (Obj*)dump; }
static void test_illegal_keys(TestBatch *batch) { Hash *hash = Hash_new(0); Float64 *key = Float64_new(1.1); Hash_Store(hash, (Obj*)key, (Obj*)CB_newf("blah")); Err_set_error(NULL); CharBuf *not_json = Json_to_json((Obj*)hash); TEST_TRUE(batch, not_json == NULL, "to_json returns NULL when fed an illegal key"); TEST_TRUE(batch, Err_get_error() != NULL, "to_json sets Err_error when fed an illegal key"); DECREF(key); DECREF(hash); }
int32_t Seg_Add_Field_IMP(Segment *self, String *field) { SegmentIVARS *const ivars = Seg_IVARS(self); Integer *num = (Integer*)Hash_Fetch(ivars->by_name, field); if (num) { return (int32_t)Int_Get_Value(num); } else { int32_t field_num = (int32_t)Vec_Get_Size(ivars->by_num); Hash_Store(ivars->by_name, field, (Obj*)Int_new(field_num)); Vec_Push(ivars->by_num, (Obj*)Str_Clone(field)); return field_num; } }
static void test_stress(TestBatch *batch) { Hash *hash = Hash_new(0); // trigger multiple rebuilds. VArray *expected = VA_new(1000); VArray *keys; VArray *values; for (uint32_t i = 0; i < 1000; i++) { CharBuf *cb = TestUtils_random_string(rand() % 1200); while (Hash_Fetch(hash, (Obj*)cb)) { DECREF(cb); cb = TestUtils_random_string(rand() % 1200); } Hash_Store(hash, (Obj*)cb, (Obj*)cb); VA_Push(expected, INCREF(cb)); } VA_Sort(expected, NULL, NULL); // Overwrite for good measure. for (uint32_t i = 0; i < 1000; i++) { CharBuf *cb = (CharBuf*)VA_Fetch(expected, i); Hash_Store(hash, (Obj*)cb, INCREF(cb)); } keys = Hash_Keys(hash); values = Hash_Values(hash); VA_Sort(keys, NULL, NULL); VA_Sort(values, NULL, NULL); TEST_TRUE(batch, VA_Equals(keys, (Obj*)expected), "stress Keys"); TEST_TRUE(batch, VA_Equals(values, (Obj*)expected), "stress Values"); DECREF(keys); DECREF(values); DECREF(expected); DECREF(hash); }
Snapshot* Snapshot_Read_File_IMP(Snapshot *self, Folder *folder, String *path) { SnapshotIVARS *const ivars = Snapshot_IVARS(self); // Eliminate all prior data. Pick a snapshot file. S_zero_out(self); ivars->path = (path != NULL && Str_Get_Size(path) > 0) ? Str_Clone(path) : IxFileNames_latest_snapshot(folder); if (ivars->path) { Hash *snap_data = (Hash*)CERTIFY(Json_slurp_json(folder, ivars->path), HASH); Obj *format_obj = CERTIFY(Hash_Fetch_Utf8(snap_data, "format", 6), OBJ); int32_t format = (int32_t)Json_obj_to_i64(format_obj); Obj *subformat_obj = Hash_Fetch_Utf8(snap_data, "subformat", 9); int32_t subformat = subformat_obj ? (int32_t)Json_obj_to_i64(subformat_obj) : 0; // Verify that we can read the index properly. if (format > Snapshot_current_file_format) { THROW(ERR, "Snapshot format too recent: %i32, %i32", format, Snapshot_current_file_format); } // Build up list of entries. Vector *list = (Vector*)INCREF(CERTIFY( Hash_Fetch_Utf8(snap_data, "entries", 7), VECTOR)); if (format == 1 || (format == 2 && subformat < 1)) { Vector *cleaned = S_clean_segment_contents(list); DECREF(list); list = cleaned; } Hash_Clear(ivars->entries); for (uint32_t i = 0, max = Vec_Get_Size(list); i < max; i++) { String *entry = (String*)CERTIFY(Vec_Fetch(list, i), STRING); Hash_Store(ivars->entries, entry, (Obj*)CFISH_TRUE); } DECREF(list); DECREF(snap_data); } return self; }