Lock* Lock_init(Lock *self, Folder *folder, const CharBuf *name, const CharBuf *host, int32_t timeout, int32_t interval) { // Validate. if (interval <= 0) { DECREF(self); THROW(ERR, "Invalid value for 'interval': %i32", interval); } ZombieCharBuf *scratch = ZCB_WRAP(name); uint32_t code_point; while (0 != (code_point = ZCB_Nip_One(scratch))) { if (isalnum(code_point) || code_point == '.' || code_point == '-' || code_point == '_' ) { continue; } DECREF(self); THROW(ERR, "Lock name contains disallowed characters: '%o'", name); } // Assign. self->folder = (Folder*)INCREF(folder); self->timeout = timeout; self->name = CB_Clone(name); self->host = CB_Clone(host); self->interval = interval; // Derive. self->lock_path = CB_newf("locks/%o.lock", name); return self; }
bool_t Folder_delete_tree(Folder *self, const CharBuf *path) { Folder *enclosing_folder = Folder_Enclosing_Folder(self, path); // Don't allow Folder to delete itself. if (!path || !CB_Get_Size(path)) { return false; } if (enclosing_folder) { ZombieCharBuf *local = IxFileNames_local_part(path, ZCB_BLANK()); if (Folder_Local_Is_Directory(enclosing_folder, (CharBuf*)local)) { Folder *inner_folder = Folder_Local_Find_Folder(enclosing_folder, (CharBuf*)local); DirHandle *dh = Folder_Local_Open_Dir(inner_folder); if (dh) { VArray *files = VA_new(20); VArray *dirs = VA_new(20); CharBuf *entry = DH_Get_Entry(dh); while (DH_Next(dh)) { VA_Push(files, (Obj*)CB_Clone(entry)); if (DH_Entry_Is_Dir(dh) && !DH_Entry_Is_Symlink(dh)) { VA_Push(dirs, (Obj*)CB_Clone(entry)); } } for (uint32_t i = 0, max = VA_Get_Size(dirs); i < max; i++) { CharBuf *name = (CharBuf*)VA_Fetch(files, i); bool_t success = Folder_Delete_Tree(inner_folder, name); if (!success && Folder_Local_Exists(inner_folder, name)) { break; } } for (uint32_t i = 0, max = VA_Get_Size(files); i < max; i++) { CharBuf *name = (CharBuf*)VA_Fetch(files, i); bool_t success = Folder_Local_Delete(inner_folder, name); if (!success && Folder_Local_Exists(inner_folder, name)) { break; } } DECREF(dirs); DECREF(files); DECREF(dh); } } return Folder_Local_Delete(enclosing_folder, (CharBuf*)local); } else { // Return failure if the entry wasn't there in the first place. return false; } }
void Indexer_commit(Indexer *self) { // Safety check. if ( !self->write_lock ) { THROW(ERR, "Can't call commit() more than once"); } if (!self->prepared) { Indexer_Prepare_Commit(self); } if (self->needs_commit) { bool_t success; // Rename temp snapshot file. CharBuf *temp_snapfile = CB_Clone(self->snapfile); CB_Chop(self->snapfile, sizeof(".temp") - 1); Snapshot_Set_Path(self->snapshot, self->snapfile); success = Folder_Rename(self->folder, temp_snapfile, self->snapfile); DECREF(temp_snapfile); if (!success) { RETHROW(INCREF(Err_get_error())); } // Purge obsolete files. FilePurger_Purge(self->file_purger); } // Release locks, invalidating the Indexer. S_release_merge_lock(self); S_release_write_lock(self); }
static void test_Mimic_and_Clone(TestBatchRunner *runner) { String *wanted = S_get_str("foo"); CharBuf *wanted_cb = S_get_cb("foo"); CharBuf *got = S_get_cb("bar"); CB_Mimic(got, (Obj*)wanted); TEST_TRUE(runner, S_cb_equals(got, wanted), "Mimic String"); DECREF(got); got = S_get_cb("bar"); CB_Mimic(got, (Obj*)wanted_cb); TEST_TRUE(runner, S_cb_equals(got, wanted), "Mimic CharBuf"); DECREF(got); got = S_get_cb("bar"); CB_Mimic_Utf8(got, "foo", 3); TEST_TRUE(runner, S_cb_equals(got, wanted), "Mimic_Utf8"); DECREF(got); got = CB_Clone(wanted_cb); TEST_TRUE(runner, S_cb_equals(got, wanted), "Clone"); DECREF(got); DECREF(wanted); DECREF(wanted_cb); }
PolyPostingList* PolyPList_init(PolyPostingList *self, const CharBuf *field, VArray *readers, I32Array *starts) { u32_t i; const u32_t num_readers = VA_Get_Size(readers); /* Init. */ self->tick = 0; self->current = NULL; /* Assign. */ self->field = CB_Clone(field); /* Get sub-posting_lists and assign offsets. */ self->sub_plists = VA_new(num_readers); for (i = 0; i < num_readers; i++) { PostingsReader *const post_reader = (PostingsReader*)ASSERT_IS_A( VA_Fetch(readers, i), POSTINGSREADER); i32_t offset = I32Arr_Get(starts, i); SegPostingList *sub_plist = (SegPostingList*)PostReader_Posting_List( post_reader, field, NULL); if (sub_plist) { ASSERT_IS_A(sub_plist, SEGPOSTINGLIST); SegPList_Set_Doc_Base(sub_plist, offset); VA_Push(self->sub_plists, (Obj*)sub_plist); } } self->num_subs = VA_Get_Size(self->sub_plists); return self; }
LockFactory* LockFact_init(LockFactory *self, Folder *folder, const CharBuf *agent_id) { self->folder = (Folder*)INCREF(folder); self->agent_id = CB_Clone(agent_id); return self; }
DirHandle* DH_init(DirHandle *self, const CharBuf *dir) { DirHandleIVARS *const ivars = DH_IVARS(self); ivars->dir = CB_Clone(dir); ivars->entry = CB_new(32); ABSTRACT_CLASS_CHECK(self, DIRHANDLE); return self; }
Hash* NumType_dump(NumericType *self) { Hash *dump = NumType_Dump_For_Schema(self); Hash_Store_Str(dump, "_class", 6, (Obj*)CB_Clone(NumType_Get_Class_Name(self))); DECREF(Hash_Delete_Str(dump, "type", 4)); return dump; }
Hash* EasyAnalyzer_dump(EasyAnalyzer *self) { EasyAnalyzerIVARS *const ivars = EasyAnalyzer_IVARS(self); EasyAnalyzer_Dump_t super_dump = SUPER_METHOD_PTR(EASYANALYZER, Lucy_EasyAnalyzer_Dump); Hash *dump = super_dump(self); Hash_Store_Str(dump, "language", 8, (Obj*)CB_Clone(ivars->language)); return dump; }
static CharBuf* S_fullpath(RAMFolder *self, const CharBuf *path) { if (CB_Get_Size(self->path)) { return CB_newf("%o/%o", self->path, path); } else { return CB_Clone(path); } }
EasyAnalyzer* EasyAnalyzer_init(EasyAnalyzer *self, const CharBuf *language) { Analyzer_init((Analyzer*)self); EasyAnalyzerIVARS *const ivars = EasyAnalyzer_IVARS(self); ivars->language = CB_Clone(language); ivars->tokenizer = StandardTokenizer_new(); ivars->normalizer = Normalizer_new(NULL, true, false); ivars->stemmer = SnowStemmer_new(language); return self; }
Hash* FullTextType_dump(FullTextType *self) { Hash *dump = FullTextType_Dump_For_Schema(self); Hash_Store_Str(dump, "_class", 6, (Obj*)CB_Clone(FullTextType_Get_Class_Name(self))); Hash_Store_Str(dump, "analyzer", 8, (Obj*)Analyzer_Dump(self->analyzer)); DECREF(Hash_Delete_Str(dump, "type", 4)); return dump; }
FileHandle* FH_do_open(FileHandle *self, const CharBuf *path, uint32_t flags) { self->path = path ? CB_Clone(path) : CB_new(0); self->flags = flags; // Track number of live FileHandles released into the wild. FH_object_count++; ABSTRACT_CLASS_CHECK(self, FILEHANDLE); return self; }
TermVector* TV_init(TermVector *self, const CharBuf *field, const CharBuf *text, I32Array *positions, I32Array *start_offsets, I32Array *end_offsets) { /* Assign. */ self->field = CB_Clone(field); self->text = CB_Clone(text); self->num_pos = I32Arr_Get_Size(positions); self->positions = (I32Array*)INCREF(positions); self->start_offsets = (I32Array*)INCREF(start_offsets); self->end_offsets = (I32Array*)INCREF(end_offsets); if ( I32Arr_Get_Size(start_offsets) != self->num_pos || I32Arr_Get_Size(end_offsets) != self->num_pos ) { THROW("Unbalanced arrays: %u32 %u32 %u32", self->num_pos, I32Arr_Get_Size(start_offsets), I32Arr_Get_Size(end_offsets)); } return self; }
int32_t Seg_add_field(Segment *self, const CharBuf *field) { Integer32 *num = (Integer32*)Hash_Fetch(self->by_name, (Obj*)field); if (num) { return Int32_Get_Value(num); } else { int32_t field_num = VA_Get_Size(self->by_num); Hash_Store(self->by_name, (Obj*)field, (Obj*)Int32_new(field_num)); VA_Push(self->by_num, (Obj*)CB_Clone(field)); return field_num; } }
CFReaderDirHandle* CFReaderDH_init(CFReaderDirHandle *self, CompoundFileReader *cf_reader) { DH_init((DirHandle*)self, CFReader_Get_Path(cf_reader)); self->cf_reader = (CompoundFileReader*)INCREF(cf_reader); self->elems = Hash_Keys(self->cf_reader->records); self->tick = -1; // Accumulate entries from real Folder. DirHandle *dh = Folder_Local_Open_Dir(self->cf_reader->real_folder); CharBuf *entry = DH_Get_Entry(dh); while (DH_Next(dh)) { VA_Push(self->elems, (Obj*)CB_Clone(entry)); } DECREF(dh); return self; }
VArray* RAMFolder_list(RAMFolder *self) { Hash *elems = self->elems; VArray *file_list = VA_new(0); CharBuf *key; Obj *ignore; Hash_Iter_Init(elems); while (Hash_Iter_Next(elems, &key, &ignore)) { VA_Push(file_list, (Obj*)CB_Clone(key)); } return file_list; }
VArray* Folder_list(Folder *self, const CharBuf *path) { Folder *local_folder = Folder_Find_Folder(self, path); VArray *list = NULL; DirHandle *dh = Folder_Local_Open_Dir(local_folder); if (dh) { CharBuf *entry = DH_Get_Entry(dh); list = VA_new(32); while (DH_Next(dh)) { VA_Push(list, (Obj*)CB_Clone(entry)); } DECREF(dh); } else { ERR_ADD_FRAME(Err_get_error()); } return list; }
RangeQuery* RangeQuery_init(RangeQuery *self, const CharBuf *field, Obj *lower_term, Obj *upper_term, bool_t include_lower, bool_t include_upper) { Query_init((Query*)self, 0.0f); self->field = CB_Clone(field); self->lower_term = lower_term ? Obj_Clone(lower_term) : NULL; self->upper_term = upper_term ? Obj_Clone(upper_term) : NULL; self->include_lower = include_lower; self->include_upper = include_upper; if (!upper_term && !lower_term) { DECREF(self); self = NULL; THROW(ERR, "Must supply at least one of 'upper_term' and 'lower_term'"); } return self; }
Snapshot* Snapshot_read_file(Snapshot *self, Folder *folder, const CharBuf *path) { // Eliminate all prior data. Pick a snapshot file. S_zero_out(self); self->path = path ? CB_Clone(path) : IxFileNames_latest_snapshot(folder); if (self->path) { Hash *snap_data = (Hash*)CERTIFY( Json_slurp_json(folder, self->path), HASH); Obj *format_obj = CERTIFY( Hash_Fetch_Str(snap_data, "format", 6), OBJ); int32_t format = (int32_t)Obj_To_I64(format_obj); Obj *subformat_obj = Hash_Fetch_Str(snap_data, "subformat", 9); int32_t subformat = subformat_obj ? (int32_t)Obj_To_I64(subformat_obj) : 0; // Verify that we can read the index properly. if (format > Snapshot_current_file_format) { THROW(ERR, "Snapshot format too recent: %i32, %i32", format, Snapshot_current_file_format); } // Build up list of entries. VArray *list = (VArray*)CERTIFY( Hash_Fetch_Str(snap_data, "entries", 7), VARRAY); INCREF(list); if (format == 1 || (format == 2 && subformat < 1)) { VArray *cleaned = S_clean_segment_contents(list); DECREF(list); list = cleaned; } Hash_Clear(self->entries); for (uint32_t i = 0, max = VA_Get_Size(list); i < max; i++) { CharBuf *entry = (CharBuf*)CERTIFY( VA_Fetch(list, i), CHARBUF); Hash_Store(self->entries, (Obj*)entry, INCREF(&EMPTY)); } DECREF(list); DECREF(snap_data); } return self; }
void Seg_write_file(Segment *self, Folder *folder) { Hash *my_metadata = Hash_new(16); // Store metadata specific to this Segment object. Hash_Store_Str(my_metadata, "count", 5, (Obj*)CB_newf("%i64", self->count)); Hash_Store_Str(my_metadata, "name", 4, (Obj*)CB_Clone(self->name)); Hash_Store_Str(my_metadata, "field_names", 11, INCREF(self->by_num)); Hash_Store_Str(my_metadata, "format", 6, (Obj*)CB_newf("%i32", 1)); Hash_Store_Str(self->metadata, "segmeta", 7, (Obj*)my_metadata); CharBuf *filename = CB_newf("%o/segmeta.json", self->name); bool_t result = Json_spew_json((Obj*)self->metadata, folder, filename); DECREF(filename); if (!result) { RETHROW(INCREF(Err_get_error())); } }
InStream* InStream_do_open(InStream *self, Obj *file) { InStreamIVARS *const ivars = InStream_IVARS(self); // Init. ivars->buf = NULL; ivars->limit = NULL; ivars->offset = 0; ivars->window = FileWindow_new(); // Obtain a FileHandle. if (Obj_Is_A(file, FILEHANDLE)) { ivars->file_handle = (FileHandle*)INCREF(file); } else if (Obj_Is_A(file, RAMFILE)) { ivars->file_handle = (FileHandle*)RAMFH_open(NULL, FH_READ_ONLY, (RAMFile*)file); } else if (Obj_Is_A(file, CHARBUF)) { ivars->file_handle = (FileHandle*)FSFH_open((CharBuf*)file, FH_READ_ONLY); } else { Err_set_error(Err_new(CB_newf("Invalid type for param 'file': '%o'", Obj_Get_Class_Name(file)))); DECREF(self); return NULL; } if (!ivars->file_handle) { ERR_ADD_FRAME(Err_get_error()); DECREF(self); return NULL; } // Get length and filename from the FileHandle. ivars->filename = CB_Clone(FH_Get_Path(ivars->file_handle)); ivars->len = FH_Length(ivars->file_handle); if (ivars->len == -1) { ERR_ADD_FRAME(Err_get_error()); DECREF(self); return NULL; } return self; }
IndexManager* IxManager_init(IndexManager *self, const CharBuf *host, LockFactory *lock_factory) { self->host = host ? CB_Clone(host) : CB_new_from_trusted_utf8("", 0); self->lock_factory = (LockFactory*)INCREF(lock_factory); self->folder = NULL; self->write_lock_timeout = 1000; self->write_lock_interval = 100; self->merge_lock_timeout = 0; self->merge_lock_interval = 1000; self->deletion_lock_timeout = 1000; self->deletion_lock_interval = 100; return self; }
CharBuf* PhraseQuery_to_string(PhraseQuery *self) { uint32_t i; uint32_t num_terms = VA_Get_Size(self->terms); CharBuf *retval = CB_Clone(self->field); CB_Cat_Trusted_Str(retval, ":\"", 2); for (i = 0; i < num_terms; i++) { Obj *term = VA_Fetch(self->terms, i); CharBuf *term_string = Obj_To_String(term); CB_Cat(retval, term_string); DECREF(term_string); if (i < num_terms - 1) { CB_Cat_Trusted_Str(retval, " ", 1); } } CB_Cat_Trusted_Str(retval, "\"", 1); return retval; }
Folder* Folder_init(Folder *self, const CharBuf *path) { // Init. self->entries = Hash_new(16); // Copy. if (path == NULL) { self->path = CB_new_from_trusted_utf8("", 0); } else { // Copy path, strip trailing slash or equivalent. self->path = CB_Clone(path); if (CB_Ends_With_Str(self->path, DIR_SEP, strlen(DIR_SEP))) { CB_Chop(self->path, 1); } } ABSTRACT_CLASS_CHECK(self, FOLDER); return self; }
static void test_Mimic_and_Clone(TestBatch *batch) { CharBuf *wanted = S_get_cb("foo"); CharBuf *got = S_get_cb("bar"); CB_Mimic(got, (Obj*)wanted); TEST_TRUE(batch, CB_Equals(wanted, (Obj*)got), "Mimic"); DECREF(got); got = S_get_cb("bar"); CB_Mimic_Str(got, "foo", 3); TEST_TRUE(batch, CB_Equals(wanted, (Obj*)got), "Mimic_Str"); DECREF(got); got = CB_Clone(wanted); TEST_TRUE(batch, CB_Equals(wanted, (Obj*)got), "Clone"); DECREF(got); DECREF(wanted); }
void Snapshot_write_file(Snapshot *self, Folder *folder, const CharBuf *path) { Hash *all_data = Hash_new(0); VArray *list = Snapshot_List(self); // Update path. DECREF(self->path); if (path) { self->path = CB_Clone(path); } else { CharBuf *latest = IxFileNames_latest_snapshot(folder); uint64_t gen = latest ? IxFileNames_extract_gen(latest) + 1 : 1; char base36[StrHelp_MAX_BASE36_BYTES]; StrHelp_to_base36(gen, &base36); self->path = CB_newf("snapshot_%s.json", &base36); DECREF(latest); } // Don't overwrite. if (Folder_Exists(folder, self->path)) { THROW(ERR, "Snapshot file '%o' already exists", self->path); } // Sort, then store file names. VA_Sort(list, NULL, NULL); Hash_Store_Str(all_data, "entries", 7, (Obj*)list); // Create a JSON-izable data structure. Hash_Store_Str(all_data, "format", 6, (Obj*)CB_newf("%i32", (int32_t)Snapshot_current_file_format) ); Hash_Store_Str(all_data, "subformat", 9, (Obj*)CB_newf("%i32", (int32_t)Snapshot_current_file_subformat) ); // Write out JSON-ized data to the new file. Json_spew_json((Obj*)all_data, folder, self->path); DECREF(all_data); }
Snapshot* Snapshot_read_file(Snapshot *self, Folder *folder, const CharBuf *filename) { /* Eliminate all prior data. Pick a snapshot file. */ S_zero_out(self); self->filename = filename ? CB_Clone(filename) : IxFileNames_latest_snapshot(folder); if (self->filename) { Hash *snap_data = (Hash*)ASSERT_IS_A( Json_slurp_json(folder, self->filename), HASH); Obj *format = ASSERT_IS_A( Hash_Fetch_Str(snap_data, "format", 6), OBJ); /* Verify that we can read the index properly. */ if (Obj_To_I64(format) > Snapshot_current_file_format) { THROW("Snapshot format too recent: %i64, %i32", Obj_To_I64(format), Snapshot_current_file_format); } /* Build up list of entries. */ { u32_t i, max; VArray *list = (VArray*)ASSERT_IS_A( Hash_Fetch_Str(snap_data, "entries", 7), VARRAY); Hash_Clear(self->entries); for (i = 0, max = VA_Get_Size(list); i < max; i++) { CharBuf *entry = (CharBuf*)ASSERT_IS_A( VA_Fetch(list, i), CHARBUF); Hash_Store(self->entries, entry, INCREF(&EMPTY)); } } DECREF(snap_data); } return self; }
void Snapshot_write_file(Snapshot *self, Folder *folder, const CharBuf *filename) { Hash *all_data = Hash_new(0); VArray *list = Snapshot_List(self); /* Update filename. */ DECREF(self->filename); if (filename) { self->filename = CB_Clone(filename); } else { CharBuf *latest = IxFileNames_latest_snapshot(folder); i32_t gen = latest ? IxFileNames_extract_gen(latest) + 1 : 1; CharBuf *base_36 = StrHelp_to_base36(gen); self->filename = CB_newf("snapshot_%o.json", base_36); DECREF(latest); DECREF(base_36); } /* Don't overwrite. */ if (Folder_Exists(folder, self->filename)) { THROW("Snapshot file '%o' already exists", self->filename); } /* Sort, then store file names. */ VA_Sort(list, NULL); Hash_Store_Str(all_data, "entries", 7, (Obj*)list); /* Create a JSON-izable data structure. */ Hash_Store_Str(all_data, "format", 6, (Obj*)CB_newf("%i32", (i32_t)Snapshot_current_file_format) ); /* Write out JSON-ized data to the new file. */ Json_spew_json((Obj*)all_data, folder, self->filename); DECREF(all_data); }
PostingPool* PostPool_init(PostingPool *self, Schema *schema, const CharBuf *field, MemoryPool *mem_pool) { Architecture *arch = Schema_Get_Architecture(schema); /* Init. */ SortExRun_init((SortExRun*)self); self->lex_instream = NULL; self->post_instream = NULL; self->lex_start = I64_MAX; self->post_start = I64_MAX; self->lex_end = 0; self->post_end = 0; self->flipped = false; self->from_seg = false; self->mem_thresh = 0; self->doc_base = 0; self->last_doc_id = 0; self->doc_map = NULL; self->post_count = 0; self->scratch = NULL; self->scratch_cap = 0; self->lex_stepper = LexStepper_new(field, Arch_Skip_Interval(arch)); /* Assign. */ self->schema = (Schema*)INCREF(schema); self->mem_pool = (MemoryPool*)INCREF(mem_pool); self->field = CB_Clone(field); /* Derive. */ self->posting = Schema_Fetch_Posting(schema, field); self->posting = (Posting*)Post_Clone(self->posting); self->type = (FieldType*)INCREF(Schema_Fetch_Type(schema, field)); self->compare = PostPoolQ_compare_rawp; return self; }