Lock*
Lock_init(Lock *self, Folder *folder, const CharBuf *name,
const CharBuf *host, int32_t timeout, int32_t interval) {
// Validate.
if (interval <= 0) {
DECREF(self);
THROW(ERR, "Invalid value for 'interval': %i32", interval);
}
ZombieCharBuf *scratch = ZCB_WRAP(name);
uint32_t code_point;
while (0 != (code_point = ZCB_Nip_One(scratch))) {
if (isalnum(code_point)
|| code_point == '.'
|| code_point == '-'
|| code_point == '_'
) {
continue;
}
DECREF(self);
THROW(ERR, "Lock name contains disallowed characters: '%o'", name);
}
// Assign.
self->folder = (Folder*)INCREF(folder);
self->timeout = timeout;
self->name = CB_Clone(name);
self->host = CB_Clone(host);
self->interval = interval;
// Derive.
self->lock_path = CB_newf("locks/%o.lock", name);
return self;
}
bool_t
Folder_delete_tree(Folder *self, const CharBuf *path) {
Folder *enclosing_folder = Folder_Enclosing_Folder(self, path);
// Don't allow Folder to delete itself.
if (!path || !CB_Get_Size(path)) {
return false;
}
if (enclosing_folder) {
ZombieCharBuf *local = IxFileNames_local_part(path, ZCB_BLANK());
if (Folder_Local_Is_Directory(enclosing_folder, (CharBuf*)local)) {
Folder *inner_folder
= Folder_Local_Find_Folder(enclosing_folder, (CharBuf*)local);
DirHandle *dh = Folder_Local_Open_Dir(inner_folder);
if (dh) {
VArray *files = VA_new(20);
VArray *dirs = VA_new(20);
CharBuf *entry = DH_Get_Entry(dh);
while (DH_Next(dh)) {
VA_Push(files, (Obj*)CB_Clone(entry));
if (DH_Entry_Is_Dir(dh) && !DH_Entry_Is_Symlink(dh)) {
VA_Push(dirs, (Obj*)CB_Clone(entry));
}
}
for (uint32_t i = 0, max = VA_Get_Size(dirs); i < max; i++) {
CharBuf *name = (CharBuf*)VA_Fetch(files, i);
bool_t success = Folder_Delete_Tree(inner_folder, name);
if (!success && Folder_Local_Exists(inner_folder, name)) {
break;
}
}
for (uint32_t i = 0, max = VA_Get_Size(files); i < max; i++) {
CharBuf *name = (CharBuf*)VA_Fetch(files, i);
bool_t success = Folder_Local_Delete(inner_folder, name);
if (!success && Folder_Local_Exists(inner_folder, name)) {
break;
}
}
DECREF(dirs);
DECREF(files);
DECREF(dh);
}
}
return Folder_Local_Delete(enclosing_folder, (CharBuf*)local);
}
else {
// Return failure if the entry wasn't there in the first place.
return false;
}
}
void
Indexer_commit(Indexer *self)
{
// Safety check.
if ( !self->write_lock ) {
THROW(ERR, "Can't call commit() more than once");
}
if (!self->prepared) {
Indexer_Prepare_Commit(self);
}
if (self->needs_commit) {
bool_t success;
// Rename temp snapshot file.
CharBuf *temp_snapfile = CB_Clone(self->snapfile);
CB_Chop(self->snapfile, sizeof(".temp") - 1);
Snapshot_Set_Path(self->snapshot, self->snapfile);
success = Folder_Rename(self->folder, temp_snapfile, self->snapfile);
DECREF(temp_snapfile);
if (!success) { RETHROW(INCREF(Err_get_error())); }
// Purge obsolete files.
FilePurger_Purge(self->file_purger);
}
// Release locks, invalidating the Indexer.
S_release_merge_lock(self);
S_release_write_lock(self);
}
static void
test_Mimic_and_Clone(TestBatchRunner *runner) {
String *wanted = S_get_str("foo");
CharBuf *wanted_cb = S_get_cb("foo");
CharBuf *got = S_get_cb("bar");
CB_Mimic(got, (Obj*)wanted);
TEST_TRUE(runner, S_cb_equals(got, wanted), "Mimic String");
DECREF(got);
got = S_get_cb("bar");
CB_Mimic(got, (Obj*)wanted_cb);
TEST_TRUE(runner, S_cb_equals(got, wanted), "Mimic CharBuf");
DECREF(got);
got = S_get_cb("bar");
CB_Mimic_Utf8(got, "foo", 3);
TEST_TRUE(runner, S_cb_equals(got, wanted), "Mimic_Utf8");
DECREF(got);
got = CB_Clone(wanted_cb);
TEST_TRUE(runner, S_cb_equals(got, wanted), "Clone");
DECREF(got);
DECREF(wanted);
DECREF(wanted_cb);
}
PolyPostingList*
PolyPList_init(PolyPostingList *self, const CharBuf *field,
VArray *readers, I32Array *starts)
{
u32_t i;
const u32_t num_readers = VA_Get_Size(readers);
/* Init. */
self->tick = 0;
self->current = NULL;
/* Assign. */
self->field = CB_Clone(field);
/* Get sub-posting_lists and assign offsets. */
self->sub_plists = VA_new(num_readers);
for (i = 0; i < num_readers; i++) {
PostingsReader *const post_reader = (PostingsReader*)ASSERT_IS_A(
VA_Fetch(readers, i), POSTINGSREADER);
i32_t offset = I32Arr_Get(starts, i);
SegPostingList *sub_plist = (SegPostingList*)PostReader_Posting_List(
post_reader, field, NULL);
if (sub_plist) {
ASSERT_IS_A(sub_plist, SEGPOSTINGLIST);
SegPList_Set_Doc_Base(sub_plist, offset);
VA_Push(self->sub_plists, (Obj*)sub_plist);
}
}
self->num_subs = VA_Get_Size(self->sub_plists);
return self;
}
LockFactory*
LockFact_init(LockFactory *self, Folder *folder, const CharBuf *agent_id)
{
self->folder = (Folder*)INCREF(folder);
self->agent_id = CB_Clone(agent_id);
return self;
}
DirHandle*
DH_init(DirHandle *self, const CharBuf *dir) {
DirHandleIVARS *const ivars = DH_IVARS(self);
ivars->dir = CB_Clone(dir);
ivars->entry = CB_new(32);
ABSTRACT_CLASS_CHECK(self, DIRHANDLE);
return self;
}
Hash*
NumType_dump(NumericType *self) {
Hash *dump = NumType_Dump_For_Schema(self);
Hash_Store_Str(dump, "_class", 6,
(Obj*)CB_Clone(NumType_Get_Class_Name(self)));
DECREF(Hash_Delete_Str(dump, "type", 4));
return dump;
}
Hash*
EasyAnalyzer_dump(EasyAnalyzer *self) {
EasyAnalyzerIVARS *const ivars = EasyAnalyzer_IVARS(self);
EasyAnalyzer_Dump_t super_dump
= SUPER_METHOD_PTR(EASYANALYZER, Lucy_EasyAnalyzer_Dump);
Hash *dump = super_dump(self);
Hash_Store_Str(dump, "language", 8, (Obj*)CB_Clone(ivars->language));
return dump;
}
static CharBuf*
S_fullpath(RAMFolder *self, const CharBuf *path)
{
if (CB_Get_Size(self->path)) {
return CB_newf("%o/%o", self->path, path);
}
else {
return CB_Clone(path);
}
}
EasyAnalyzer*
EasyAnalyzer_init(EasyAnalyzer *self, const CharBuf *language) {
Analyzer_init((Analyzer*)self);
EasyAnalyzerIVARS *const ivars = EasyAnalyzer_IVARS(self);
ivars->language = CB_Clone(language);
ivars->tokenizer = StandardTokenizer_new();
ivars->normalizer = Normalizer_new(NULL, true, false);
ivars->stemmer = SnowStemmer_new(language);
return self;
}
Hash*
FullTextType_dump(FullTextType *self) {
Hash *dump = FullTextType_Dump_For_Schema(self);
Hash_Store_Str(dump, "_class", 6,
(Obj*)CB_Clone(FullTextType_Get_Class_Name(self)));
Hash_Store_Str(dump, "analyzer", 8,
(Obj*)Analyzer_Dump(self->analyzer));
DECREF(Hash_Delete_Str(dump, "type", 4));
return dump;
}
FileHandle*
FH_do_open(FileHandle *self, const CharBuf *path, uint32_t flags) {
self->path = path ? CB_Clone(path) : CB_new(0);
self->flags = flags;
// Track number of live FileHandles released into the wild.
FH_object_count++;
ABSTRACT_CLASS_CHECK(self, FILEHANDLE);
return self;
}
TermVector*
TV_init(TermVector *self, const CharBuf *field, const CharBuf *text,
I32Array *positions, I32Array *start_offsets, I32Array *end_offsets)
{
/* Assign. */
self->field = CB_Clone(field);
self->text = CB_Clone(text);
self->num_pos = I32Arr_Get_Size(positions);
self->positions = (I32Array*)INCREF(positions);
self->start_offsets = (I32Array*)INCREF(start_offsets);
self->end_offsets = (I32Array*)INCREF(end_offsets);
if ( I32Arr_Get_Size(start_offsets) != self->num_pos
|| I32Arr_Get_Size(end_offsets) != self->num_pos
) {
THROW("Unbalanced arrays: %u32 %u32 %u32", self->num_pos,
I32Arr_Get_Size(start_offsets), I32Arr_Get_Size(end_offsets));
}
return self;
}
int32_t
Seg_add_field(Segment *self, const CharBuf *field) {
Integer32 *num = (Integer32*)Hash_Fetch(self->by_name, (Obj*)field);
if (num) {
return Int32_Get_Value(num);
}
else {
int32_t field_num = VA_Get_Size(self->by_num);
Hash_Store(self->by_name, (Obj*)field, (Obj*)Int32_new(field_num));
VA_Push(self->by_num, (Obj*)CB_Clone(field));
return field_num;
}
}
CFReaderDirHandle*
CFReaderDH_init(CFReaderDirHandle *self, CompoundFileReader *cf_reader) {
DH_init((DirHandle*)self, CFReader_Get_Path(cf_reader));
self->cf_reader = (CompoundFileReader*)INCREF(cf_reader);
self->elems = Hash_Keys(self->cf_reader->records);
self->tick = -1;
// Accumulate entries from real Folder.
DirHandle *dh = Folder_Local_Open_Dir(self->cf_reader->real_folder);
CharBuf *entry = DH_Get_Entry(dh);
while (DH_Next(dh)) {
VA_Push(self->elems, (Obj*)CB_Clone(entry));
}
DECREF(dh);
return self;
}
VArray*
RAMFolder_list(RAMFolder *self)
{
Hash *elems = self->elems;
VArray *file_list = VA_new(0);
CharBuf *key;
Obj *ignore;
Hash_Iter_Init(elems);
while (Hash_Iter_Next(elems, &key, &ignore)) {
VA_Push(file_list, (Obj*)CB_Clone(key));
}
return file_list;
}
VArray*
Folder_list(Folder *self, const CharBuf *path) {
Folder *local_folder = Folder_Find_Folder(self, path);
VArray *list = NULL;
DirHandle *dh = Folder_Local_Open_Dir(local_folder);
if (dh) {
CharBuf *entry = DH_Get_Entry(dh);
list = VA_new(32);
while (DH_Next(dh)) { VA_Push(list, (Obj*)CB_Clone(entry)); }
DECREF(dh);
}
else {
ERR_ADD_FRAME(Err_get_error());
}
return list;
}
RangeQuery*
RangeQuery_init(RangeQuery *self, const CharBuf *field, Obj *lower_term,
Obj *upper_term, bool_t include_lower, bool_t include_upper) {
Query_init((Query*)self, 0.0f);
self->field = CB_Clone(field);
self->lower_term = lower_term ? Obj_Clone(lower_term) : NULL;
self->upper_term = upper_term ? Obj_Clone(upper_term) : NULL;
self->include_lower = include_lower;
self->include_upper = include_upper;
if (!upper_term && !lower_term) {
DECREF(self);
self = NULL;
THROW(ERR, "Must supply at least one of 'upper_term' and 'lower_term'");
}
return self;
}
Snapshot*
Snapshot_read_file(Snapshot *self, Folder *folder, const CharBuf *path)
{
// Eliminate all prior data. Pick a snapshot file.
S_zero_out(self);
self->path = path ? CB_Clone(path) : IxFileNames_latest_snapshot(folder);
if (self->path) {
Hash *snap_data = (Hash*)CERTIFY(
Json_slurp_json(folder, self->path), HASH);
Obj *format_obj = CERTIFY(
Hash_Fetch_Str(snap_data, "format", 6), OBJ);
int32_t format = (int32_t)Obj_To_I64(format_obj);
Obj *subformat_obj = Hash_Fetch_Str(snap_data, "subformat", 9);
int32_t subformat = subformat_obj
? (int32_t)Obj_To_I64(subformat_obj)
: 0;
// Verify that we can read the index properly.
if (format > Snapshot_current_file_format) {
THROW(ERR, "Snapshot format too recent: %i32, %i32",
format, Snapshot_current_file_format);
}
// Build up list of entries.
VArray *list = (VArray*)CERTIFY(
Hash_Fetch_Str(snap_data, "entries", 7), VARRAY);
INCREF(list);
if (format == 1 || (format == 2 && subformat < 1)) {
VArray *cleaned = S_clean_segment_contents(list);
DECREF(list);
list = cleaned;
}
Hash_Clear(self->entries);
for (uint32_t i = 0, max = VA_Get_Size(list); i < max; i++) {
CharBuf *entry = (CharBuf*)CERTIFY(
VA_Fetch(list, i), CHARBUF);
Hash_Store(self->entries, (Obj*)entry, INCREF(&EMPTY));
}
DECREF(list);
DECREF(snap_data);
}
return self;
}
void
Seg_write_file(Segment *self, Folder *folder) {
Hash *my_metadata = Hash_new(16);
// Store metadata specific to this Segment object.
Hash_Store_Str(my_metadata, "count", 5,
(Obj*)CB_newf("%i64", self->count));
Hash_Store_Str(my_metadata, "name", 4, (Obj*)CB_Clone(self->name));
Hash_Store_Str(my_metadata, "field_names", 11, INCREF(self->by_num));
Hash_Store_Str(my_metadata, "format", 6, (Obj*)CB_newf("%i32", 1));
Hash_Store_Str(self->metadata, "segmeta", 7, (Obj*)my_metadata);
CharBuf *filename = CB_newf("%o/segmeta.json", self->name);
bool_t result = Json_spew_json((Obj*)self->metadata, folder, filename);
DECREF(filename);
if (!result) { RETHROW(INCREF(Err_get_error())); }
}
InStream*
InStream_do_open(InStream *self, Obj *file) {
InStreamIVARS *const ivars = InStream_IVARS(self);
// Init.
ivars->buf = NULL;
ivars->limit = NULL;
ivars->offset = 0;
ivars->window = FileWindow_new();
// Obtain a FileHandle.
if (Obj_Is_A(file, FILEHANDLE)) {
ivars->file_handle = (FileHandle*)INCREF(file);
}
else if (Obj_Is_A(file, RAMFILE)) {
ivars->file_handle
= (FileHandle*)RAMFH_open(NULL, FH_READ_ONLY, (RAMFile*)file);
}
else if (Obj_Is_A(file, CHARBUF)) {
ivars->file_handle
= (FileHandle*)FSFH_open((CharBuf*)file, FH_READ_ONLY);
}
else {
Err_set_error(Err_new(CB_newf("Invalid type for param 'file': '%o'",
Obj_Get_Class_Name(file))));
DECREF(self);
return NULL;
}
if (!ivars->file_handle) {
ERR_ADD_FRAME(Err_get_error());
DECREF(self);
return NULL;
}
// Get length and filename from the FileHandle.
ivars->filename = CB_Clone(FH_Get_Path(ivars->file_handle));
ivars->len = FH_Length(ivars->file_handle);
if (ivars->len == -1) {
ERR_ADD_FRAME(Err_get_error());
DECREF(self);
return NULL;
}
return self;
}
IndexManager*
IxManager_init(IndexManager *self, const CharBuf *host,
LockFactory *lock_factory)
{
self->host = host
? CB_Clone(host)
: CB_new_from_trusted_utf8("", 0);
self->lock_factory = (LockFactory*)INCREF(lock_factory);
self->folder = NULL;
self->write_lock_timeout = 1000;
self->write_lock_interval = 100;
self->merge_lock_timeout = 0;
self->merge_lock_interval = 1000;
self->deletion_lock_timeout = 1000;
self->deletion_lock_interval = 100;
return self;
}
CharBuf*
PhraseQuery_to_string(PhraseQuery *self)
{
uint32_t i;
uint32_t num_terms = VA_Get_Size(self->terms);
CharBuf *retval = CB_Clone(self->field);
CB_Cat_Trusted_Str(retval, ":\"", 2);
for (i = 0; i < num_terms; i++) {
Obj *term = VA_Fetch(self->terms, i);
CharBuf *term_string = Obj_To_String(term);
CB_Cat(retval, term_string);
DECREF(term_string);
if (i < num_terms - 1) {
CB_Cat_Trusted_Str(retval, " ", 1);
}
}
CB_Cat_Trusted_Str(retval, "\"", 1);
return retval;
}
Folder*
Folder_init(Folder *self, const CharBuf *path) {
// Init.
self->entries = Hash_new(16);
// Copy.
if (path == NULL) {
self->path = CB_new_from_trusted_utf8("", 0);
}
else {
// Copy path, strip trailing slash or equivalent.
self->path = CB_Clone(path);
if (CB_Ends_With_Str(self->path, DIR_SEP, strlen(DIR_SEP))) {
CB_Chop(self->path, 1);
}
}
ABSTRACT_CLASS_CHECK(self, FOLDER);
return self;
}
static void
test_Mimic_and_Clone(TestBatch *batch) {
CharBuf *wanted = S_get_cb("foo");
CharBuf *got = S_get_cb("bar");
CB_Mimic(got, (Obj*)wanted);
TEST_TRUE(batch, CB_Equals(wanted, (Obj*)got), "Mimic");
DECREF(got);
got = S_get_cb("bar");
CB_Mimic_Str(got, "foo", 3);
TEST_TRUE(batch, CB_Equals(wanted, (Obj*)got), "Mimic_Str");
DECREF(got);
got = CB_Clone(wanted);
TEST_TRUE(batch, CB_Equals(wanted, (Obj*)got), "Clone");
DECREF(got);
DECREF(wanted);
}
void
Snapshot_write_file(Snapshot *self, Folder *folder, const CharBuf *path)
{
Hash *all_data = Hash_new(0);
VArray *list = Snapshot_List(self);
// Update path.
DECREF(self->path);
if (path) {
self->path = CB_Clone(path);
}
else {
CharBuf *latest = IxFileNames_latest_snapshot(folder);
uint64_t gen = latest ? IxFileNames_extract_gen(latest) + 1 : 1;
char base36[StrHelp_MAX_BASE36_BYTES];
StrHelp_to_base36(gen, &base36);
self->path = CB_newf("snapshot_%s.json", &base36);
DECREF(latest);
}
// Don't overwrite.
if (Folder_Exists(folder, self->path)) {
THROW(ERR, "Snapshot file '%o' already exists", self->path);
}
// Sort, then store file names.
VA_Sort(list, NULL, NULL);
Hash_Store_Str(all_data, "entries", 7, (Obj*)list);
// Create a JSON-izable data structure.
Hash_Store_Str(all_data, "format", 6,
(Obj*)CB_newf("%i32", (int32_t)Snapshot_current_file_format) );
Hash_Store_Str(all_data, "subformat", 9,
(Obj*)CB_newf("%i32", (int32_t)Snapshot_current_file_subformat) );
// Write out JSON-ized data to the new file.
Json_spew_json((Obj*)all_data, folder, self->path);
DECREF(all_data);
}
Snapshot*
Snapshot_read_file(Snapshot *self, Folder *folder, const CharBuf *filename)
{
/* Eliminate all prior data. Pick a snapshot file. */
S_zero_out(self);
self->filename = filename
? CB_Clone(filename)
: IxFileNames_latest_snapshot(folder);
if (self->filename) {
Hash *snap_data = (Hash*)ASSERT_IS_A(
Json_slurp_json(folder, self->filename), HASH);
Obj *format = ASSERT_IS_A(
Hash_Fetch_Str(snap_data, "format", 6), OBJ);
/* Verify that we can read the index properly. */
if (Obj_To_I64(format) > Snapshot_current_file_format) {
THROW("Snapshot format too recent: %i64, %i32",
Obj_To_I64(format), Snapshot_current_file_format);
}
/* Build up list of entries. */
{
u32_t i, max;
VArray *list = (VArray*)ASSERT_IS_A(
Hash_Fetch_Str(snap_data, "entries", 7), VARRAY);
Hash_Clear(self->entries);
for (i = 0, max = VA_Get_Size(list); i < max; i++) {
CharBuf *entry = (CharBuf*)ASSERT_IS_A(
VA_Fetch(list, i), CHARBUF);
Hash_Store(self->entries, entry, INCREF(&EMPTY));
}
}
DECREF(snap_data);
}
return self;
}
void
Snapshot_write_file(Snapshot *self, Folder *folder, const CharBuf *filename)
{
Hash *all_data = Hash_new(0);
VArray *list = Snapshot_List(self);
/* Update filename. */
DECREF(self->filename);
if (filename) {
self->filename = CB_Clone(filename);
}
else {
CharBuf *latest = IxFileNames_latest_snapshot(folder);
i32_t gen = latest ? IxFileNames_extract_gen(latest) + 1 : 1;
CharBuf *base_36 = StrHelp_to_base36(gen);
self->filename = CB_newf("snapshot_%o.json", base_36);
DECREF(latest);
DECREF(base_36);
}
/* Don't overwrite. */
if (Folder_Exists(folder, self->filename)) {
THROW("Snapshot file '%o' already exists", self->filename);
}
/* Sort, then store file names. */
VA_Sort(list, NULL);
Hash_Store_Str(all_data, "entries", 7, (Obj*)list);
/* Create a JSON-izable data structure. */
Hash_Store_Str(all_data, "format", 6,
(Obj*)CB_newf("%i32", (i32_t)Snapshot_current_file_format) );
/* Write out JSON-ized data to the new file. */
Json_spew_json((Obj*)all_data, folder, self->filename);
DECREF(all_data);
}
PostingPool*
PostPool_init(PostingPool *self, Schema *schema,
const CharBuf *field, MemoryPool *mem_pool)
{
Architecture *arch = Schema_Get_Architecture(schema);
/* Init. */
SortExRun_init((SortExRun*)self);
self->lex_instream = NULL;
self->post_instream = NULL;
self->lex_start = I64_MAX;
self->post_start = I64_MAX;
self->lex_end = 0;
self->post_end = 0;
self->flipped = false;
self->from_seg = false;
self->mem_thresh = 0;
self->doc_base = 0;
self->last_doc_id = 0;
self->doc_map = NULL;
self->post_count = 0;
self->scratch = NULL;
self->scratch_cap = 0;
self->lex_stepper = LexStepper_new(field, Arch_Skip_Interval(arch));
/* Assign. */
self->schema = (Schema*)INCREF(schema);
self->mem_pool = (MemoryPool*)INCREF(mem_pool);
self->field = CB_Clone(field);
/* Derive. */
self->posting = Schema_Fetch_Posting(schema, field);
self->posting = (Posting*)Post_Clone(self->posting);
self->type = (FieldType*)INCREF(Schema_Fetch_Type(schema, field));
self->compare = PostPoolQ_compare_rawp;
return self;
}
