static bool S_obtain_deletion_lock(PolyReader *self) { PolyReaderIVARS *const ivars = PolyReader_IVARS(self); ivars->deletion_lock = IxManager_Make_Deletion_Lock(ivars->manager); Lock_Clear_Stale(ivars->deletion_lock); if (!Lock_Obtain(ivars->deletion_lock)) { DECREF(ivars->deletion_lock); ivars->deletion_lock = NULL; return false; } return true; }
static void ObtainOneLock(struct Lock *lock, int how) { switch (how) { case READ_LOCK: if (!WillBlock(lock, how)) (lock) -> readers_reading++; else Lock_Obtain(lock, how); break; case WRITE_LOCK: case SHARED_LOCK: if (!WillBlock(lock, how)) (lock) -> excl_locked = how; else Lock_Obtain(lock, how); break; default: PANICPRINT("unknown locking type"); fprintf(stderr,"%d\n",how); } }
static void S_obtain_write_lock(BackgroundMerger *self) { BackgroundMergerIVARS *const ivars = BGMerger_IVARS(self); Lock *write_lock = IxManager_Make_Write_Lock(ivars->manager); Lock_Clear_Stale(write_lock); if (Lock_Obtain(write_lock)) { // Only assign if successful, otherwise DESTROY unlocks -- bad! ivars->write_lock = write_lock; } else { DECREF(write_lock); } }
static bool S_obtain_read_lock(PolyReader *self, const CharBuf *snapshot_file_name) { PolyReaderIVARS *const ivars = PolyReader_IVARS(self); ivars->read_lock = IxManager_Make_Snapshot_Read_Lock(ivars->manager, snapshot_file_name); Lock_Clear_Stale(ivars->read_lock); if (!Lock_Obtain(ivars->read_lock)) { DECREF(ivars->read_lock); ivars->read_lock = NULL; return false; } return true; }
static void S_obtain_merge_lock(BackgroundMerger *self) { BackgroundMergerIVARS *const ivars = BGMerger_IVARS(self); Lock *merge_lock = IxManager_Make_Merge_Lock(ivars->manager); Lock_Clear_Stale(merge_lock); if (Lock_Obtain(merge_lock)) { // Only assign if successful, same rationale as above. ivars->merge_lock = merge_lock; } else { // We can't get the merge lock, so it seems there must be another // BackgroundMerger running. DECREF(merge_lock); } }
void FilePurger_purge(FilePurger *self) { Lock *deletion_lock = IxManager_Make_Deletion_Lock(self->manager); // Obtain deletion lock, purge files, release deletion lock. Lock_Clear_Stale(deletion_lock); if (Lock_Obtain(deletion_lock)) { Folder *folder = self->folder; Hash *failures = Hash_new(0); VArray *purgables; VArray *snapshots; S_discover_unused(self, &purgables, &snapshots); // Attempt to delete entries -- if failure, no big deal, just try // again later. Proceed in reverse lexical order so that directories // get deleted after they've been emptied. VA_Sort(purgables, NULL, NULL); for (uint32_t i = VA_Get_Size(purgables); i--; ) { CharBuf *entry = (CharBuf*)VA_fetch(purgables, i); if (Hash_Fetch(self->disallowed, (Obj*)entry)) { continue; } if (!Folder_Delete(folder, entry)) { if (Folder_Exists(folder, entry)) { Hash_Store(failures, (Obj*)entry, INCREF(&EMPTY)); } } } for (uint32_t i = 0, max = VA_Get_Size(snapshots); i < max; i++) { Snapshot *snapshot = (Snapshot*)VA_Fetch(snapshots, i); bool_t snapshot_has_failures = false; if (Hash_Get_Size(failures)) { // Only delete snapshot files if all of their entries were // successfully deleted. VArray *entries = Snapshot_List(snapshot); for (uint32_t j = VA_Get_Size(entries); j--; ) { CharBuf *entry = (CharBuf*)VA_Fetch(entries, j); if (Hash_Fetch(failures, (Obj*)entry)) { snapshot_has_failures = true; break; } } DECREF(entries); } if (!snapshot_has_failures) { CharBuf *snapfile = Snapshot_Get_Path(snapshot); Folder_Delete(folder, snapfile); } } DECREF(failures); DECREF(purgables); DECREF(snapshots); Lock_Release(deletion_lock); } else { WARN("Can't obtain deletion lock, skipping deletion of " "obsolete files"); } DECREF(deletion_lock); }
Indexer* Indexer_init(Indexer *self, Schema *schema, Obj *index, IndexManager *manager, int32_t flags) { bool_t create = (flags & Indexer_CREATE) ? true : false; bool_t truncate = (flags & Indexer_TRUNCATE) ? true : false; Folder *folder = S_init_folder(index, create); Lock *write_lock; CharBuf *latest_snapfile; Snapshot *latest_snapshot = Snapshot_new(); // Init. self->stock_doc = Doc_new(NULL, 0); self->truncate = false; self->optimize = false; self->prepared = false; self->needs_commit = false; self->snapfile = NULL; self->merge_lock = NULL; // Assign. self->folder = folder; self->manager = manager ? (IndexManager*)INCREF(manager) : IxManager_new(NULL, NULL); IxManager_Set_Folder(self->manager, folder); // Get a write lock for this folder. write_lock = IxManager_Make_Write_Lock(self->manager); Lock_Clear_Stale(write_lock); if (Lock_Obtain(write_lock)) { // Only assign if successful, otherwise DESTROY unlocks -- bad! self->write_lock = write_lock; } else { DECREF(write_lock); DECREF(self); RETHROW(INCREF(Err_get_error())); } // Find the latest snapshot or create a new one. latest_snapfile = IxFileNames_latest_snapshot(folder); if (latest_snapfile) { Snapshot_Read_File(latest_snapshot, folder, latest_snapfile); } // Look for an existing Schema if one wasn't supplied. if (schema) { self->schema = (Schema*)INCREF(schema); } else { if (!latest_snapfile) { THROW(ERR, "No Schema supplied, and can't find one in the index"); } else { CharBuf *schema_file = S_find_schema_file(latest_snapshot); Hash *dump = (Hash*)Json_slurp_json(folder, schema_file); if (dump) { // read file successfully self->schema = (Schema*)CERTIFY( VTable_Load_Obj(SCHEMA, (Obj*)dump), SCHEMA); schema = self->schema; DECREF(dump); schema_file = NULL; } else { THROW(ERR, "Failed to parse %o", schema_file); } } } // If we're clobbering, start with an empty Snapshot and an empty // PolyReader. Otherwise, start with the most recent Snapshot and an // up-to-date PolyReader. if (truncate) { self->snapshot = Snapshot_new(); self->polyreader = PolyReader_new(schema, folder, NULL, NULL, NULL); self->truncate = true; } else { // TODO: clone most recent snapshot rather than read it twice. self->snapshot = (Snapshot*)INCREF(latest_snapshot); self->polyreader = latest_snapfile ? PolyReader_open((Obj*)folder, NULL, NULL) : PolyReader_new(schema, folder, NULL, NULL, NULL); if (latest_snapfile) { // Make sure than any existing fields which may have been // dynamically added during past indexing sessions get added. Schema *old_schema = PolyReader_Get_Schema(self->polyreader); Schema_Eat(schema, old_schema); } } // Zap detritus from previous sessions. { // Note: we have to feed FilePurger with the most recent snapshot file // now, but with the Indexer's snapshot later. FilePurger *file_purger = FilePurger_new(folder, latest_snapshot, self->manager); FilePurger_Purge(file_purger); DECREF(file_purger); } // Create a new segment. { int64_t new_seg_num = IxManager_Highest_Seg_Num(self->manager, latest_snapshot) + 1; Lock *merge_lock = IxManager_Make_Merge_Lock(self->manager); uint32_t i, max; if (Lock_Is_Locked(merge_lock)) { // If there's a background merge process going on, stay out of its // way. Hash *merge_data = IxManager_Read_Merge_Data(self->manager); Obj *cutoff_obj = merge_data ? Hash_Fetch_Str(merge_data, "cutoff", 6) : NULL; if (!cutoff_obj) { DECREF(merge_lock); DECREF(merge_data); THROW(ERR, "Background merge detected, but can't read merge data"); } else { int64_t cutoff = Obj_To_I64(cutoff_obj); if (cutoff >= new_seg_num) { new_seg_num = cutoff + 1; } } DECREF(merge_data); } self->segment = Seg_new(new_seg_num); // Add all known fields to Segment. { VArray *fields = Schema_All_Fields(schema); for (i = 0, max = VA_Get_Size(fields); i < max; i++) { Seg_Add_Field(self->segment, (CharBuf*)VA_Fetch(fields, i)); } DECREF(fields); } DECREF(merge_lock); } // Create new SegWriter and FilePurger. self->file_purger = FilePurger_new(folder, self->snapshot, self->manager); self->seg_writer = SegWriter_new(self->schema, self->snapshot, self->segment, self->polyreader); SegWriter_Prep_Seg_Dir(self->seg_writer); // Grab a local ref to the DeletionsWriter. self->del_writer = (DeletionsWriter*)INCREF( SegWriter_Get_Del_Writer(self->seg_writer)); DECREF(latest_snapfile); DECREF(latest_snapshot); return self; }
static bool_t S_maybe_merge(Indexer *self, VArray *seg_readers) { bool_t merge_happened = false; uint32_t num_seg_readers = VA_Get_Size(seg_readers); Lock *merge_lock = IxManager_Make_Merge_Lock(self->manager); bool_t got_merge_lock = Lock_Obtain(merge_lock); int64_t cutoff; VArray *to_merge; uint32_t i, max; if (got_merge_lock) { self->merge_lock = merge_lock; cutoff = 0; } else { // If something else holds the merge lock, don't interfere. Hash *merge_data = IxManager_Read_Merge_Data(self->manager); if (merge_data) { Obj *cutoff_obj = Hash_Fetch_Str(merge_data, "cutoff", 6); if (cutoff_obj) { cutoff = Obj_To_I64(cutoff_obj); } else { cutoff = I64_MAX; } DECREF(merge_data); } else { cutoff = I64_MAX; } DECREF(merge_lock); } // Get a list of segments to recycle. Validate and confirm that there are // no dupes in the list. to_merge = IxManager_Recycle(self->manager, self->polyreader, self->del_writer, cutoff, self->optimize); { Hash *seen = Hash_new(VA_Get_Size(to_merge)); for (i = 0, max = VA_Get_Size(to_merge); i < max; i++) { SegReader *seg_reader = (SegReader*)CERTIFY( VA_Fetch(to_merge, i), SEGREADER); CharBuf *seg_name = SegReader_Get_Seg_Name(seg_reader); if (Hash_Fetch(seen, (Obj*)seg_name)) { DECREF(seen); DECREF(to_merge); THROW(ERR, "Recycle() tried to merge segment '%o' twice", seg_name); } Hash_Store(seen, (Obj*)seg_name, INCREF(&EMPTY)); } DECREF(seen); } // Consolidate segments if either sparse or optimizing forced. for (i = 0, max = VA_Get_Size(to_merge); i < max; i++) { SegReader *seg_reader = (SegReader*)VA_Fetch(to_merge, i); int64_t seg_num = SegReader_Get_Seg_Num(seg_reader); Matcher *deletions = DelWriter_Seg_Deletions(self->del_writer, seg_reader); I32Array *doc_map = DelWriter_Generate_Doc_Map(self->del_writer, deletions, SegReader_Doc_Max(seg_reader), (int32_t)Seg_Get_Count(self->segment) ); if (seg_num <= cutoff) { THROW(ERR, "Segment %o violates cutoff (%i64 <= %i64)", SegReader_Get_Seg_Name(seg_reader), seg_num, cutoff); } SegWriter_Merge_Segment(self->seg_writer, seg_reader, doc_map); merge_happened = true; DECREF(deletions); DECREF(doc_map); } // Write out new deletions. if (DelWriter_Updated(self->del_writer)) { // Only write out if they haven't all been applied. if (VA_Get_Size(to_merge) != num_seg_readers) { DelWriter_Finish(self->del_writer); } } DECREF(to_merge); return merge_happened; }