bool
StrCbTest_Unchanged_By_Callback_IMP(StringCallbackTest *self, String *str) {
String *before = Str_Clone(str);
StrCbTest_Callback(self);
return Str_Equals(str, (Obj*)before);
}
static void
test_iterator_substring(TestBatchRunner *runner) {
String *string = Str_newf("a%sb%sc%sd", smiley, smiley, smiley);
StringIterator *start = Str_Top(string);
StringIterator *end = Str_Tail(string);
{
String *substring = StrIter_crop(start, end);
TEST_TRUE(runner, Str_Equals(substring, (Obj*)string),
"StrIter_crop whole string");
DECREF(substring);
}
StrIter_Advance(start, 2);
StrIter_Recede(end, 2);
{
String *substring = StrIter_crop(start, end);
String *wanted = Str_newf("b%sc", smiley);
TEST_TRUE(runner, Str_Equals(substring, (Obj*)wanted),
"StrIter_crop");
TEST_TRUE(runner, StrIter_Starts_With(start, wanted),
"Starts_With returns true");
TEST_TRUE(runner, StrIter_Ends_With(end, wanted),
"Ends_With returns true");
DECREF(wanted);
DECREF(substring);
}
{
String *short_str = Str_newf("b%sx", smiley);
TEST_FALSE(runner, StrIter_Starts_With(start, short_str),
"Starts_With returns false");
TEST_FALSE(runner, StrIter_Ends_With(start, short_str),
"Ends_With returns false");
String *long_str = Str_newf("b%sxxxxxxxxxxxx%sc", smiley, smiley);
TEST_FALSE(runner, StrIter_Starts_With(start, long_str),
"Starts_With long string returns false");
TEST_FALSE(runner, StrIter_Ends_With(end, long_str),
"Ends_With long string returns false");
DECREF(short_str);
DECREF(long_str);
}
{
String *substring = StrIter_crop(end, NULL);
String *wanted = Str_newf("%sd", smiley);
TEST_TRUE(runner, Str_Equals(substring, (Obj*)wanted),
"StrIter_crop with NULL tail");
DECREF(wanted);
DECREF(substring);
}
{
String *substring = StrIter_crop(NULL, start);
String *wanted = Str_newf("a%s", smiley);
TEST_TRUE(runner, Str_Equals(substring, (Obj*)wanted),
"StrIter_crop with NULL top");
DECREF(wanted);
DECREF(substring);
}
DECREF(start);
DECREF(end);
DECREF(string);
}
static void
S_discover_unused(FilePurger *self, Vector **purgables_ptr,
Vector **snapshots_ptr) {
FilePurgerIVARS *const ivars = FilePurger_IVARS(self);
Folder *folder = ivars->folder;
DirHandle *dh = Folder_Open_Dir(folder, NULL);
if (!dh) { RETHROW(INCREF(Err_get_error())); }
Vector *spared = Vec_new(1);
Vector *snapshots = Vec_new(1);
String *snapfile = NULL;
// Start off with the list of files in the current snapshot.
if (ivars->snapshot) {
Vector *entries = Snapshot_List(ivars->snapshot);
Vector *referenced = S_find_all_referenced(folder, entries);
Vec_Push_All(spared, referenced);
DECREF(entries);
DECREF(referenced);
snapfile = Snapshot_Get_Path(ivars->snapshot);
if (snapfile) { Vec_Push(spared, INCREF(snapfile)); }
}
Hash *candidates = Hash_new(64);
while (DH_Next(dh)) {
String *entry = DH_Get_Entry(dh);
if (Str_Starts_With_Utf8(entry, "snapshot_", 9)
&& Str_Ends_With_Utf8(entry, ".json", 5)
&& (!snapfile || !Str_Equals(entry, (Obj*)snapfile))
) {
Snapshot *snapshot
= Snapshot_Read_File(Snapshot_new(), folder, entry);
Lock *lock
= IxManager_Make_Snapshot_Read_Lock(ivars->manager, entry);
Vector *snap_list = Snapshot_List(snapshot);
Vector *referenced = S_find_all_referenced(folder, snap_list);
// DON'T obtain the lock -- only see whether another
// entity holds a lock on the snapshot file.
if (lock) {
Lock_Clear_Stale(lock);
}
if (lock && Lock_Is_Locked(lock)) {
// The snapshot file is locked, which means someone's using
// that version of the index -- protect all of its entries.
uint32_t new_size = Vec_Get_Size(spared)
+ Vec_Get_Size(referenced)
+ 1;
Vec_Grow(spared, new_size);
Vec_Push(spared, (Obj*)Str_Clone(entry));
Vec_Push_All(spared, referenced);
}
else {
// No one's using this snapshot, so all of its entries are
// candidates for deletion.
for (uint32_t i = 0, max = Vec_Get_Size(referenced); i < max; i++) {
String *file = (String*)Vec_Fetch(referenced, i);
Hash_Store(candidates, file, (Obj*)CFISH_TRUE);
}
Vec_Push(snapshots, INCREF(snapshot));
}
DECREF(referenced);
DECREF(snap_list);
DECREF(snapshot);
DECREF(lock);
}
DECREF(entry);
}
DECREF(dh);
// Clean up after a dead segment consolidation.
S_zap_dead_merge(self, candidates);
// Eliminate any current files from the list of files to be purged.
for (uint32_t i = 0, max = Vec_Get_Size(spared); i < max; i++) {
String *filename = (String*)Vec_Fetch(spared, i);
DECREF(Hash_Delete(candidates, filename));
}
// Pass back purgables and Snapshots.
*purgables_ptr = Hash_Keys(candidates);
*snapshots_ptr = snapshots;
DECREF(candidates);
DECREF(spared);
}
VArray*
ProximityCompiler_Highlight_Spans_IMP(ProximityCompiler *self,
Searcher *searcher, DocVector *doc_vec,
String *field) {
ProximityCompilerIVARS *const ivars = ProximityCompiler_IVARS(self);
ProximityQueryIVARS *const parent_ivars
= ProximityQuery_IVARS((ProximityQuery*)ivars->parent);
VArray *const terms = parent_ivars->terms;
VArray *const spans = VA_new(0);
const uint32_t num_terms = VA_Get_Size(terms);
UNUSED_VAR(searcher);
// Bail if no terms or field doesn't match.
if (!num_terms) { return spans; }
if (!Str_Equals(field, (Obj*)parent_ivars->field)) { return spans; }
VArray *term_vectors = VA_new(num_terms);
BitVector *posit_vec = BitVec_new(0);
BitVector *other_posit_vec = BitVec_new(0);
for (uint32_t i = 0; i < num_terms; i++) {
Obj *term = VA_Fetch(terms, i);
TermVector *term_vector
= DocVec_Term_Vector(doc_vec, field, (String*)term);
// Bail if any term is missing.
if (!term_vector) {
break;
}
VA_Push(term_vectors, (Obj*)term_vector);
if (i == 0) {
// Set initial positions from first term.
I32Array *positions = TV_Get_Positions(term_vector);
for (uint32_t j = I32Arr_Get_Size(positions); j > 0; j--) {
BitVec_Set(posit_vec, I32Arr_Get(positions, j - 1));
}
}
else {
// Filter positions using logical "and".
I32Array *positions = TV_Get_Positions(term_vector);
BitVec_Clear_All(other_posit_vec);
for (uint32_t j = I32Arr_Get_Size(positions); j > 0; j--) {
int32_t pos = I32Arr_Get(positions, j - 1) - i;
if (pos >= 0) {
BitVec_Set(other_posit_vec, pos);
}
}
BitVec_And(posit_vec, other_posit_vec);
}
}
// Proceed only if all terms are present.
uint32_t num_tvs = VA_Get_Size(term_vectors);
if (num_tvs == num_terms) {
TermVector *first_tv = (TermVector*)VA_Fetch(term_vectors, 0);
TermVector *last_tv
= (TermVector*)VA_Fetch(term_vectors, num_tvs - 1);
I32Array *tv_start_positions = TV_Get_Positions(first_tv);
I32Array *tv_end_positions = TV_Get_Positions(last_tv);
I32Array *tv_start_offsets = TV_Get_Start_Offsets(first_tv);
I32Array *tv_end_offsets = TV_Get_End_Offsets(last_tv);
uint32_t terms_max = num_terms - 1;
I32Array *valid_posits = BitVec_To_Array(posit_vec);
uint32_t num_valid_posits = I32Arr_Get_Size(valid_posits);
uint32_t j = 0;
float weight = ProximityCompiler_Get_Weight(self);
uint32_t i = 0;
// Add only those starts/ends that belong to a valid position.
for (uint32_t posit_tick = 0; posit_tick < num_valid_posits; posit_tick++) {
int32_t valid_start_posit = I32Arr_Get(valid_posits, posit_tick);
int32_t valid_end_posit = valid_start_posit + terms_max;
int32_t start_offset = 0, end_offset = 0;
for (uint32_t max = I32Arr_Get_Size(tv_start_positions); i < max; i++) {
if (I32Arr_Get(tv_start_positions, i) == valid_start_posit) {
start_offset = I32Arr_Get(tv_start_offsets, i);
break;
}
}
for (uint32_t max = I32Arr_Get_Size(tv_end_positions); j < max; j++) {
if (I32Arr_Get(tv_end_positions, j) == valid_end_posit) {
end_offset = I32Arr_Get(tv_end_offsets, j);
break;
}
}
VA_Push(spans, (Obj*)Span_new(start_offset,
end_offset - start_offset, weight));
i++, j++;
}
DECREF(valid_posits);
}
DECREF(other_posit_vec);
DECREF(posit_vec);
DECREF(term_vectors);
return spans;
}
void
BGMerger_Prepare_Commit_IMP(BackgroundMerger *self) {
BackgroundMergerIVARS *const ivars = BGMerger_IVARS(self);
Vector *seg_readers = PolyReader_Get_Seg_Readers(ivars->polyreader);
uint32_t num_seg_readers = Vec_Get_Size(seg_readers);
uint32_t segs_merged = 0;
if (ivars->prepared) {
THROW(ERR, "Can't call Prepare_Commit() more than once");
}
// Maybe merge existing index data.
if (num_seg_readers) {
segs_merged = S_maybe_merge(self);
}
if (!segs_merged) {
// Nothing merged. Leave `needs_commit` false and bail out.
ivars->prepared = true;
return;
}
// Finish the segment and write a new snapshot file.
else {
Folder *folder = ivars->folder;
Snapshot *snapshot = ivars->snapshot;
// Write out new deletions.
if (DelWriter_Updated(ivars->del_writer)) {
// Only write out if they haven't all been applied.
if (segs_merged != num_seg_readers) {
DelWriter_Finish(ivars->del_writer);
}
}
// Finish the segment.
SegWriter_Finish(ivars->seg_writer);
// Grab the write lock.
S_obtain_write_lock(self);
if (!ivars->write_lock) {
RETHROW(INCREF(Err_get_error()));
}
// Write temporary snapshot file.
DECREF(ivars->snapfile);
String *snapfile = IxManager_Make_Snapshot_Filename(ivars->manager);
ivars->snapfile = Str_Cat_Trusted_Utf8(snapfile, ".temp", 5);
DECREF(snapfile);
Folder_Delete(folder, ivars->snapfile);
Snapshot_Write_File(snapshot, folder, ivars->snapfile);
// Determine whether the index has been updated while this background
// merge process was running.
String *start_snapfile
= Snapshot_Get_Path(PolyReader_Get_Snapshot(ivars->polyreader));
Snapshot *latest_snapshot
= Snapshot_Read_File(Snapshot_new(), ivars->folder, NULL);
String *latest_snapfile = Snapshot_Get_Path(latest_snapshot);
bool index_updated
= !Str_Equals(start_snapfile, (Obj*)latest_snapfile);
if (index_updated) {
/* See if new deletions have been applied since this
* background merge process started against any of the
* segments we just merged away. If that's true, we need to
* write another segment which applies the deletions against
* the new composite segment.
*/
S_merge_updated_deletions(self);
// Add the fresh content to our snapshot. (It's important to
// run this AFTER S_merge_updated_deletions, because otherwise
// we couldn't tell whether the deletion counts changed.)
Vector *files = Snapshot_List(latest_snapshot);
for (uint32_t i = 0, max = Vec_Get_Size(files); i < max; i++) {
String *file = (String*)Vec_Fetch(files, i);
if (Str_Starts_With_Utf8(file, "seg_", 4)) {
int64_t gen = (int64_t)IxFileNames_extract_gen(file);
if (gen > ivars->cutoff) {
Snapshot_Add_Entry(ivars->snapshot, file);
}
}
}
DECREF(files);
// Since the snapshot content has changed, we need to rewrite it.
Folder_Delete(folder, ivars->snapfile);
Snapshot_Write_File(snapshot, folder, ivars->snapfile);
}
DECREF(latest_snapshot);
ivars->needs_commit = true;
}
// Close reader, so that we can delete its files if appropriate.
PolyReader_Close(ivars->polyreader);
ivars->prepared = true;
}
