static void
test_highlighting(TestBatchRunner *runner) {
Schema *schema = Schema_new();
StandardTokenizer *tokenizer = StandardTokenizer_new();
FullTextType *plain_type = FullTextType_new((Analyzer*)tokenizer);
FullTextType_Set_Highlightable(plain_type, true);
FullTextType *dunked_type = FullTextType_new((Analyzer*)tokenizer);
FullTextType_Set_Highlightable(dunked_type, true);
FullTextType_Set_Boost(dunked_type, 0.1f);
String *content = (String*)SSTR_WRAP_UTF8("content", 7);
Schema_Spec_Field(schema, content, (FieldType*)plain_type);
String *alt = (String*)SSTR_WRAP_UTF8("alt", 3);
Schema_Spec_Field(schema, alt, (FieldType*)dunked_type);
DECREF(plain_type);
DECREF(dunked_type);
DECREF(tokenizer);
RAMFolder *folder = RAMFolder_new(NULL);
Indexer *indexer = Indexer_new(schema, (Obj*)folder, NULL, 0);
Doc *doc = Doc_new(NULL, 0);
String *string = (String *)SSTR_WRAP_UTF8(TEST_STRING, TEST_STRING_LEN);
Doc_Store(doc, content, (Obj*)string);
Indexer_Add_Doc(indexer, doc, 1.0f);
DECREF(doc);
doc = Doc_new(NULL, 0);
string = (String *)SSTR_WRAP_UTF8("\"I see,\" said the blind man.", 28);
Doc_Store(doc, content, (Obj*)string);
Indexer_Add_Doc(indexer, doc, 1.0f);
DECREF(doc);
doc = Doc_new(NULL, 0);
string = (String *)SSTR_WRAP_UTF8("x but not why or 2ee", 20);
Doc_Store(doc, content, (Obj*)string);
string = (String *)SSTR_WRAP_UTF8(TEST_STRING
" and extra stuff so it scores lower",
TEST_STRING_LEN + 35);
Doc_Store(doc, alt, (Obj*)string);
Indexer_Add_Doc(indexer, doc, 1.0f);
DECREF(doc);
Indexer_Commit(indexer);
DECREF(indexer);
Searcher *searcher = (Searcher*)IxSearcher_new((Obj*)folder);
Obj *query = (Obj*)SSTR_WRAP_UTF8("\"x y z\" AND " PHI, 14);
Hits *hits = Searcher_Hits(searcher, query, 0, 10, NULL);
test_Raw_Excerpt(runner, searcher, query);
test_Highlight_Excerpt(runner, searcher, query);
test_Create_Excerpt(runner, searcher, query, hits);
DECREF(hits);
DECREF(searcher);
DECREF(folder);
DECREF(schema);
}
static void
test_hl_selection(TestBatchRunner *runner) {
Schema *schema = Schema_new();
StandardTokenizer *tokenizer = StandardTokenizer_new();
FullTextType *plain_type = FullTextType_new((Analyzer*)tokenizer);
FullTextType_Set_Highlightable(plain_type, true);
String *content = (String*)SSTR_WRAP_UTF8("content", 7);
Schema_Spec_Field(schema, content, (FieldType*)plain_type);
DECREF(plain_type);
DECREF(tokenizer);
RAMFolder *folder = RAMFolder_new(NULL);
Indexer *indexer = Indexer_new(schema, (Obj*)folder, NULL, 0);
static char test_string[] =
"bla bla bla bla bla bla bla bla bla bla bla bla bla bla bla bla. "
"bla bla bla bla bla bla bla bla bla bla bla bla bla bla bla bla. "
"bla bla bla bla bla bla bla bla bla bla bla bla bla bla bla bla. "
"bla bla bla bla bla bla bla bla bla bla bla bla bla bla bla bla. "
"bla bla bla bla bla bla bla bla bla bla bla bla bla bla bla bla. "
"bla bla bla bla bla bla bla bla bla bla bla bla bla bla bla bla bla bla bla NNN bla. "
"bla bla bla bla bla bla bla bla bla bla bla bla bla bla bla bla bla bla bla bla bla bla bla bla bla bla bla bla bla bla bla bla bla bla bla. "
"bla bla bla MMM bla bla bla bla bla bla bla bla bla bla bla bla. "
"bla bla bla bla bla bla bla bla bla bla bla bla bla bla bla bla. "
"bla bla bla bla bla bla bla bla bla bla bla bla bla bla bla bla. "
"bla bla bla bla bla bla bla bla bla bla bla bla bla bla bla bla. "
"bla bla bla bla bla bla bla bla bla bla bla bla bla bla bla bla. "
"bla bla bla bla bla bla bla bla bla bla bla bla bla bla bla bla. ";
Doc *doc = Doc_new(NULL, 0);
String *string = (String *)SSTR_WRAP_UTF8(test_string, strlen(test_string));
Doc_Store(doc, content, (Obj*)string);
Indexer_Add_Doc(indexer, doc, 1.0f);
DECREF(doc);
Indexer_Commit(indexer);
DECREF(indexer);
Searcher *searcher = (Searcher*)IxSearcher_new((Obj*)folder);
Obj *query = (Obj*)SSTR_WRAP_UTF8("NNN MMM", 7);
Highlighter *highlighter = Highlighter_new(searcher, query, content, 200);
Hits *hits = Searcher_Hits(searcher, query, 0, 10, NULL);
HitDoc *hit = Hits_Next(hits);
String *excerpt = Highlighter_Create_Excerpt(highlighter, hit);
String *mmm = (String*)SSTR_WRAP_UTF8("MMM", 3);
String *nnn = (String*)SSTR_WRAP_UTF8("NNN", 3);
TEST_TRUE(runner, Str_Find(excerpt, mmm) >= 0 || Str_Find(excerpt, nnn) >= 0,
"Sentence boundary algo doesn't chop terms");
DECREF(excerpt);
DECREF(hit);
DECREF(hits);
DECREF(highlighter);
DECREF(searcher);
DECREF(folder);
DECREF(schema);
}
void
TestQPSyntax_run_tests(Folder *index)
{
uint32_t i;
TestBatch *batch = TestBatch_new(58);
IndexSearcher *searcher = IxSearcher_new((Obj*)index);
QueryParser *qparser = QParser_new(IxSearcher_Get_Schema(searcher),
NULL, NULL, NULL);
QParser_Set_Heed_Colons(qparser, true);
TestBatch_Plan(batch);
for (i = 0; leaf_test_funcs[i] != NULL; i++) {
kino_TestQPSyntax_test_t test_func = leaf_test_funcs[i];
TestQueryParser *test_case = test_func();
Query *tree = QParser_Tree(qparser, test_case->query_string);
Query *expanded = QParser_Expand_Leaf(qparser, test_case->tree);
Query *parsed = QParser_Parse(qparser, test_case->query_string);
Hits *hits = IxSearcher_Hits(searcher, (Obj*)parsed, 0, 10, NULL);
TEST_TRUE(batch, Query_Equals(tree, (Obj*)test_case->tree),
"tree() %s", (char*)CB_Get_Ptr8(test_case->query_string));
TEST_TRUE(batch, Query_Equals(expanded, (Obj*)test_case->expanded),
"expand_leaf() %s", (char*)CB_Get_Ptr8(test_case->query_string));
TEST_INT_EQ(batch, Hits_Total_Hits(hits), test_case->num_hits,
"hits: %s", (char*)CB_Get_Ptr8(test_case->query_string));
DECREF(hits);
DECREF(parsed);
DECREF(expanded);
DECREF(tree);
DECREF(test_case);
}
for (i = 0; syntax_test_funcs[i] != NULL; i++) {
kino_TestQPSyntax_test_t test_func = syntax_test_funcs[i];
TestQueryParser *test_case = test_func();
Query *tree = QParser_Tree(qparser, test_case->query_string);
Query *parsed = QParser_Parse(qparser, test_case->query_string);
Hits *hits = IxSearcher_Hits(searcher, (Obj*)parsed, 0, 10, NULL);
TEST_TRUE(batch, Query_Equals(tree, (Obj*)test_case->tree),
"tree() %s", (char*)CB_Get_Ptr8(test_case->query_string));
TEST_INT_EQ(batch, Hits_Total_Hits(hits), test_case->num_hits,
"hits: %s", (char*)CB_Get_Ptr8(test_case->query_string));
DECREF(hits);
DECREF(parsed);
DECREF(tree);
DECREF(test_case);
}
DECREF(batch);
DECREF(searcher);
DECREF(qparser);
}
DefaultDeletionsWriter*
DefDelWriter_init(DefaultDeletionsWriter *self, Schema *schema,
Snapshot *snapshot, Segment *segment,
PolyReader *polyreader) {
DataWriter_init((DataWriter*)self, schema, snapshot, segment, polyreader);
DefaultDeletionsWriterIVARS *const ivars = DefDelWriter_IVARS(self);
ivars->seg_readers = PolyReader_Seg_Readers(polyreader);
uint32_t num_seg_readers = VA_Get_Size(ivars->seg_readers);
ivars->seg_starts = PolyReader_Offsets(polyreader);
ivars->bit_vecs = VA_new(num_seg_readers);
ivars->updated = (bool*)CALLOCATE(num_seg_readers, sizeof(bool));
ivars->searcher = IxSearcher_new((Obj*)polyreader);
ivars->name_to_tick = Hash_new(num_seg_readers);
// Materialize a BitVector of deletions for each segment.
for (uint32_t i = 0; i < num_seg_readers; i++) {
SegReader *seg_reader = (SegReader*)VA_Fetch(ivars->seg_readers, i);
BitVector *bit_vec = BitVec_new(SegReader_Doc_Max(seg_reader));
DeletionsReader *del_reader
= (DeletionsReader*)SegReader_Fetch(
seg_reader, Class_Get_Name(DELETIONSREADER));
Matcher *seg_dels = del_reader
? DelReader_Iterator(del_reader)
: NULL;
if (seg_dels) {
int32_t del;
while (0 != (del = Matcher_Next(seg_dels))) {
BitVec_Set(bit_vec, del);
}
DECREF(seg_dels);
}
VA_Store(ivars->bit_vecs, i, (Obj*)bit_vec);
Hash_Store(ivars->name_to_tick,
(Obj*)SegReader_Get_Seg_Name(seg_reader),
(Obj*)Int32_new(i));
}
return self;
}
int
main() {
// Initialize the library.
lucy_bootstrap_parcel();
Schema *schema = S_create_schema();
String *folder = Str_newf("lucy_index");
S_index_documents(schema, folder);
IndexSearcher *searcher = IxSearcher_new((Obj*)folder);
S_search(searcher, "ullamco");
S_search(searcher, "ut OR laborum");
S_search(searcher, "\"fugiat nulla\"");
DECREF(schema);
DECREF(folder);
DECREF(searcher);
return 0;
}
void
TestQPLogic_Run_IMP(TestQueryParserLogic *self, TestBatchRunner *runner) {
TestBatchRunner_Plan(runner, (TestBatch*)self, 258);
uint32_t i;
Folder *folder = S_create_index();
IndexSearcher *searcher = IxSearcher_new((Obj*)folder);
QueryParser *or_parser = QParser_new(IxSearcher_Get_Schema(searcher),
NULL, NULL, NULL);
String *AND = SSTR_WRAP_C("AND");
QueryParser *and_parser = QParser_new(IxSearcher_Get_Schema(searcher),
NULL, AND, NULL);
QParser_Set_Heed_Colons(or_parser, true);
QParser_Set_Heed_Colons(and_parser, true);
// Run logical tests with default boolop of OR.
for (i = 0; logical_test_funcs[i] != NULL; i++) {
LUCY_TestQPLogic_Logical_Test_t test_func = logical_test_funcs[i];
TestQueryParser *test_case_obj = test_func(BOOLOP_OR);
TestQueryParserIVARS *test_case = TestQP_IVARS(test_case_obj);
Query *tree = QParser_Tree(or_parser, test_case->query_string);
Query *parsed = QParser_Parse(or_parser, test_case->query_string);
Hits *hits = IxSearcher_Hits(searcher, (Obj*)parsed, 0, 10, NULL);
char *qstr = Str_To_Utf8(test_case->query_string);
TEST_TRUE(runner, Query_Equals(tree, (Obj*)test_case->tree),
"tree() OR %s", qstr);
TEST_INT_EQ(runner, Hits_Total_Hits(hits), test_case->num_hits,
"hits: OR %s", qstr);
free(qstr);
DECREF(hits);
DECREF(parsed);
DECREF(tree);
DECREF(test_case_obj);
}
// Run logical tests with default boolop of AND.
for (i = 0; logical_test_funcs[i] != NULL; i++) {
LUCY_TestQPLogic_Logical_Test_t test_func = logical_test_funcs[i];
TestQueryParser *test_case_obj = test_func(BOOLOP_AND);
TestQueryParserIVARS *test_case = TestQP_IVARS(test_case_obj);
Query *tree = QParser_Tree(and_parser, test_case->query_string);
Query *parsed = QParser_Parse(and_parser, test_case->query_string);
Hits *hits = IxSearcher_Hits(searcher, (Obj*)parsed, 0, 10, NULL);
char *qstr = Str_To_Utf8(test_case->query_string);
TEST_TRUE(runner, Query_Equals(tree, (Obj*)test_case->tree),
"tree() AND %s", qstr);
TEST_INT_EQ(runner, Hits_Total_Hits(hits), test_case->num_hits,
"hits: AND %s", qstr);
free(qstr);
DECREF(hits);
DECREF(parsed);
DECREF(tree);
DECREF(test_case_obj);
}
// Run tests for QParser_Prune().
for (i = 0; prune_test_funcs[i] != NULL; i++) {
LUCY_TestQPLogic_Prune_Test_t test_func = prune_test_funcs[i];
TestQueryParser *test_case_obj = test_func();
TestQueryParserIVARS *test_case = TestQP_IVARS(test_case_obj);
String *qstring = test_case->tree
? Query_To_String(test_case->tree)
: Str_new_from_trusted_utf8("(NULL)", 6);
char *qstr = Str_To_Utf8(qstring);
Query *tree = test_case->tree;
Query *wanted = test_case->expanded;
Query *pruned = QParser_Prune(or_parser, tree);
Query *expanded;
Hits *hits;
TEST_TRUE(runner, Query_Equals(pruned, (Obj*)wanted),
"prune() %s", qstr);
expanded = QParser_Expand(or_parser, pruned);
hits = IxSearcher_Hits(searcher, (Obj*)expanded, 0, 10, NULL);
TEST_INT_EQ(runner, Hits_Total_Hits(hits), test_case->num_hits,
"hits: %s", qstr);
free(qstr);
DECREF(hits);
DECREF(expanded);
DECREF(pruned);
DECREF(qstring);
DECREF(test_case_obj);
}
DECREF(and_parser);
DECREF(or_parser);
DECREF(searcher);
DECREF(folder);
}
static void
test_query_parser_syntax(TestBatchRunner *runner) {
if (!RegexTokenizer_is_available()) {
for (uint32_t i = 0; leaf_test_funcs[i] != NULL; i++) {
SKIP(runner, 3, "RegexTokenizer not available");
}
for (uint32_t i = 0; syntax_test_funcs[i] != NULL; i++) {
SKIP(runner, 2, "RegexTokenizer not available");
}
return;
}
Folder *index = build_index();
IndexSearcher *searcher = IxSearcher_new((Obj*)index);
QueryParser *qparser = QParser_new(IxSearcher_Get_Schema(searcher),
NULL, NULL, NULL);
QParser_Set_Heed_Colons(qparser, true);
for (uint32_t i = 0; leaf_test_funcs[i] != NULL; i++) {
LUCY_TestQPSyntax_Test_t test_func = leaf_test_funcs[i];
TestQueryParser *test_case = test_func();
TestQueryParserIVARS *ivars = TestQP_IVARS(test_case);
Query *tree = QParser_Tree(qparser, ivars->query_string);
Query *expanded = QParser_Expand_Leaf(qparser, ivars->tree);
Query *parsed = QParser_Parse(qparser, ivars->query_string);
Hits *hits = IxSearcher_Hits(searcher, (Obj*)parsed, 0, 10, NULL);
TEST_TRUE(runner, Query_Equals(tree, (Obj*)ivars->tree),
"tree() %s", Str_Get_Ptr8(ivars->query_string));
TEST_TRUE(runner, Query_Equals(expanded, (Obj*)ivars->expanded),
"expand_leaf() %s", Str_Get_Ptr8(ivars->query_string));
TEST_INT_EQ(runner, Hits_Total_Hits(hits), ivars->num_hits,
"hits: %s", Str_Get_Ptr8(ivars->query_string));
DECREF(hits);
DECREF(parsed);
DECREF(expanded);
DECREF(tree);
DECREF(test_case);
}
for (uint32_t i = 0; syntax_test_funcs[i] != NULL; i++) {
LUCY_TestQPSyntax_Test_t test_func = syntax_test_funcs[i];
TestQueryParser *test_case = test_func();
TestQueryParserIVARS *ivars = TestQP_IVARS(test_case);
Query *tree = QParser_Tree(qparser, ivars->query_string);
Query *parsed = QParser_Parse(qparser, ivars->query_string);
Hits *hits = IxSearcher_Hits(searcher, (Obj*)parsed, 0, 10, NULL);
TEST_TRUE(runner, Query_Equals(tree, (Obj*)ivars->tree),
"tree() %s", Str_Get_Ptr8(ivars->query_string));
TEST_INT_EQ(runner, Hits_Total_Hits(hits), ivars->num_hits,
"hits: %s", Str_Get_Ptr8(ivars->query_string));
DECREF(hits);
DECREF(parsed);
DECREF(tree);
DECREF(test_case);
}
DECREF(searcher);
DECREF(qparser);
DECREF(index);
}
void
TestQPLogic_run_tests() {
uint32_t i;
TestBatch *batch = TestBatch_new(258);
Folder *folder = S_create_index();
IndexSearcher *searcher = IxSearcher_new((Obj*)folder);
QueryParser *or_parser = QParser_new(IxSearcher_Get_Schema(searcher),
NULL, NULL, NULL);
ZombieCharBuf *AND = ZCB_WRAP_STR("AND", 3);
QueryParser *and_parser = QParser_new(IxSearcher_Get_Schema(searcher),
NULL, (CharBuf*)AND, NULL);
QParser_Set_Heed_Colons(or_parser, true);
QParser_Set_Heed_Colons(and_parser, true);
TestBatch_Plan(batch);
// Run logical tests with default boolop of OR.
for (i = 0; logical_test_funcs[i] != NULL; i++) {
Lucy_TestQPLogic_Logical_Test_t test_func = logical_test_funcs[i];
TestQueryParser *test_case = test_func(BOOLOP_OR);
Query *tree = QParser_Tree(or_parser, test_case->query_string);
Query *parsed = QParser_Parse(or_parser, test_case->query_string);
Hits *hits = IxSearcher_Hits(searcher, (Obj*)parsed, 0, 10, NULL);
TEST_TRUE(batch, Query_Equals(tree, (Obj*)test_case->tree),
"tree() OR %s", (char*)CB_Get_Ptr8(test_case->query_string));
TEST_INT_EQ(batch, Hits_Total_Hits(hits), test_case->num_hits,
"hits: OR %s", (char*)CB_Get_Ptr8(test_case->query_string));
DECREF(hits);
DECREF(parsed);
DECREF(tree);
DECREF(test_case);
}
// Run logical tests with default boolop of AND.
for (i = 0; logical_test_funcs[i] != NULL; i++) {
Lucy_TestQPLogic_Logical_Test_t test_func = logical_test_funcs[i];
TestQueryParser *test_case = test_func(BOOLOP_AND);
Query *tree = QParser_Tree(and_parser, test_case->query_string);
Query *parsed = QParser_Parse(and_parser, test_case->query_string);
Hits *hits = IxSearcher_Hits(searcher, (Obj*)parsed, 0, 10, NULL);
TEST_TRUE(batch, Query_Equals(tree, (Obj*)test_case->tree),
"tree() AND %s", (char*)CB_Get_Ptr8(test_case->query_string));
TEST_INT_EQ(batch, Hits_Total_Hits(hits), test_case->num_hits,
"hits: AND %s", (char*)CB_Get_Ptr8(test_case->query_string));
DECREF(hits);
DECREF(parsed);
DECREF(tree);
DECREF(test_case);
}
// Run tests for QParser_Prune().
for (i = 0; prune_test_funcs[i] != NULL; i++) {
Lucy_TestQPLogic_Prune_Test_t test_func = prune_test_funcs[i];
TestQueryParser *test_case = test_func();
CharBuf *qstring = test_case->tree
? Query_To_String(test_case->tree)
: CB_new_from_trusted_utf8("(NULL)", 6);
Query *tree = test_case->tree;
Query *wanted = test_case->expanded;
Query *pruned = QParser_Prune(or_parser, tree);
Query *expanded;
Hits *hits;
TEST_TRUE(batch, Query_Equals(pruned, (Obj*)wanted),
"prune() %s", (char*)CB_Get_Ptr8(qstring));
expanded = QParser_Expand(or_parser, pruned);
hits = IxSearcher_Hits(searcher, (Obj*)expanded, 0, 10, NULL);
TEST_INT_EQ(batch, Hits_Total_Hits(hits), test_case->num_hits,
"hits: %s", (char*)CB_Get_Ptr8(qstring));
DECREF(hits);
DECREF(expanded);
DECREF(pruned);
DECREF(qstring);
DECREF(test_case);
}
DECREF(and_parser);
DECREF(or_parser);
DECREF(searcher);
DECREF(folder);
DECREF(batch);
}
