void
TestQPSyntax_run_tests(Folder *index)
{
u32_t i;
TestBatch *batch = Test_new_batch("TestQueryParserSyntax", 48, NULL);
Searcher *searcher = Searcher_new((Obj*)index);
QueryParser *qparser = QParser_new(Searcher_Get_Schema(searcher),
NULL, NULL, NULL);
QParser_Set_Heed_Colons(qparser, true);
PLAN(batch);
for (i = 0; leaf_test_funcs[i] != NULL; i++) {
kino_TestQPSyntax_test_t test_func = leaf_test_funcs[i];
TestQueryParser *test_case = test_func(i);
Query *tree = QParser_Tree(qparser, test_case->query_string);
Query *expanded = QParser_Expand_Leaf(qparser, test_case->tree);
Query *parsed = QParser_Parse(qparser, test_case->query_string);
Hits *hits = Searcher_Hits(searcher, (Obj*)parsed, 0, 10, NULL);
ASSERT_TRUE(batch, Query_Equals(tree, (Obj*)test_case->tree),
"tree() %s", test_case->query_string->ptr);
ASSERT_TRUE(batch, Query_Equals(expanded, (Obj*)test_case->expanded),
"expand_leaf() %s", test_case->query_string->ptr);
ASSERT_INT_EQ(batch, Hits_Total_Hits(hits), test_case->num_hits,
"hits: %s", test_case->query_string->ptr);
DECREF(hits);
DECREF(parsed);
DECREF(expanded);
DECREF(tree);
DECREF(test_case);
}
for (i = 0; syntax_test_funcs[i] != NULL; i++) {
kino_TestQPSyntax_test_t test_func = syntax_test_funcs[i];
TestQueryParser *test_case = test_func(i);
Query *tree = QParser_Tree(qparser, test_case->query_string);
Query *parsed = QParser_Parse(qparser, test_case->query_string);
Hits *hits = Searcher_Hits(searcher, (Obj*)parsed, 0, 10, NULL);
ASSERT_TRUE(batch, Query_Equals(tree, (Obj*)test_case->tree),
"tree() %s", test_case->query_string->ptr);
ASSERT_INT_EQ(batch, Hits_Total_Hits(hits), test_case->num_hits,
"hits: %s", test_case->query_string->ptr);
DECREF(hits);
DECREF(parsed);
DECREF(tree);
DECREF(test_case);
}
batch->destroy(batch);
DECREF(searcher);
DECREF(qparser);
}
static void
test_highlighting(TestBatchRunner *runner) {
Schema *schema = Schema_new();
StandardTokenizer *tokenizer = StandardTokenizer_new();
FullTextType *plain_type = FullTextType_new((Analyzer*)tokenizer);
FullTextType_Set_Highlightable(plain_type, true);
FullTextType *dunked_type = FullTextType_new((Analyzer*)tokenizer);
FullTextType_Set_Highlightable(dunked_type, true);
FullTextType_Set_Boost(dunked_type, 0.1f);
String *content = (String*)SSTR_WRAP_UTF8("content", 7);
Schema_Spec_Field(schema, content, (FieldType*)plain_type);
String *alt = (String*)SSTR_WRAP_UTF8("alt", 3);
Schema_Spec_Field(schema, alt, (FieldType*)dunked_type);
DECREF(plain_type);
DECREF(dunked_type);
DECREF(tokenizer);
RAMFolder *folder = RAMFolder_new(NULL);
Indexer *indexer = Indexer_new(schema, (Obj*)folder, NULL, 0);
Doc *doc = Doc_new(NULL, 0);
String *string = (String *)SSTR_WRAP_UTF8(TEST_STRING, TEST_STRING_LEN);
Doc_Store(doc, content, (Obj*)string);
Indexer_Add_Doc(indexer, doc, 1.0f);
DECREF(doc);
doc = Doc_new(NULL, 0);
string = (String *)SSTR_WRAP_UTF8("\"I see,\" said the blind man.", 28);
Doc_Store(doc, content, (Obj*)string);
Indexer_Add_Doc(indexer, doc, 1.0f);
DECREF(doc);
doc = Doc_new(NULL, 0);
string = (String *)SSTR_WRAP_UTF8("x but not why or 2ee", 20);
Doc_Store(doc, content, (Obj*)string);
string = (String *)SSTR_WRAP_UTF8(TEST_STRING
" and extra stuff so it scores lower",
TEST_STRING_LEN + 35);
Doc_Store(doc, alt, (Obj*)string);
Indexer_Add_Doc(indexer, doc, 1.0f);
DECREF(doc);
Indexer_Commit(indexer);
DECREF(indexer);
Searcher *searcher = (Searcher*)IxSearcher_new((Obj*)folder);
Obj *query = (Obj*)SSTR_WRAP_UTF8("\"x y z\" AND " PHI, 14);
Hits *hits = Searcher_Hits(searcher, query, 0, 10, NULL);
test_Raw_Excerpt(runner, searcher, query);
test_Highlight_Excerpt(runner, searcher, query);
test_Create_Excerpt(runner, searcher, query, hits);
DECREF(hits);
DECREF(searcher);
DECREF(folder);
DECREF(schema);
}
static void
test_hl_selection(TestBatchRunner *runner) {
Schema *schema = Schema_new();
StandardTokenizer *tokenizer = StandardTokenizer_new();
FullTextType *plain_type = FullTextType_new((Analyzer*)tokenizer);
FullTextType_Set_Highlightable(plain_type, true);
String *content = (String*)SSTR_WRAP_UTF8("content", 7);
Schema_Spec_Field(schema, content, (FieldType*)plain_type);
DECREF(plain_type);
DECREF(tokenizer);
RAMFolder *folder = RAMFolder_new(NULL);
Indexer *indexer = Indexer_new(schema, (Obj*)folder, NULL, 0);
static char test_string[] =
"bla bla bla bla bla bla bla bla bla bla bla bla bla bla bla bla. "
"bla bla bla bla bla bla bla bla bla bla bla bla bla bla bla bla. "
"bla bla bla bla bla bla bla bla bla bla bla bla bla bla bla bla. "
"bla bla bla bla bla bla bla bla bla bla bla bla bla bla bla bla. "
"bla bla bla bla bla bla bla bla bla bla bla bla bla bla bla bla. "
"bla bla bla bla bla bla bla bla bla bla bla bla bla bla bla bla bla bla bla NNN bla. "
"bla bla bla bla bla bla bla bla bla bla bla bla bla bla bla bla bla bla bla bla bla bla bla bla bla bla bla bla bla bla bla bla bla bla bla. "
"bla bla bla MMM bla bla bla bla bla bla bla bla bla bla bla bla. "
"bla bla bla bla bla bla bla bla bla bla bla bla bla bla bla bla. "
"bla bla bla bla bla bla bla bla bla bla bla bla bla bla bla bla. "
"bla bla bla bla bla bla bla bla bla bla bla bla bla bla bla bla. "
"bla bla bla bla bla bla bla bla bla bla bla bla bla bla bla bla. "
"bla bla bla bla bla bla bla bla bla bla bla bla bla bla bla bla. ";
Doc *doc = Doc_new(NULL, 0);
String *string = (String *)SSTR_WRAP_UTF8(test_string, strlen(test_string));
Doc_Store(doc, content, (Obj*)string);
Indexer_Add_Doc(indexer, doc, 1.0f);
DECREF(doc);
Indexer_Commit(indexer);
DECREF(indexer);
Searcher *searcher = (Searcher*)IxSearcher_new((Obj*)folder);
Obj *query = (Obj*)SSTR_WRAP_UTF8("NNN MMM", 7);
Highlighter *highlighter = Highlighter_new(searcher, query, content, 200);
Hits *hits = Searcher_Hits(searcher, query, 0, 10, NULL);
HitDoc *hit = Hits_Next(hits);
String *excerpt = Highlighter_Create_Excerpt(highlighter, hit);
String *mmm = (String*)SSTR_WRAP_UTF8("MMM", 3);
String *nnn = (String*)SSTR_WRAP_UTF8("NNN", 3);
TEST_TRUE(runner, Str_Find(excerpt, mmm) >= 0 || Str_Find(excerpt, nnn) >= 0,
"Sentence boundary algo doesn't chop terms");
DECREF(excerpt);
DECREF(hit);
DECREF(hits);
DECREF(highlighter);
DECREF(searcher);
DECREF(folder);
DECREF(schema);
}
static void
test_Create_Excerpt(TestBatchRunner *runner, Searcher *searcher, Obj *query,
Hits *hits) {
String *content = (String*)SSTR_WRAP_UTF8("content", 7);
Highlighter *highlighter = Highlighter_new(searcher, query, content, 200);
HitDoc *hit = Hits_Next(hits);
String *excerpt = Highlighter_Create_Excerpt(highlighter, hit);
TEST_TRUE(runner,
Str_Find_Utf8(excerpt,
"<strong>Φ</strong> a b c d <strong>x y z</strong>",
54) >= 0,
"highlighter tagged phrase and single term");
DECREF(excerpt);
String *pre_tag = (String*)SSTR_WRAP_UTF8("\x1B[1m", 4);
Highlighter_Set_Pre_Tag(highlighter, pre_tag);
String *post_tag = (String*)SSTR_WRAP_UTF8("\x1B[0m", 4);
Highlighter_Set_Post_Tag(highlighter, post_tag);
excerpt = Highlighter_Create_Excerpt(highlighter, hit);
TEST_TRUE(runner,
Str_Find_Utf8(excerpt,
"\x1B[1mΦ\x1B[0m a b c d \x1B[1mx y z\x1B[0m",
36) >= 0,
"set_pre_tag and set_post_tag");
DECREF(excerpt);
DECREF(hit);
hit = Hits_Next(hits);
excerpt = Highlighter_Create_Excerpt(highlighter, hit);
TEST_TRUE(runner,
Str_Find_Utf8(excerpt, "x", 1) >= 0,
"excerpt field with partial hit doesn't cause highlighter freakout");
DECREF(excerpt);
DECREF(hit);
DECREF(highlighter);
query = (Obj*)SSTR_WRAP_UTF8("x \"x y z\" AND b", 15);
hits = Searcher_Hits(searcher, query, 0, 10, NULL);
highlighter = Highlighter_new(searcher, query, content, 200);
hit = Hits_Next(hits);
excerpt = Highlighter_Create_Excerpt(highlighter, hit);
TEST_TRUE(runner,
Str_Find_Utf8(excerpt,
"<strong>b</strong> c d <strong>x y z</strong>",
45) >= 0,
"query with same word in both phrase and term doesn't cause freakout");
DECREF(excerpt);
DECREF(hit);
DECREF(highlighter);
DECREF(hits);
query = (Obj*)SSTR_WRAP_UTF8("blind", 5);
hits = Searcher_Hits(searcher, query, 0, 10, NULL);
highlighter = Highlighter_new(searcher, query, content, 200);
hit = Hits_Next(hits);
excerpt = Highlighter_Create_Excerpt(highlighter, hit);
TEST_TRUE(runner,
Str_Find_Utf8(excerpt, """, 6) >= 0,
"HTML entity encoded properly");
DECREF(excerpt);
DECREF(hit);
DECREF(highlighter);
DECREF(hits);
query = (Obj*)SSTR_WRAP_UTF8("why", 3);
hits = Searcher_Hits(searcher, query, 0, 10, NULL);
highlighter = Highlighter_new(searcher, query, content, 200);
hit = Hits_Next(hits);
excerpt = Highlighter_Create_Excerpt(highlighter, hit);
TEST_TRUE(runner,
Str_Find_Utf8(excerpt, "Φ", 6) == -1,
"no ellipsis for short excerpt");
DECREF(excerpt);
DECREF(hit);
DECREF(highlighter);
DECREF(hits);
Obj *term = (Obj*)SSTR_WRAP_UTF8("x", 1);
query = (Obj*)TermQuery_new(content, term);
hits = Searcher_Hits(searcher, query, 0, 10, NULL);
hit = Hits_Next(hits);
highlighter = Highlighter_new(searcher, query, content, 200);
excerpt = Highlighter_Create_Excerpt(highlighter, hit);
TEST_TRUE(runner,
Str_Find_Utf8(excerpt, "strong", 5) >= 0,
"specify field highlights correct field...");
DECREF(excerpt);
DECREF(highlighter);
String *alt = (String*)SSTR_WRAP_UTF8("alt", 3);
highlighter = Highlighter_new(searcher, query, alt, 200);
excerpt = Highlighter_Create_Excerpt(highlighter, hit);
TEST_TRUE(runner,
Str_Find_Utf8(excerpt, "strong", 5) == -1,
"... but not another field");
DECREF(excerpt);
DECREF(highlighter);
DECREF(hit);
DECREF(hits);
DECREF(query);
}
void
TestQPLogic_run_tests()
{
u32_t i;
TestBatch *batch = Test_new_batch("TestQueryParserLogic", 178, NULL);
Folder *folder = S_create_index();
Searcher *searcher = Searcher_new((Obj*)folder);
QueryParser *or_parser = QParser_new(Searcher_Get_Schema(searcher),
NULL, NULL, NULL);
static ZombieCharBuf AND = ZCB_LITERAL("AND");
QueryParser *and_parser = QParser_new(Searcher_Get_Schema(searcher),
NULL, (CharBuf*)&AND, NULL);
QParser_Set_Heed_Colons(or_parser, true);
QParser_Set_Heed_Colons(and_parser, true);
PLAN(batch);
/* Run logical tests with default boolop of OR. */
for (i = 0; logical_test_funcs[i] != NULL; i++) {
kino_TestQPLogic_logical_test_t test_func = logical_test_funcs[i];
TestQueryParser *test_case = test_func(BOOLOP_OR);
Query *tree = QParser_Tree(or_parser, test_case->query_string);
Query *parsed = QParser_Parse(or_parser, test_case->query_string);
Hits *hits = Searcher_Hits(searcher, (Obj*)parsed, 0, 10, NULL);
ASSERT_TRUE(batch, Query_Equals(tree, (Obj*)test_case->tree),
"tree() OR %s", test_case->query_string->ptr);
ASSERT_INT_EQ(batch, Hits_Total_Hits(hits), test_case->num_hits,
"hits: OR %s", test_case->query_string->ptr);
DECREF(hits);
DECREF(parsed);
DECREF(tree);
DECREF(test_case);
}
/* Run logical tests with default boolop of AND. */
for (i = 0; logical_test_funcs[i] != NULL; i++) {
kino_TestQPLogic_logical_test_t test_func = logical_test_funcs[i];
TestQueryParser *test_case = test_func(BOOLOP_AND);
Query *tree = QParser_Tree(and_parser, test_case->query_string);
Query *parsed = QParser_Parse(and_parser, test_case->query_string);
Hits *hits = Searcher_Hits(searcher, (Obj*)parsed, 0, 10, NULL);
ASSERT_TRUE(batch, Query_Equals(tree, (Obj*)test_case->tree),
"tree() AND %s", test_case->query_string->ptr);
ASSERT_INT_EQ(batch, Hits_Total_Hits(hits), test_case->num_hits,
"hits: AND %s", test_case->query_string->ptr);
DECREF(hits);
DECREF(parsed);
DECREF(tree);
DECREF(test_case);
}
/* Run tests for QParser_Prune(). */
for (i = 0; prune_test_funcs[i] != NULL; i++) {
kino_TestQPLogic_prune_test_t test_func = prune_test_funcs[i];
TestQueryParser *test_case = test_func();
CharBuf *qstring = test_case->tree
? Obj_To_String(test_case->tree)
: CB_new_from_trusted_utf8("(NULL)", 6);
Query *tree = test_case->tree;
Query *wanted = test_case->expanded;
Query *pruned = QParser_Prune(or_parser, tree);
Query *expanded;
Hits *hits;
ASSERT_TRUE(batch, Query_Equals(pruned, (Obj*)wanted),
"prune() %s", qstring->ptr);
expanded = QParser_Expand(or_parser, pruned);
hits = Searcher_Hits(searcher, (Obj*)expanded, 0, 10, NULL);
ASSERT_INT_EQ(batch, Hits_Total_Hits(hits), test_case->num_hits,
"hits: %s", qstring->ptr);
DECREF(hits);
DECREF(expanded);
DECREF(pruned);
DECREF(qstring);
DECREF(test_case);
}
DECREF(and_parser);
DECREF(or_parser);
DECREF(searcher);
DECREF(folder);
batch->destroy(batch);
}
