static void test_highlighting(TestBatchRunner *runner) { Schema *schema = Schema_new(); StandardTokenizer *tokenizer = StandardTokenizer_new(); FullTextType *plain_type = FullTextType_new((Analyzer*)tokenizer); FullTextType_Set_Highlightable(plain_type, true); FullTextType *dunked_type = FullTextType_new((Analyzer*)tokenizer); FullTextType_Set_Highlightable(dunked_type, true); FullTextType_Set_Boost(dunked_type, 0.1f); String *content = (String*)SSTR_WRAP_UTF8("content", 7); Schema_Spec_Field(schema, content, (FieldType*)plain_type); String *alt = (String*)SSTR_WRAP_UTF8("alt", 3); Schema_Spec_Field(schema, alt, (FieldType*)dunked_type); DECREF(plain_type); DECREF(dunked_type); DECREF(tokenizer); RAMFolder *folder = RAMFolder_new(NULL); Indexer *indexer = Indexer_new(schema, (Obj*)folder, NULL, 0); Doc *doc = Doc_new(NULL, 0); String *string = (String *)SSTR_WRAP_UTF8(TEST_STRING, TEST_STRING_LEN); Doc_Store(doc, content, (Obj*)string); Indexer_Add_Doc(indexer, doc, 1.0f); DECREF(doc); doc = Doc_new(NULL, 0); string = (String *)SSTR_WRAP_UTF8("\"I see,\" said the blind man.", 28); Doc_Store(doc, content, (Obj*)string); Indexer_Add_Doc(indexer, doc, 1.0f); DECREF(doc); doc = Doc_new(NULL, 0); string = (String *)SSTR_WRAP_UTF8("x but not why or 2ee", 20); Doc_Store(doc, content, (Obj*)string); string = (String *)SSTR_WRAP_UTF8(TEST_STRING " and extra stuff so it scores lower", TEST_STRING_LEN + 35); Doc_Store(doc, alt, (Obj*)string); Indexer_Add_Doc(indexer, doc, 1.0f); DECREF(doc); Indexer_Commit(indexer); DECREF(indexer); Searcher *searcher = (Searcher*)IxSearcher_new((Obj*)folder); Obj *query = (Obj*)SSTR_WRAP_UTF8("\"x y z\" AND " PHI, 14); Hits *hits = Searcher_Hits(searcher, query, 0, 10, NULL); test_Raw_Excerpt(runner, searcher, query); test_Highlight_Excerpt(runner, searcher, query); test_Create_Excerpt(runner, searcher, query, hits); DECREF(hits); DECREF(searcher); DECREF(folder); DECREF(schema); }
static void test_hl_selection(TestBatchRunner *runner) { Schema *schema = Schema_new(); StandardTokenizer *tokenizer = StandardTokenizer_new(); FullTextType *plain_type = FullTextType_new((Analyzer*)tokenizer); FullTextType_Set_Highlightable(plain_type, true); String *content = (String*)SSTR_WRAP_UTF8("content", 7); Schema_Spec_Field(schema, content, (FieldType*)plain_type); DECREF(plain_type); DECREF(tokenizer); RAMFolder *folder = RAMFolder_new(NULL); Indexer *indexer = Indexer_new(schema, (Obj*)folder, NULL, 0); static char test_string[] = "bla bla bla bla bla bla bla bla bla bla bla bla bla bla bla bla. " "bla bla bla bla bla bla bla bla bla bla bla bla bla bla bla bla. " "bla bla bla bla bla bla bla bla bla bla bla bla bla bla bla bla. " "bla bla bla bla bla bla bla bla bla bla bla bla bla bla bla bla. " "bla bla bla bla bla bla bla bla bla bla bla bla bla bla bla bla. " "bla bla bla bla bla bla bla bla bla bla bla bla bla bla bla bla bla bla bla NNN bla. " "bla bla bla bla bla bla bla bla bla bla bla bla bla bla bla bla bla bla bla bla bla bla bla bla bla bla bla bla bla bla bla bla bla bla bla. " "bla bla bla MMM bla bla bla bla bla bla bla bla bla bla bla bla. " "bla bla bla bla bla bla bla bla bla bla bla bla bla bla bla bla. " "bla bla bla bla bla bla bla bla bla bla bla bla bla bla bla bla. " "bla bla bla bla bla bla bla bla bla bla bla bla bla bla bla bla. " "bla bla bla bla bla bla bla bla bla bla bla bla bla bla bla bla. " "bla bla bla bla bla bla bla bla bla bla bla bla bla bla bla bla. "; Doc *doc = Doc_new(NULL, 0); String *string = (String *)SSTR_WRAP_UTF8(test_string, strlen(test_string)); Doc_Store(doc, content, (Obj*)string); Indexer_Add_Doc(indexer, doc, 1.0f); DECREF(doc); Indexer_Commit(indexer); DECREF(indexer); Searcher *searcher = (Searcher*)IxSearcher_new((Obj*)folder); Obj *query = (Obj*)SSTR_WRAP_UTF8("NNN MMM", 7); Highlighter *highlighter = Highlighter_new(searcher, query, content, 200); Hits *hits = Searcher_Hits(searcher, query, 0, 10, NULL); HitDoc *hit = Hits_Next(hits); String *excerpt = Highlighter_Create_Excerpt(highlighter, hit); String *mmm = (String*)SSTR_WRAP_UTF8("MMM", 3); String *nnn = (String*)SSTR_WRAP_UTF8("NNN", 3); TEST_TRUE(runner, Str_Find(excerpt, mmm) >= 0 || Str_Find(excerpt, nnn) >= 0, "Sentence boundary algo doesn't chop terms"); DECREF(excerpt); DECREF(hit); DECREF(hits); DECREF(highlighter); DECREF(searcher); DECREF(folder); DECREF(schema); }
EasyAnalyzer* EasyAnalyzer_init(EasyAnalyzer *self, const CharBuf *language) { Analyzer_init((Analyzer*)self); EasyAnalyzerIVARS *const ivars = EasyAnalyzer_IVARS(self); ivars->language = CB_Clone(language); ivars->tokenizer = StandardTokenizer_new(); ivars->normalizer = Normalizer_new(NULL, true, false); ivars->stemmer = SnowStemmer_new(language); return self; }
static void test_Dump_Load_and_Equals(TestBatchRunner *runner) { StandardTokenizer *tokenizer = StandardTokenizer_new(); Normalizer *normalizer = Normalizer_new(NULL, true, false); FullTextType *type = FullTextType_new((Analyzer*)tokenizer); FullTextType *other = FullTextType_new((Analyzer*)normalizer); FullTextType *boost_differs = FullTextType_new((Analyzer*)tokenizer); FullTextType *not_indexed = FullTextType_new((Analyzer*)tokenizer); FullTextType *not_stored = FullTextType_new((Analyzer*)tokenizer); FullTextType *highlightable = FullTextType_new((Analyzer*)tokenizer); Obj *dump = (Obj*)FullTextType_Dump(type); Obj *clone = Freezer_load(dump); Obj *another_dump = (Obj*)FullTextType_Dump_For_Schema(type); FullTextType_Set_Boost(boost_differs, 1.5); FullTextType_Set_Indexed(not_indexed, false); FullTextType_Set_Stored(not_stored, false); FullTextType_Set_Highlightable(highlightable, true); // (This step is normally performed by Schema_Load() internally.) Hash_Store_Utf8((Hash*)another_dump, "analyzer", 8, INCREF(tokenizer)); FullTextType *another_clone = FullTextType_Load(type, another_dump); TEST_FALSE(runner, FullTextType_Equals(type, (Obj*)boost_differs), "Equals() false with different boost"); TEST_FALSE(runner, FullTextType_Equals(type, (Obj*)other), "Equals() false with different Analyzer"); TEST_FALSE(runner, FullTextType_Equals(type, (Obj*)not_indexed), "Equals() false with indexed => false"); TEST_FALSE(runner, FullTextType_Equals(type, (Obj*)not_stored), "Equals() false with stored => false"); TEST_FALSE(runner, FullTextType_Equals(type, (Obj*)highlightable), "Equals() false with highlightable => true"); TEST_TRUE(runner, FullTextType_Equals(type, (Obj*)clone), "Dump => Load round trip"); TEST_TRUE(runner, FullTextType_Equals(type, (Obj*)another_clone), "Dump_For_Schema => Load round trip"); DECREF(another_clone); DECREF(dump); DECREF(clone); DECREF(another_dump); DECREF(highlightable); DECREF(not_stored); DECREF(not_indexed); DECREF(boost_differs); DECREF(other); DECREF(type); DECREF(normalizer); DECREF(tokenizer); }
static void test_Dump_Load_and_Equals(TestBatchRunner *runner) { StandardTokenizer *tokenizer = StandardTokenizer_new(); Obj *dump = StandardTokenizer_Dump(tokenizer); StandardTokenizer *clone = (StandardTokenizer*)StandardTokenizer_Load(tokenizer, dump); TEST_TRUE(runner, StandardTokenizer_Equals(tokenizer, (Obj*)clone), "Dump => Load round trip"); DECREF(tokenizer); DECREF(dump); DECREF(clone); }
static void test_Compare_Values(TestBatchRunner *runner) { StandardTokenizer *tokenizer = StandardTokenizer_new(); FullTextType *type = FullTextType_new((Analyzer*)tokenizer); StackString *a = SSTR_WRAP_UTF8("a", 1); StackString *b = SSTR_WRAP_UTF8("b", 1); TEST_TRUE(runner, FullTextType_Compare_Values(type, (Obj*)a, (Obj*)b) < 0, "a less than b"); TEST_TRUE(runner, FullTextType_Compare_Values(type, (Obj*)b, (Obj*)a) > 0, "b greater than a"); TEST_TRUE(runner, FullTextType_Compare_Values(type, (Obj*)b, (Obj*)b) == 0, "b equals b"); DECREF(type); DECREF(tokenizer); }
static void test_tokenizer(TestBatchRunner *runner) { StandardTokenizer *tokenizer = StandardTokenizer_new(); String *word = SSTR_WRAP_C( " ." "tha\xCC\x82t's" ":" "1,02\xC2\xADZ4.38" "\xE0\xB8\x81\xC2\xAD\xC2\xAD" "\xF0\xA0\x80\x80" "a" "/"); Vector *got = StandardTokenizer_Split(tokenizer, word); String *token = (String*)Vec_Fetch(got, 0); TEST_TRUE(runner, token && Str_is_a(token, STRING) && Str_Equals_Utf8(token, "tha\xcc\x82t's", 8), "Token: %s", Str_Get_Ptr8(token)); token = (String*)Vec_Fetch(got, 1); TEST_TRUE(runner, token && Str_is_a(token, STRING) && Str_Equals_Utf8(token, "1,02\xC2\xADZ4.38", 11), "Token: %s", Str_Get_Ptr8(token)); token = (String*)Vec_Fetch(got, 2); TEST_TRUE(runner, token && Str_is_a(token, STRING) && Str_Equals_Utf8(token, "\xE0\xB8\x81\xC2\xAD\xC2\xAD", 7), "Token: %s", Str_Get_Ptr8(token)); token = (String*)Vec_Fetch(got, 3); TEST_TRUE(runner, token && Str_is_a(token, STRING) && Str_Equals_Utf8(token, "\xF0\xA0\x80\x80", 4), "Token: %s", Str_Get_Ptr8(token)); token = (String*)Vec_Fetch(got, 4); TEST_TRUE(runner, token && Str_is_a(token, STRING) && Str_Equals_Utf8(token, "a", 1), "Token: %s", Str_Get_Ptr8(token)); DECREF(got); FSFolder *modules_folder = TestUtils_modules_folder(); if (modules_folder == NULL) { SKIP(runner, 1372, "Can't locate test data"); } else { String *path = Str_newf("unicode/ucd/WordBreakTest.json"); Vector *tests = (Vector*)Json_slurp_json((Folder*)modules_folder, path); if (!tests) { RETHROW(Err_get_error()); } for (uint32_t i = 0, max = Vec_Get_Size(tests); i < max; i++) { Hash *test = (Hash*)Vec_Fetch(tests, i); String *text = (String*)Hash_Fetch_Utf8(test, "text", 4); Vector *wanted = (Vector*)Hash_Fetch_Utf8(test, "words", 5); Vector *got = StandardTokenizer_Split(tokenizer, text); TEST_TRUE(runner, Vec_Equals(wanted, (Obj*)got), "UCD test #%d", i + 1); DECREF(got); } DECREF(tests); DECREF(modules_folder); DECREF(path); } DECREF(tokenizer); }