Stemmer::Stemmer( const FTSLanguage language ) { _stemmer = NULL; if ( language.str() != "none" ) _stemmer = sb_stemmer_new(language.str().c_str(), "UTF_8"); }
Tokenizer::Tokenizer( const FTSLanguage& language, StringData str ) : _pos(0), _raw( str ) { _english = ( language.str() == "english" ); _skipWhitespace(); _previousWhiteSpace = true; }