コード例 #1
0
 TokenStreamPtr WhitespaceAnalyzer::reusableTokenStream(const String& fieldName, ReaderPtr reader)
 {
     TokenizerPtr tokenizer(boost::dynamic_pointer_cast<Tokenizer>(getPreviousTokenStream()));
     if (!tokenizer)
     {
         tokenizer = newLucene<WhitespaceTokenizer>(reader);
         setPreviousTokenStream(tokenizer);
     }
     else
         tokenizer->reset(reader);
     return tokenizer;
 }
コード例 #2
0
 TokenStreamPtr ChineseAnalyzer::reusableTokenStream(const String& fieldName, ReaderPtr reader)
 {
     ChineseAnalyzerSavedStreamsPtr streams(boost::dynamic_pointer_cast<ChineseAnalyzerSavedStreams>(getPreviousTokenStream()));
     if (!streams)
     {
         streams = newLucene<ChineseAnalyzerSavedStreams>();
         streams->source = newLucene<ChineseTokenizer>(reader);
         streams->result = newLucene<ChineseFilter>(streams->source);
         setPreviousTokenStream(streams);
     }
     else
         streams->source->reset(reader);
     return streams->result;
 }
コード例 #3
0
TokenStreamPtr RussianAnalyzer::reusableTokenStream(const String& fieldName, const ReaderPtr& reader) {
    RussianAnalyzerSavedStreamsPtr streams(boost::dynamic_pointer_cast<RussianAnalyzerSavedStreams>(getPreviousTokenStream()));
    if (!streams) {
        streams = newLucene<RussianAnalyzerSavedStreams>();
        streams->source = newLucene<RussianLetterTokenizer>(reader);
        streams->result = newLucene<LowerCaseFilter>(streams->source);
        streams->result = newLucene<StopFilter>(StopFilter::getEnablePositionIncrementsVersionDefault(matchVersion), streams->result, stopSet);
        streams->result = newLucene<RussianStemFilter>(streams->result);
        setPreviousTokenStream(streams);
    } else {
        streams->source->reset(reader);
    }
    return streams->result;
}
コード例 #4
0
 TokenStreamPtr DutchAnalyzer::reusableTokenStream(const String& fieldName, ReaderPtr reader)
 {
     DutchAnalyzerSavedStreamsPtr streams(boost::dynamic_pointer_cast<DutchAnalyzerSavedStreams>(getPreviousTokenStream()));
     if (!streams)
     {
         streams = newLucene<DutchAnalyzerSavedStreams>();
         streams->source = newLucene<StandardTokenizer>(matchVersion, reader);
         streams->result = newLucene<StandardFilter>(streams->source);
         streams->result = newLucene<StopFilter>(StopFilter::getEnablePositionIncrementsVersionDefault(matchVersion), streams->result, stoptable);
         streams->result = newLucene<DutchStemFilter>(streams->result, excltable);
         setPreviousTokenStream(streams);
     }
     else
         streams->source->reset(reader);
     return streams->result;
 }
コード例 #5
0
 TokenStreamPtr StandardAnalyzer::reusableTokenStream(const String& fieldName, ReaderPtr reader)
 {
     StandardAnalyzerSavedStreamsPtr streams = boost::dynamic_pointer_cast<StandardAnalyzerSavedStreams>(getPreviousTokenStream());
     if (!streams)
     {
         streams = newLucene<StandardAnalyzerSavedStreams>();
         setPreviousTokenStream(streams);
         streams->tokenStream = newLucene<StandardTokenizer>(matchVersion, reader);
         streams->filteredTokenStream = newLucene<StandardFilter>(streams->tokenStream);
         streams->filteredTokenStream = newLucene<LowerCaseFilter>(streams->filteredTokenStream);
         streams->filteredTokenStream = newLucene<StopFilter>(enableStopPositionIncrements, streams->filteredTokenStream, stopSet);
     }
     else
         streams->tokenStream->reset(reader);
     streams->tokenStream->setMaxTokenLength(maxTokenLength);
     
     streams->tokenStream->setReplaceInvalidAcronym(replaceInvalidAcronym);
     
     return streams->filteredTokenStream;
 }
コード例 #6
0
 void DutchAnalyzer::setStemExclusionTable(HashSet<String> exclusions)
 {
     excltable = exclusions;
     setPreviousTokenStream(LuceneObjectPtr()); // force a new stemmer to be created
 }