inline void UnigramDictionary::onTerminal(const int probability,
const TerminalAttributes& terminalAttributes, Correction *correction,
WordsPriorityQueuePool *queuePool, const bool addToMasterQueue,
const int currentWordIndex) {
const int inputIndex = correction->getInputIndex();
const bool addToSubQueue = inputIndex < SUB_QUEUE_MAX_COUNT;
int wordLength;
unsigned short* wordPointer;
if ((currentWordIndex == FIRST_WORD_INDEX) && addToMasterQueue) {
WordsPriorityQueue *masterQueue = queuePool->getMasterQueue();
const int finalProbability =
correction->getFinalProbability(probability, &wordPointer, &wordLength);
if (finalProbability != NOT_A_PROBABILITY) {
addWord(wordPointer, wordLength, finalProbability, masterQueue);
const int shortcutProbability = finalProbability > 0 ? finalProbability - 1 : 0;
// Please note that the shortcut candidates will be added to the master queue only.
TerminalAttributes::ShortcutIterator iterator =
terminalAttributes.getShortcutIterator();
while (iterator.hasNextShortcutTarget()) {
// TODO: addWord only supports weak ordering, meaning we have no means
// to control the order of the shortcuts relative to one another or to the word.
// We need to either modulate the probability of each shortcut according
// to its own shortcut probability or to make the queue
// so that the insert order is protected inside the queue for words
// with the same score. For the moment we use -1 to make sure the shortcut will
// never be in front of the word.
uint16_t shortcutTarget[MAX_WORD_LENGTH_INTERNAL];
const int shortcutTargetStringLength = iterator.getNextShortcutTarget(
MAX_WORD_LENGTH_INTERNAL, shortcutTarget);
addWord(shortcutTarget, shortcutTargetStringLength, shortcutProbability,
masterQueue);
}
}
}
// We only allow two words + other error correction for words with SUB_QUEUE_MIN_WORD_LENGTH
// or more length.
if (inputIndex >= SUB_QUEUE_MIN_WORD_LENGTH && addToSubQueue) {
WordsPriorityQueue *subQueue;
subQueue = queuePool->getSubQueue(currentWordIndex, inputIndex);
if (!subQueue) {
return;
}
const int finalProbability = correction->getFinalProbabilityForSubQueue(
probability, &wordPointer, &wordLength, inputIndex);
addWord(wordPointer, wordLength, finalProbability, subQueue);
}
}
inline void UnigramDictionary::onTerminal(const int probability,
const TerminalAttributes& terminalAttributes, Correction *correction,
WordsPriorityQueuePool *queuePool, const bool addToMasterQueue,
const int currentWordIndex) const {
const int inputIndex = correction->getInputIndex();
const bool addToSubQueue = inputIndex < SUB_QUEUE_MAX_COUNT;
int wordLength;
unsigned short *wordPointer;
if ((currentWordIndex == FIRST_WORD_INDEX) && addToMasterQueue) {
WordsPriorityQueue *masterQueue = queuePool->getMasterQueue();
const int finalProbability =
correction->getFinalProbability(probability, &wordPointer, &wordLength);
if (0 != finalProbability && !terminalAttributes.isBlacklistedOrNotAWord()) {
// If the probability is 0, we don't want to add this word. However we still
// want to add its shortcuts (including a possible whitelist entry) if any.
// Furthermore, if this is not a word (shortcut only for example) or a blacklisted
// entry then we never want to suggest this.
addWord(wordPointer, wordLength, finalProbability, masterQueue,
Dictionary::KIND_CORRECTION);
}
const int shortcutProbability = finalProbability > 0 ? finalProbability - 1 : 0;
// Please note that the shortcut candidates will be added to the master queue only.
TerminalAttributes::ShortcutIterator iterator =
terminalAttributes.getShortcutIterator();
while (iterator.hasNextShortcutTarget()) {
// TODO: addWord only supports weak ordering, meaning we have no means
// to control the order of the shortcuts relative to one another or to the word.
// We need to either modulate the probability of each shortcut according
// to its own shortcut probability or to make the queue
// so that the insert order is protected inside the queue for words
// with the same score. For the moment we use -1 to make sure the shortcut will
// never be in front of the word.
uint16_t shortcutTarget[MAX_WORD_LENGTH_INTERNAL];
int shortcutFrequency;
const int shortcutTargetStringLength = iterator.getNextShortcutTarget(
MAX_WORD_LENGTH_INTERNAL, shortcutTarget, &shortcutFrequency);
int shortcutScore;
int kind;
if (shortcutFrequency == BinaryFormat::WHITELIST_SHORTCUT_FREQUENCY
&& correction->sameAsTyped()) {
shortcutScore = S_INT_MAX;
kind = Dictionary::KIND_WHITELIST;
} else {
shortcutScore = shortcutProbability;
kind = Dictionary::KIND_CORRECTION;
}
addWord(shortcutTarget, shortcutTargetStringLength, shortcutScore,
masterQueue, kind);
}
}
// We only allow two words + other error correction for words with SUB_QUEUE_MIN_WORD_LENGTH
// or more length.
if (inputIndex >= SUB_QUEUE_MIN_WORD_LENGTH && addToSubQueue) {
WordsPriorityQueue *subQueue;
subQueue = queuePool->getSubQueue(currentWordIndex, inputIndex);
if (!subQueue) {
return;
}
const int finalProbability = correction->getFinalProbabilityForSubQueue(
probability, &wordPointer, &wordLength, inputIndex);
addWord(wordPointer, wordLength, finalProbability, subQueue, Dictionary::KIND_CORRECTION);
}
}