コード例 #1
0
inline void UnigramDictionary::onTerminal(const int probability,
        const TerminalAttributes& terminalAttributes, Correction *correction,
        WordsPriorityQueuePool *queuePool, const bool addToMasterQueue,
        const int currentWordIndex) {
    const int inputIndex = correction->getInputIndex();
    const bool addToSubQueue = inputIndex < SUB_QUEUE_MAX_COUNT;

    int wordLength;
    unsigned short* wordPointer;

    if ((currentWordIndex == FIRST_WORD_INDEX) && addToMasterQueue) {
        WordsPriorityQueue *masterQueue = queuePool->getMasterQueue();
        const int finalProbability =
                correction->getFinalProbability(probability, &wordPointer, &wordLength);
        if (finalProbability != NOT_A_PROBABILITY) {
            addWord(wordPointer, wordLength, finalProbability, masterQueue);

            const int shortcutProbability = finalProbability > 0 ? finalProbability - 1 : 0;
            // Please note that the shortcut candidates will be added to the master queue only.
            TerminalAttributes::ShortcutIterator iterator =
                    terminalAttributes.getShortcutIterator();
            while (iterator.hasNextShortcutTarget()) {
                // TODO: addWord only supports weak ordering, meaning we have no means
                // to control the order of the shortcuts relative to one another or to the word.
                // We need to either modulate the probability of each shortcut according
                // to its own shortcut probability or to make the queue
                // so that the insert order is protected inside the queue for words
                // with the same score. For the moment we use -1 to make sure the shortcut will
                // never be in front of the word.
                uint16_t shortcutTarget[MAX_WORD_LENGTH_INTERNAL];
                const int shortcutTargetStringLength = iterator.getNextShortcutTarget(
                        MAX_WORD_LENGTH_INTERNAL, shortcutTarget);
                addWord(shortcutTarget, shortcutTargetStringLength, shortcutProbability,
                        masterQueue);
            }
        }
    }

    // We only allow two words + other error correction for words with SUB_QUEUE_MIN_WORD_LENGTH
    // or more length.
    if (inputIndex >= SUB_QUEUE_MIN_WORD_LENGTH && addToSubQueue) {
        WordsPriorityQueue *subQueue;
        subQueue = queuePool->getSubQueue(currentWordIndex, inputIndex);
        if (!subQueue) {
            return;
        }
        const int finalProbability = correction->getFinalProbabilityForSubQueue(
                probability, &wordPointer, &wordLength, inputIndex);
        addWord(wordPointer, wordLength, finalProbability, subQueue);
    }
}
inline void UnigramDictionary::onTerminal(const int probability,
        const TerminalAttributes& terminalAttributes, Correction *correction,
        WordsPriorityQueuePool *queuePool, const bool addToMasterQueue,
        const int currentWordIndex) const {
    const int inputIndex = correction->getInputIndex();
    const bool addToSubQueue = inputIndex < SUB_QUEUE_MAX_COUNT;

    int wordLength;
    unsigned short *wordPointer;

    if ((currentWordIndex == FIRST_WORD_INDEX) && addToMasterQueue) {
        WordsPriorityQueue *masterQueue = queuePool->getMasterQueue();
        const int finalProbability =
                correction->getFinalProbability(probability, &wordPointer, &wordLength);

        if (0 != finalProbability && !terminalAttributes.isBlacklistedOrNotAWord()) {
            // If the probability is 0, we don't want to add this word. However we still
            // want to add its shortcuts (including a possible whitelist entry) if any.
            // Furthermore, if this is not a word (shortcut only for example) or a blacklisted
            // entry then we never want to suggest this.
            addWord(wordPointer, wordLength, finalProbability, masterQueue,
                    Dictionary::KIND_CORRECTION);
        }

        const int shortcutProbability = finalProbability > 0 ? finalProbability - 1 : 0;
        // Please note that the shortcut candidates will be added to the master queue only.
        TerminalAttributes::ShortcutIterator iterator =
                terminalAttributes.getShortcutIterator();
        while (iterator.hasNextShortcutTarget()) {
            // TODO: addWord only supports weak ordering, meaning we have no means
            // to control the order of the shortcuts relative to one another or to the word.
            // We need to either modulate the probability of each shortcut according
            // to its own shortcut probability or to make the queue
            // so that the insert order is protected inside the queue for words
            // with the same score. For the moment we use -1 to make sure the shortcut will
            // never be in front of the word.
            uint16_t shortcutTarget[MAX_WORD_LENGTH_INTERNAL];
            int shortcutFrequency;
            const int shortcutTargetStringLength = iterator.getNextShortcutTarget(
                    MAX_WORD_LENGTH_INTERNAL, shortcutTarget, &shortcutFrequency);
            int shortcutScore;
            int kind;
            if (shortcutFrequency == BinaryFormat::WHITELIST_SHORTCUT_FREQUENCY
                    && correction->sameAsTyped()) {
                shortcutScore = S_INT_MAX;
                kind = Dictionary::KIND_WHITELIST;
            } else {
                shortcutScore = shortcutProbability;
                kind = Dictionary::KIND_CORRECTION;
            }
            addWord(shortcutTarget, shortcutTargetStringLength, shortcutScore,
                    masterQueue, kind);
        }
    }

    // We only allow two words + other error correction for words with SUB_QUEUE_MIN_WORD_LENGTH
    // or more length.
    if (inputIndex >= SUB_QUEUE_MIN_WORD_LENGTH && addToSubQueue) {
        WordsPriorityQueue *subQueue;
        subQueue = queuePool->getSubQueue(currentWordIndex, inputIndex);
        if (!subQueue) {
            return;
        }
        const int finalProbability = correction->getFinalProbabilityForSubQueue(
                probability, &wordPointer, &wordLength, inputIndex);
        addWord(wordPointer, wordLength, finalProbability, subQueue, Dictionary::KIND_CORRECTION);
    }
}