Example #1
0
Query*
QParser_Expand_Leaf_IMP(QueryParser *self, Query *query) {
    QueryParserIVARS *const ivars = QParser_IVARS(self);
    LeafQuery *leaf_query = (LeafQuery*)query;
    Schema    *schema     = ivars->schema;
    bool       is_phrase  = false;
    bool       ambiguous  = false;

    // Determine whether we can actually process the input.
    if (!Query_is_a(query, LEAFQUERY)) { return NULL; }
    String *full_text = LeafQuery_Get_Text(leaf_query);
    if (!Str_Get_Size(full_text)) { return NULL; }

    // If quoted, always generate PhraseQuery.
    StringIterator *top  = Str_Top(full_text);
    StringIterator *tail = Str_Tail(full_text);
    StrIter_Skip_Next_Whitespace(top);
    StrIter_Skip_Prev_Whitespace(tail);
    if (StrIter_Starts_With_Utf8(top, "\"", 1)) {
        is_phrase = true;
        StrIter_Advance(top, 1);
        if (StrIter_Ends_With_Utf8(tail, "\"", 1)
            && !StrIter_Ends_With_Utf8(tail, "\\\"", 2)
        ) {
            StrIter_Recede(tail, 1);
        }
    }
    String *source_text = StrIter_substring(top, tail);

    // Either use LeafQuery's field or default to Parser's list.
    Vector *fields;
    if (LeafQuery_Get_Field(leaf_query)) {
        fields = Vec_new(1);
        Vec_Push(fields, INCREF(LeafQuery_Get_Field(leaf_query)));
    }
    else {
        fields = (Vector*)INCREF(ivars->fields);
    }

    CharBuf *unescape_buf = CB_new(Str_Get_Size(source_text));
    Vector  *queries      = Vec_new(Vec_Get_Size(fields));
    for (uint32_t i = 0, max = Vec_Get_Size(fields); i < max; i++) {
        String   *field    = (String*)Vec_Fetch(fields, i);
        Analyzer *analyzer = ivars->analyzer
                             ? ivars->analyzer
                             : Schema_Fetch_Analyzer(schema, field);

        if (!analyzer) {
            Vec_Push(queries,
                    (Obj*)QParser_Make_Term_Query(self, field,
                                                  (Obj*)source_text));
        }
        else {
            // Extract token texts.
            String *split_source = S_unescape(self, source_text, unescape_buf);
            Vector *maybe_texts = Analyzer_Split(analyzer, split_source);
            uint32_t num_maybe_texts = Vec_Get_Size(maybe_texts);
            Vector *token_texts = Vec_new(num_maybe_texts);

            // Filter out zero-length token texts.
            for (uint32_t j = 0; j < num_maybe_texts; j++) {
                String *token_text = (String*)Vec_Fetch(maybe_texts, j);
                if (Str_Get_Size(token_text)) {
                    Vec_Push(token_texts, INCREF(token_text));
                }
            }

            if (Vec_Get_Size(token_texts) == 0) {
                /* Query might include stop words.  Who knows? */
                ambiguous = true;
            }

            // Add either a TermQuery or a PhraseQuery.
            if (is_phrase || Vec_Get_Size(token_texts) > 1) {
                Vec_Push(queries, (Obj*)
                        QParser_Make_Phrase_Query(self, field, token_texts));
            }
            else if (Vec_Get_Size(token_texts) == 1) {
                Vec_Push(queries,
                        (Obj*)QParser_Make_Term_Query(self, field, Vec_Fetch(token_texts, 0)));
            }

            DECREF(token_texts);
            DECREF(maybe_texts);
            DECREF(split_source);
        }
    }

    Query *retval;
    if (Vec_Get_Size(queries) == 0) {
        retval = (Query*)NoMatchQuery_new();
        if (ambiguous) {
            NoMatchQuery_Set_Fails_To_Match((NoMatchQuery*)retval, false);
        }
    }
    else if (Vec_Get_Size(queries) == 1) {
        retval = (Query*)INCREF(Vec_Fetch(queries, 0));
    }
    else {
        retval = QParser_Make_OR_Query(self, queries);
    }

    // Clean up.
    DECREF(unescape_buf);
    DECREF(queries);
    DECREF(fields);
    DECREF(source_text);
    DECREF(tail);
    DECREF(top);

    return retval;
}
Example #2
0
// Find an ending boundary before the current position given by the iterator.
// Skip up to max_skip code points plus potential whitespace. Update the
// iterator and return number of code points skipped. Return true if a
// ending edge (sentence) was found.
bool
S_find_ending_boundary(StringIterator *tail, uint32_t max_skip,
                       uint32_t *num_skipped_ptr) {
    int32_t code_point;

    // Check if we're at an ending boundary already. Don't check for a word
    // boundary because we need space for a trailing ellipsis.

    StringIterator *iter = StrIter_Clone(tail);

    do {
        code_point = StrIter_Next(iter);

        if (code_point == STRITER_DONE) {
            // Skip remaining whitespace.
            *num_skipped_ptr = StrIter_Skip_Prev_Whitespace(tail);
            DECREF(iter);
            return true;
        }
    } while (StrHelp_is_whitespace(code_point));

    // Keep track of the first word boundary.
    StringIterator *word = NULL;
    uint32_t word_offset = 0;

    StrIter_Assign(iter, tail);

    for (uint32_t i = 0;
            STRITER_DONE != (code_point = StrIter_Prev(iter));
            ++i)
    {
        if (code_point == '.') {
            StrIter_Assign(tail, iter);
            StrIter_Advance(tail, 1); // Include period.
            *num_skipped_ptr = i;
            DECREF(word);
            DECREF(iter);
            return true;
        }

        if (StrHelp_is_whitespace(code_point)) {
            if (word == NULL) {
                word = StrIter_Clone(iter);
                word_offset = i + 1;
            }
        }
        else if (i >= max_skip) {
            // Break only at non-whitespace to allow another sentence
            // boundary to be found.
            break;
        }
    }

    if (word == NULL) {
        // Make space for ellipsis.
        *num_skipped_ptr = StrIter_Recede(tail, 1);
    }
    else {
        // Use word boundary if no sentence boundary was found.
        StrIter_Assign(tail, word);

        // Strip whitespace and punctuation that collides with an ellipsis.
        while (STRITER_DONE != (code_point = StrIter_Prev(tail))) {
            if (!StrHelp_is_whitespace(code_point)
                    && code_point != '.'
                    && code_point != ','
                    && code_point != ';'
                    && code_point != ':'
                    && code_point != ':'
                    && code_point != '?'
                    && code_point != '!'
               ) {
                StrIter_Advance(tail, 1); // Back up.
                break;
            }
            ++word_offset;
        }

        *num_skipped_ptr = word_offset;
    }

    DECREF(word);
    DECREF(iter);
    return false;
}