static void
test_iterator(TestBatchRunner *runner) {
static const int32_t code_points[] = {
0x41,
0x7F,
0x80,
0x7FF,
0x800,
0xFFFF,
0x10000,
0x10FFFF
};
static size_t num_code_points
= sizeof(code_points) / sizeof(code_points[0]);
CharBuf *buf = CB_new(0);
for (size_t i = 0; i < num_code_points; ++i) {
CB_Cat_Char(buf, code_points[i]);
}
String *string = CB_To_String(buf);
{
StringIterator *iter = Str_Top(string);
TEST_TRUE(runner, StrIter_Equals(iter, (Obj*)iter),
"StringIterator equal to self");
TEST_FALSE(runner, StrIter_Equals(iter, (Obj*)CFISH_TRUE),
"StringIterator not equal non-iterators");
DECREF(iter);
}
{
StringIterator *top = Str_Top(string);
StringIterator *tail = Str_Tail(string);
TEST_INT_EQ(runner, StrIter_Compare_To(top, (Obj*)tail), -1,
"Compare_To top < tail");
TEST_INT_EQ(runner, StrIter_Compare_To(tail, (Obj*)top), 1,
"Compare_To tail > top");
TEST_INT_EQ(runner, StrIter_Compare_To(top, (Obj*)top), 0,
"Compare_To top == top");
StringIterator *clone = StrIter_Clone(top);
TEST_TRUE(runner, StrIter_Equals(clone, (Obj*)top), "Clone");
StrIter_Assign(clone, tail);
TEST_TRUE(runner, StrIter_Equals(clone, (Obj*)tail), "Assign");
String *other = Str_newf("Other string");
StringIterator *other_iter = Str_Top(other);
TEST_FALSE(runner, StrIter_Equals(other_iter, (Obj*)tail),
"Equals returns false for different strings");
StrIter_Assign(clone, other_iter);
TEST_TRUE(runner, StrIter_Equals(clone, (Obj*)other_iter),
"Assign iterator with different string");
DECREF(other);
DECREF(other_iter);
DECREF(clone);
DECREF(top);
DECREF(tail);
}
{
StringIterator *iter = Str_Top(string);
for (size_t i = 0; i < num_code_points; ++i) {
TEST_TRUE(runner, StrIter_Has_Next(iter), "Has_Next %d", i);
int32_t code_point = StrIter_Next(iter);
TEST_INT_EQ(runner, code_point, code_points[i], "Next %d", i);
}
TEST_TRUE(runner, !StrIter_Has_Next(iter),
"Has_Next at end of string");
TEST_INT_EQ(runner, StrIter_Next(iter), STR_OOB,
"Next at end of string");
StringIterator *tail = Str_Tail(string);
TEST_TRUE(runner, StrIter_Equals(iter, (Obj*)tail), "Equals tail");
DECREF(tail);
DECREF(iter);
}
{
StringIterator *iter = Str_Tail(string);
for (size_t i = num_code_points; i--;) {
TEST_TRUE(runner, StrIter_Has_Prev(iter), "Has_Prev %d", i);
int32_t code_point = StrIter_Prev(iter);
TEST_INT_EQ(runner, code_point, code_points[i], "Prev %d", i);
}
TEST_TRUE(runner, !StrIter_Has_Prev(iter),
"Has_Prev at end of string");
TEST_INT_EQ(runner, StrIter_Prev(iter), STR_OOB,
"Prev at start of string");
StringIterator *top = Str_Top(string);
TEST_TRUE(runner, StrIter_Equals(iter, (Obj*)top), "Equals top");
DECREF(top);
DECREF(iter);
}
{
StringIterator *iter = Str_Top(string);
StrIter_Next(iter);
TEST_INT_EQ(runner, StrIter_Advance(iter, 2), 2,
"Advance returns number of code points");
TEST_INT_EQ(runner, StrIter_Next(iter), code_points[3],
"Advance works");
TEST_INT_EQ(runner,
StrIter_Advance(iter, 1000000), num_code_points - 4,
"Advance past end of string");
StrIter_Prev(iter);
TEST_INT_EQ(runner, StrIter_Recede(iter, 2), 2,
"Recede returns number of code points");
TEST_INT_EQ(runner, StrIter_Prev(iter), code_points[num_code_points-4],
"Recede works");
TEST_INT_EQ(runner, StrIter_Recede(iter, 1000000), num_code_points - 4,
"Recede past start of string");
DECREF(iter);
}
DECREF(string);
DECREF(buf);
}
Vector*
QueryLexer_Tokenize_IMP(QueryLexer *self, String *query_string) {
QueryLexerIVARS *const ivars = QueryLexer_IVARS(self);
Vector *elems = Vec_new(0);
if (!query_string) { return elems; }
StringIterator *iter = Str_Top(query_string);
while (StrIter_Has_Next(iter)) {
ParserElem *elem = NULL;
if (StrIter_Skip_Whitespace(iter)) {
// Fast-forward past whitespace.
continue;
}
if (ivars->heed_colons) {
ParserElem *elem = S_consume_field(iter);
if (elem) {
Vec_Push(elems, (Obj*)elem);
}
}
int32_t code_point = StrIter_Next(iter);
switch (code_point) {
case '(':
elem = ParserElem_new(TOKEN_OPEN_PAREN, NULL);
break;
case ')':
elem = ParserElem_new(TOKEN_CLOSE_PAREN, NULL);
break;
case '+':
if (StrIter_Has_Next(iter)
&& !StrIter_Skip_Whitespace(iter)
) {
elem = ParserElem_new(TOKEN_PLUS, NULL);
}
else {
elem = ParserElem_new(TOKEN_STRING, (Obj*)Str_newf("+"));
}
break;
case '-':
if (StrIter_Has_Next(iter)
&& !StrIter_Skip_Whitespace(iter)
) {
elem = ParserElem_new(TOKEN_MINUS, NULL);
}
else {
elem = ParserElem_new(TOKEN_STRING, (Obj*)Str_newf("-"));
}
break;
case '"':
StrIter_Recede(iter, 1);
elem = S_consume_quoted_string(iter);
break;
case 'O':
StrIter_Recede(iter, 1);
elem = S_consume_keyword(iter, "OR", 2, TOKEN_OR);
if (!elem) {
elem = S_consume_text(iter);
}
break;
case 'A':
StrIter_Recede(iter, 1);
elem = S_consume_keyword(iter, "AND", 3, TOKEN_AND);
if (!elem) {
elem = S_consume_text(iter);
}
break;
case 'N':
StrIter_Recede(iter, 1);
elem = S_consume_keyword(iter, "NOT", 3, TOKEN_NOT);
if (!elem) {
elem = S_consume_text(iter);
}
break;
default:
StrIter_Recede(iter, 1);
elem = S_consume_text(iter);
break;
}
Vec_Push(elems, (Obj*)elem);
}
DECREF(iter);
return elems;
}
String*
Highlighter_Highlight_Excerpt_IMP(Highlighter *self, VArray *spans,
String *raw_excerpt, int32_t top) {
int32_t hl_start = 0;
int32_t hl_end = 0;
StringIterator *iter = Str_Top(raw_excerpt);
StringIterator *temp = Str_Top(raw_excerpt);
CharBuf *buf = CB_new(Str_Get_Size(raw_excerpt) + 32);
CharBuf *encode_buf = NULL;
int32_t raw_excerpt_end = top + Str_Length(raw_excerpt);
for (uint32_t i = 0, max = VA_Get_Size(spans); i < max; i++) {
Span *span = (Span*)VA_Fetch(spans, i);
int32_t offset = Span_Get_Offset(span);
if (offset < top) {
continue;
}
else if (offset >= raw_excerpt_end) {
break;
}
else {
int32_t relative_start = offset - top;
int32_t relative_end = relative_start + Span_Get_Length(span);
if (relative_start <= hl_end) {
if (relative_end > hl_end) {
hl_end = relative_end;
}
}
else {
if (hl_start < hl_end) {
// Highlight previous section
int32_t highlighted_len = hl_end - hl_start;
StrIter_Assign(temp, iter);
StrIter_Advance(iter, highlighted_len);
String *to_cat = StrIter_substring(temp, iter);
String *encoded = S_do_encode(self, to_cat, &encode_buf);
String *hl_frag = Highlighter_Highlight(self, encoded);
CB_Cat(buf, hl_frag);
DECREF(hl_frag);
DECREF(encoded);
DECREF(to_cat);
}
int32_t non_highlighted_len = relative_start - hl_end;
StrIter_Assign(temp, iter);
StrIter_Advance(iter, non_highlighted_len);
String *to_cat = StrIter_substring(temp, iter);
String *encoded = S_do_encode(self, to_cat, &encode_buf);
CB_Cat(buf, (String*)encoded);
DECREF(encoded);
DECREF(to_cat);
hl_start = relative_start;
hl_end = relative_end;
}
}
}
if (hl_start < hl_end) {
// Highlight final section
int32_t highlighted_len = hl_end - hl_start;
StrIter_Assign(temp, iter);
StrIter_Advance(iter, highlighted_len);
String *to_cat = StrIter_substring(temp, iter);
String *encoded = S_do_encode(self, to_cat, &encode_buf);
String *hl_frag = Highlighter_Highlight(self, encoded);
CB_Cat(buf, hl_frag);
DECREF(hl_frag);
DECREF(encoded);
DECREF(to_cat);
}
// Last text, beyond last highlight span.
if (StrIter_Has_Next(iter)) {
String *to_cat = StrIter_substring(iter, NULL);
String *encoded = S_do_encode(self, to_cat, &encode_buf);
CB_Cat(buf, encoded);
DECREF(encoded);
DECREF(to_cat);
}
String *highlighted = CB_Yield_String(buf);
DECREF(encode_buf);
DECREF(buf);
DECREF(temp);
DECREF(iter);
return highlighted;
}
