/// the non-empty non-space spans in s[0..len)
inline void addTokenSpans(TokenSpans &spans, char const* s, Position len) {
Position i = 0;
for (;;) {
if (i == len) return;
if (s[i] != ' ') break;
++i;
}
TokenSpan span;
span.first = span.second = i;
for (;;) {
assert(span.second < len && s[span.second] != ' ');
if (++span.second == len) {
spans.push_back(span);
return;
}
if (s[span.second] == ' ') {
spans.push_back(span);
for (;;) {
if (s[++span.second] != ' ') break;
if (span.second == len) return;
}
span.first = span.second;
}
}
}
void operator()(std::string const& word, TokenSpan span) const {
if (spans_) spans_->push_back(span);
tokens_.push_back(word);
}
void operator()(Unicode c, Position pos) const {
if (spans_) spans_->push_back(TokenSpan(pos, pos + 1));
tokens_.push_back(Util::utf8s(c));
}
void operator()(Slice const& word, TokenSpan span) const {
if (spans_) spans_->push_back(span);
tokens_.push_back(std::string(word.first, word.second));
}