void advanceInput()
{
RecognizerSharedStateType *state = get_state();
state->set_tokenStartCharIndex(getCharIndex());
state->set_tokenStartCharPositionInLine(getCharPositionInLine());
state->set_tokenStartLine(getLine());
}
std::wstring CommonToken::toString() {
std::wstring channelStr = L"";
if (channel > 0) {
channelStr = std::wstring(L",channel=") + std::to_wstring(channel);
}
std::wstring txt = getText();
if (txt != L"") {
replaceAll(txt, L"\n",L"\\n");
replaceAll(txt, L"\r",L"\\r");
replaceAll(txt, L"\t",L"\\t");
} else {
txt = L"<no text>";
}
return std::wstring(L"[@") + std::to_wstring(getTokenIndex()) + std::wstring(L",") + std::to_wstring(start) + std::wstring(L":") + std::to_wstring(stop) + std::wstring(L"='") + txt + std::wstring(L"',<") + std::to_wstring(type) + std::wstring(L">") + channelStr + std::wstring(L",") + std::to_wstring(line) + std::wstring(L":") + std::to_wstring(getCharPositionInLine()) + std::wstring(L"]");
}
