char* GTokenizer::appendToToken(const char* string)
{
while(*string != '\0')
{
bufferChar(*string);
string++;
}
return nullTerminate();
}
// this ctor can be used to save the work of counting the length if we have a length already, as with static HashedCharArrays.
TextFragment::TextFragment(const char* pChars, int len) noexcept : mSize(len)
{
create();
if(mpText)
{
std::copy(pChars, pChars + mSize, mpText);
nullTerminate();
}
}
TextFragment::TextFragment(const char* pChars) noexcept : mSize(strlen(pChars))
{
create();
// a bad alloc will result in this being a null object.
// copy the input string into local storage
if(mpText)
{
std::copy(pChars, pChars + mSize, mpText);
nullTerminate();
}
}
TextFragment::TextFragment(utf::codepoint_type c) noexcept
{
if(!utf::internal::validate_codepoint(c))
{
c = 0x2639; // sad face
}
// all possible codepoints fit into local text
char* end = utf::internal::utf_traits<utf::utf8>::encode(c, mLocalText);
mSize = end - mLocalText;
mpText = mLocalText;
nullTerminate();
}
char* GTokenizer::nextWhile(GCharSet& set, size_t minLen)
{
m_pBufPos = m_pBufStart;
while(has_more())
{
char c = peek();
if(!set.find(c))
break;
c = get();
bufferChar(c);
}
if((size_t)(m_pBufPos - m_pBufStart) < minLen)
throw Ex("Unexpected token on line ", to_str(m_line), ", col ", to_str(col()));
return nullTerminate();
}
char* GTokenizer::nextUntilNotEscaped(char escapeChar, GCharSet& delimeters)
{
m_pBufPos = m_pBufStart;
char cCur = '\0';
while(has_more())
{
char c = peek();
if(delimeters.find(c) && cCur != escapeChar)
break;
c = get();
bufferChar(c);
cCur = c;
}
return nullTerminate();
}
char* GTokenizer::nextUntil(GCharSet& delimeters, size_t minLen)
{
m_pBufPos = m_pBufStart;
while(has_more())
{
char c = peek();
if(delimeters.find(c))
break;
c = get();
bufferChar(c);
}
if((size_t)(m_pBufPos - m_pBufStart) < minLen)
throw Ex("On line ", to_str(m_line), ", col ", to_str(col()), ", expected a token of at least size ", to_str(minLen), ", but got only ", to_str(m_pBufPos - m_pBufStart));
return nullTerminate();
}
// just copy the data. If we want to optimize and use reference-counted strings at some point,
// look at fix_str for ideas.
TextFragment& TextFragment::operator=(const TextFragment& b) noexcept
{
if(this != &b)
{
dispose();
mSize = b.mSize;
create();
if(mpText)
{
const char* bText = b.mpText;
std::copy(bText, bText + mSize, mpText);
nullTerminate();
}
}
return *this;
}
void TextFragment::construct (const char* s1, size_t len1,
const char* s2 , size_t len2 ,
const char* s3 , size_t len3 ,
const char* s4 , size_t len4
) noexcept
{
mSize = (len1 + len2 + len3 + len4);
create();
if(mpText)
{
if(len1) std::copy (s1, s1 + len1, mpText);
if(len2) std::copy (s2, s2 + len2, mpText + len1);
if(len3) std::copy (s3, s3 + len3, mpText + len1 + len2);
if(len4) std::copy (s4, s4 + len4, mpText + len1 + len2 + len3);
nullTerminate();
}
}
void TextFragment::moveDataFromOther(TextFragment& b)
{
mSize = b.mSize;
if(mSize >= kShortFragmentSize)
{
// move the data
mpText = b.mpText;
}
else
{
// point to local storage and copy data
mpText = mLocalText;
std::copy(b.mLocalText, b.mLocalText + mSize, mLocalText);
nullTerminate();
}
// mark b as empty, nothing to dispose
b.mpText = b.mLocalText;
b.mSize = 0;
b.nullTerminate();
}
char* GTokenizer::nextArg(GCharSet& delimiters, char escapeChar)
{
m_pBufPos = m_pBufStart;
char c = peek();
if(c == '"')
{
bufferChar('"');
advance(1);
GCharSet cs("\"\n");
while(has_more())
{
char c2 = peek();
if(cs.find(c2))
break;
c2 = get();
bufferChar(c2);
}
if(peek() != '"')
throw Ex("Expected matching double-quotes on line ",
to_str(m_line), ", col ", to_str(col()));
bufferChar('"');
advance(1);
while(!delimiters.find(peek()))
advance(1);
return nullTerminate();
}
else if(c == '\'')
{
bufferChar('\'');
advance(1);
GCharSet cs("'\n");
while(has_more())
{
char c2 = peek();
if(cs.find(c2))
break;
c2 = get();
bufferChar(c2);
}
if(peek() != '\'')
throw Ex("Expected a matching single-quote on line ", to_str(m_line),
", col ", to_str(col()));
bufferChar('\'');
advance(1);
while(!delimiters.find(peek()))
advance(1);
return nullTerminate();
}
bool inEscapeMode = false;
while(has_more())
{
char c2 = peek();
if(inEscapeMode)
{
if(c2 == '\n')
{
throw Ex("Error: '", to_str(escapeChar), "' character used as "
"last character on a line to attempt to extend string over "
"two lines on line" , to_str(m_line), ", col ",
to_str(col()) );
}
c2 = get();
bufferChar(c2);
inEscapeMode = false;
}
else
{
if(c2 == '\n' || delimiters.find(c2)){ break; }
c2 = get();
if(c2 == escapeChar) { inEscapeMode = true; }
else { bufferChar(c2); }
}
}
return nullTerminate();
}
TextFragment::TextFragment() noexcept
{
mSize = 0;
mpText = mLocalText;
nullTerminate();
}
