wxString queryTokenizer::GetNextToken()
{
// we need to override wxStringTokenizer, because we have to handle quotes
wxString str;
bool foundQuote = false;
do
{
wxString s = wxStringTokenizer::GetNextToken();
str.Append(s);
int quotePos;
do
{
quotePos = s.Find('"');
if (quotePos >= 0)
{
foundQuote = !foundQuote;
s = s.Mid(quotePos + 1);
}
}
while (quotePos >= 0);
if (foundQuote)
str.Append(delimiter);
}
while (foundQuote & HasMoreTokens());
return str;
}
wxString CSVTokenizer::GetNextToken()
{
wxString token;
if ( !HasMoreTokens() )
return token;
// skip leading blanks if not quoted.
while (m_pos < m_string.length() && m_string[m_pos] == wxT(' '))
m_pos ++;
// Are we a quoted field? Must handle this special.
bool quoted_string = (m_string[m_pos] == wxT('\"'));
bool inquote = false;
size_t pos = m_pos;
// find the end of this token.
for (; pos < m_string.length(); pos++)
{
if (quoted_string && m_string[pos] == wxT('\"'))
inquote = !inquote;
if (!inquote)
{
// Check to see if we have found the end of this token.
// Tokens normally end with a ',' delimiter.
if (m_string[pos] == wxT(','))
break;
// Last token is delimited by '\n' or by end of string.
if (m_string[pos] == wxT('\n') && pos == m_string.length() - 1)
break;
}
}
if (quoted_string && !inquote)
{
token.assign(m_string, m_pos + 1, pos - m_pos - 2); // Remove leading and trailing quotes
// Remove double doublequote chars, replace with single doublequote chars
token.Replace(wxT("\"\""), wxT("\""), true);
}
else
token.assign(m_string, m_pos, pos - m_pos);
if (quoted_string && inquote)
{
wxLogNotice(wxT("unterminated double quoted string: %s\n"), token.c_str());
}
m_pos = pos + 1; // Skip token and delimiter
if (m_pos > m_string.length()) // Perhaps no delimiter if at end of string if orig string didn't have '\n'.
m_pos = m_string.length();
return token;
}
//Public Function 4162
string StringTokenizer::GetLastToken()
{
string StringToken;
TokenString = InputString;
while(HasMoreTokens())
{
StringToken = GetNextToken();
}
return(StringToken);
}
wxString wxStringTokenizer::GetNextToken()
{
wxString token;
do
{
if ( !HasMoreTokens() )
{
break;
}
m_hasMoreTokens = MoreTokens_Unknown;
// find the end of this token
wxString::const_iterator pos =
find_first_of(m_delims, m_delimsLen, m_pos, m_stringEnd);
// and the start of the next one
if ( pos == m_stringEnd )
{
// no more delimiters, the token is everything till the end of
// string
token.assign(m_pos, m_stringEnd);
// skip the token
m_pos = m_stringEnd;
// it wasn't terminated
m_lastDelim = wxT('\0');
}
else // we found a delimiter at pos
{
// in wxTOKEN_RET_DELIMS mode we return the delimiter character
// with token, otherwise leave it out
wxString::const_iterator tokenEnd(pos);
if ( m_mode == wxTOKEN_RET_DELIMS )
++tokenEnd;
token.assign(m_pos, tokenEnd);
// skip the token and the trailing delimiter
m_pos = pos + 1;
m_lastDelim = (pos == m_stringEnd) ? wxT('\0') : (wxChar)*pos;
}
}
while ( !AllowEmpty() && token.empty() );
return token;
}
wxString wxStringTokenizer::GetNextToken()
{
// strtok() doesn't return empty tokens, all other modes do
bool allowEmpty = m_mode != wxTOKEN_STRTOK;
wxString token;
do
{
if ( !HasMoreTokens() )
{
break;
}
// find the end of this token
size_t pos = m_string.find_first_of(m_delims);
// and the start of the next one
if ( pos == wxString::npos )
{
// no more delimiters, the token is everything till the end of
// string
token = m_string;
m_pos += m_string.length();
m_string.clear();
// no more tokens in this string, even in wxTOKEN_RET_EMPTY_ALL
// mode (we will return the trailing one right now in this case)
m_hasMore = false;
}
else
{
size_t pos2 = pos + 1;
// in wxTOKEN_RET_DELIMS mode we return the delimiter character
// with token
token = wxString(m_string, m_mode == wxTOKEN_RET_DELIMS ? pos2
: pos);
// remove token with the following it delimiter from string
m_string.erase(0, pos2);
// keep track of the position in the original string too
m_pos += pos2;
}
}
while ( !allowEmpty && token.empty() );
return token;
}
OP_STATUS StringTokenizer::NextToken(OpString &token)
{
if (!HasMoreTokens())
return OpStatus::ERR;
const int find_pos = m_input_string.FindFirstOf(m_tokens);
if (find_pos != KNotFound)
RETURN_IF_ERROR(token.Set(m_input_string.CStr(), find_pos));
else
RETURN_IF_ERROR(token.Set(m_input_string));
m_input_string.Delete(0, find_pos != KNotFound ? find_pos + 1 : KAll);
RETURN_IF_ERROR(StringUtils::Strip(m_input_string, m_tokens));
return OpStatus::OK;
}
wxString wxStringTokenizer::GetNextToken()
{
wxString token;
do
{
if ( !HasMoreTokens() )
{
break;
}
// find the end of this token
size_t pos = m_string.find_first_of(m_delims, m_pos);
// and the start of the next one
if ( pos == wxString::npos )
{
// no more delimiters, the token is everything till the end of
// string
token.assign(m_string, m_pos, wxString::npos);
// skip the token
m_pos = m_string.length();
// it wasn't terminated
m_lastDelim = _T('\0');
}
else // we found a delimiter at pos
{
// in wxTOKEN_RET_DELIMS mode we return the delimiter character
// with token, otherwise leave it out
size_t len = pos - m_pos;
if ( m_mode == wxTOKEN_RET_DELIMS )
len++;
token.assign(m_string, m_pos, len);
// skip the token and the trailing delimiter
m_pos = pos + 1;
m_lastDelim = m_string[pos];
}
}
while ( !AllowEmpty() && token.empty() );
return token;
}
//Public Function 4165
string StringTokenizer::GetToken(int TokenPosition)
{
int TokenCount = 0;
string StringToken;
TokenString = InputString;
while(HasMoreTokens())
{
if(TokenCount == TokenPosition)
{
break;
}
StringToken = GetNextToken();
TokenCount++;
}
return(StringToken);
}
//Public Function 4167
int StringTokenizer::HasMoreTokens(char * DelimiterChar)
{
SetDelimiterString(DelimiterChar);
return(HasMoreTokens());
}
