// Return the last fetched token skipping push backed token if it exists
Token* SqlParser::GetLastToken()
{
Token *token = NULL;
if(_push_back_token != NULL)
token = _push_back_token->prev;
else
token = _tokens.GetLastNoCurrent();
// Return previous non-blank, non-comment token
while(token && (token->IsBlank() == true || token->type == TOKEN_COMMENT))
token = token->prev;
return token;
}
// Skip blanks, tabs and newlines
Token* Token::SkipSpaces(Token *token)
{
if(token == NULL)
return NULL;
Token *cur = token;
// Skip spaces
while(cur != NULL)
{
if(cur->IsBlank() == false)
break;
cur = cur->next;
}
return cur;
}
// Get next token from the input
Token* SqlParser::GetNextToken()
{
// Check push back tokens first
while(_push_back_token != NULL)
{
Token *token = _push_back_token;
// Multiple tokens can be push back
_push_back_token = token->next;
if(token->IsBlank() == true || token->type == TOKEN_COMMENT)
continue;
return token;
}
Token *token = new Token();
bool exists = false;
while(true)
{
if(_next_start == NULL || _remain_size <= 0)
{
delete token;
return NULL;
}
SkipSpaceTokens();
// Check for a word first as it can start with special symbol such as _ @
exists = GetWordToken(token);
if(exists == true)
break;
// Check for a single char token
exists = GetSingleCharToken(token);
if(exists == true)
break;
}
return token;
}
// Set Removed flag for the token
void Token::Remove(Token *token, bool remove_spaces_before)
{
if(token == NULL)
return;
token->flags |= TOKEN_REMOVED;
Token *prev = token->prev;
Token *next = token->next;
if(prev == NULL)
return;
if(remove_spaces_before == true)
{
// Remove spaces token before the removed word (recursively), but only if there is a blank or ) , ;
// after removed token (otherwise tokens can be merged)
if(prev->Compare(' ', L' ') == true || prev->Compare('\t', L'\t') == true)
{
if(next != NULL)
{
if(next->IsBlank() == true || next->Compare(')', L')') == true ||
next->Compare(',', L',') == true || next->Compare(';', L';') == true)
Remove(prev);
}
else
// Next token can be not selected yet
if(token->next_start != NULL && token->remain_size > 1)
{
const char *cur = token->next_start;
if(*cur == ' ' || *cur == ')' || *cur == ',' || *cur == ';')
Remove(prev);
}
}
}
// In case cases better to remove spaces after: "SET var = 1" to "var := 1" when SET is removed
else
{
if(next != NULL && next->Compare(' ', L' ') == true)
Remove(next);
}
}
// Get the previous non-blank, non comment token
Token* SqlParser::GetPrevToken(Token *token)
{
if(token == NULL)
return NULL;
Token *prev = NULL;
Token *cur = token->prev;
while(cur != NULL)
{
if(cur->IsBlank() == false && cur->type != TOKEN_COMMENT)
{
prev = cur;
break;
}
cur = cur->prev;
}
return prev;
}
// Generate output
void SqlParser::CreateOutputString(const char **output, int *out_size)
{
if(output == NULL)
return;
Token *token = _tokens.GetFirst();
int len = 0;
int not_removed = 0;
int removed = 0;
// Calculate the output size in bytes and format output
while(token != NULL)
{
bool r = false;
bool n = false;
// If token removed its target length is 0
len += token->GetTargetLength();
if(token->IsRemoved() == false)
{
r = token->Compare('\r', L'\r');
if(r == false)
n = token->Compare('\n', L'\n');
}
// Check if we need to remove newline (0D0A \r\n on Windows, 0D on Unix)
if(r == true || n == true)
{
// String was not empty and all tokens removed
if(not_removed == 0 && removed != 0)
{
Token::Remove(token);
len--;
// If current is \r and next is \n remove \n
if(r == true && token->next != NULL && token->next->Compare('\n', L'\n') == true)
{
// Its size will be reduced in the next iteration
Token::Remove(token->next);
}
// Remove all spaces in this empty line
Token *cur = token->prev;
while(cur != NULL)
{
// Remove until new line
if(cur->Compare('\n', L'\n') || cur->Compare('\r', L'\r'))
break;
if(cur->IsBlank() == true && cur->IsRemoved() == false)
{
Token::Remove(cur);
len--;
}
cur = cur->prev;
}
}
not_removed = 0;
removed = 0;
}
// Calculate the number of removed and not removed tokens in line
else
{
if(token->IsBlank() == false)
{
if(token->IsRemoved() == true)
removed++;
else
not_removed++;
}
}
token = _tokens.GetNext();
}
if(len == 0)
{
*output = NULL;
if(out_size != NULL)
*out_size = 0;
return;
}
// Allocate buffer
char *out = new char[len + 1]; *out = 0;
token = _tokens.GetFirst();
int cur_len = 0;
while(token != NULL)
{
token->AppendTarget(out, &cur_len);
token = _tokens.GetNext();
}
out[cur_len] = 0;
*output = out;
if(out_size != NULL)
*out_size = cur_len;
}
