int32 BaseParser::getObject(char **buf, const TokenDesc *tokens, char **name, char **data) {
skipCharacters(buf, _whiteSpace);
// skip comment lines.
while (**buf == ';') {
*buf = strchr(*buf, '\n');
_parserLine++;
skipCharacters(buf, _whiteSpace);
}
if (! **buf) { // at end of file
return PARSERR_EOF;
}
// find the token.
// TODO: for now just use brute force. Improve later.
while (tokens->id != 0) {
if (!scumm_strnicmp(tokens->token, *buf, strlen(tokens->token))) {
// here we could be matching PART of a string
// we could detect this here or the token list
// could just have the longer tokens first in the list
break;
}
++tokens;
}
if (tokens->id == 0) {
char *p = strchr(*buf, '\n');
if (p && p > *buf) {
strncpy(_lastOffender, *buf, MIN((uint32)255, (uint32)(p - *buf))); // TODO, clean
} else {
strcpy(_lastOffender, "");
}
return PARSERR_TOKENNOTFOUND;
}
// skip the token
*buf += strlen(tokens->token);
skipCharacters(buf, _whiteSpace);
// get optional name
*name = getSubText(buf, '\'', '\''); // single quotes
skipCharacters(buf, _whiteSpace);
// get optional data
if (**buf == '=') { // An assignment rather than a command/object.
*data = getAssignmentText(buf);
} else {
*data = getSubText(buf, '{', '}');
}
return tokens->id;
}
char *BaseParser::getAssignmentText(char **buf) {
++*buf; // skip the '='
skipCharacters(buf, _whiteSpace);
char *result = *buf;
if (*result == '"') {
result = getSubText(buf, '"', '"');
} else {
// now, we need to find the next whitespace to end the data
char theChar;
while ((theChar = **buf) != 0) {
if (theChar <= 0x20) { // space and control chars
break;
}
++*buf;
}
**buf = 0; // null terminate it
if (theChar) { // skip the terminator
++*buf;
}
}
return result;
}
/*!
* \brief Get the next token in the stream
*/
HllTokenizer::Token HllTokenizer::getNext()
{
Token next;
// Start out by moving past any whitespace
skipCharacters(whitespace);
// Check if we've reached the end of the file
if(!fillBuffer(1)) {
next = createToken(Token::TypeEnd, "");
return next;
}
// Scan through the list of literals and see if any match
if(scanLiteral(literals, next)) {
return next;
}
// If no literals matched, see if an identifier can be constructed
if(std::isalpha(buffer()[0]) || buffer()[0] == '_') {
size_t len = 0;
while(std::isalpha(buffer()[len]) || buffer()[len] == '_') {
len++;
if(!fillBuffer(len + 1)) {
break;
}
}
TokenType type = TypeIdentifier;
std::string string = buffer().substr(0, len);
// Check if the string is a keyword
for(std::string &keyword : keywords) {
if(string == keyword) {
type = TypeLiteral;
break;
}
}
// Construct a token out of the characters found
next = createToken(type, string);
emptyBuffer(len);
return next;
}
// If an identifier couldn't be found, check for a number
if(std::isdigit(buffer()[0])) {
size_t len = 0;
while(std::isdigit(buffer()[len])) {
len++;
if(!fillBuffer(len + 1)) {
break;
}
}
// Construct a token out of the characters found
next = createToken(TypeNumber, buffer().substr(0, len));
emptyBuffer(len);
return next;
}
if(buffer()[0] == '\"') {
size_t len = 1;
while(true) {
if(!fillBuffer(len + 1)) {
setError("Unterminated string literal");
return next;
}
if(buffer()[len] == '\"') {
break;
}
len++;
}
// Construct a token out of the characters found
std::string text = buffer().substr(1, len - 1);
emptyBuffer(len + 1);
if(evaluateEscapes(text)) {
next = createToken(TypeString, text);
}
return next;
}
if(buffer()[0] == '\'') {
size_t len = 1;
while(true) {
if(!fillBuffer(len + 1)) {
setError("Unterminated character literal");
return next;
}
if(buffer()[len] == '\'') {
break;
}
len++;
}
// Construct a token out of the characters found
std::string text = buffer().substr(1, len - 1);
emptyBuffer(len + 1);
if(evaluateEscapes(text)) {
if(text.size() == 1) {
next = createToken(TypeChar, text);
} else {
setError("Invalid character literal");
}
}
return next;
}
// Nothing matched, log an error
std::stringstream ss;
ss << "Illegal symbol '" << buffer()[0] << "'";
setError(ss.str());
return next;
}
