TokenType ScannerContextBase::lookupToken(int start, int end, TokenMap map){
TokenType rangeTokenType = TokenType::None;
string possibleToken = _scanner->range(start, end);
TokenMap::const_iterator it = map.find(possibleToken);
if(it != map.end()){
// found matching token
rangeTokenType = it->second;
}
return rangeTokenType;
};
// ######################################################################
void NeoBrain::saveSpeechFile(const TokenMap& tokenMap,
const std::string& fname)
{
rutz::shared_ptr<ParamMap> pmap(new ParamMap);
int numTokenTypes = 0;
for (TokenMap::const_iterator
itr = tokenMap.begin(), stop = tokenMap.end();
itr != stop; ++itr)
{
rutz::shared_ptr<ParamMap> submap(new ParamMap);
submap->putStringParam("NAME", (*itr).first);
int numTokens = 0;
for (size_t j = 0; j < (*itr).second.tokens.size(); ++j)
{
rutz::shared_ptr<ParamMap> subsubmap(new ParamMap);
subsubmap->putIntParam("LOW", (*itr).second.tokens[j].low);
subsubmap->putIntParam("HIGH", (*itr).second.tokens[j].high);
int numTextItems = 0;
for (size_t i = 0; i < (*itr).second.tokens[j].textList.size(); ++i)
{
subsubmap->putStringParam(sformat("TEXT_ITEM_%d", numTextItems),
(*itr).second.tokens[j].textList[i]);
++numTextItems;
}
subsubmap->putIntParam("NUM_TEXT_ITEMS", numTextItems);
submap->putSubpmap(sformat("TOKEN_%d", numTokens), subsubmap);
++numTokens;
}
submap->putIntParam("NUM_TOKENS", numTokens);
pmap->putSubpmap(sformat("TOKEN_TYPE_%d", numTokenTypes), submap);
++numTokenTypes;
}
pmap->putIntParam("NUM_TOKEN_TYPES", numTokenTypes);
pmap->format(fname);
}
// ######################################################################
bool NeoBrain::readSpeechFile(TokenMap& tokenMap,
const std::string& fname)
{
rutz::shared_ptr<ParamMap> pmap;
try
{
pmap = ParamMap::loadPmapFile(fname);
}
catch (std::exception& e)
{
REPORT_CURRENT_EXCEPTION;
return false;
}
ASSERT(pmap.is_valid());
GVX_ERR_CONTEXT(rutz::sfmt("unpacking pmap file %s", fname.c_str()));
tokenMap.clear();
const int numTokenTypes = pmap->getIntParam("NUM_TOKEN_TYPES");
for (int i = 0; i < numTokenTypes; ++i)
{
rutz::shared_ptr<ParamMap> submap =
pmap->getSubpmap(sformat("TOKEN_TYPE_%d", i));
GVX_ERR_CONTEXT(rutz::sfmt("unpacking TOKEN_TYPE_%d", i));
const std::string name = submap->getStringParam("NAME");
TokenType toktype;
const int numTokens = submap->getIntParam("NUM_TOKENS");
for (int k = 0; k < numTokens; ++k)
{
rutz::shared_ptr<ParamMap> subsubmap =
submap->getSubpmap(sformat("TOKEN_%d", k));
GVX_ERR_CONTEXT(rutz::sfmt("unpacking TOKEN_%d", k));
SpeechToken tok;
tok.low = subsubmap->getIntParam("LOW");
tok.high = subsubmap->getIntParam("HIGH");
const int numTextItems = subsubmap->getIntParam("NUM_TEXT_ITEMS");
for (int j = 0; j < numTextItems; ++j)
{
tok.textList.push_back(subsubmap->getStringParam
(sformat("TEXT_ITEM_%d", j)));
}
toktype.tokens.push_back(tok);
}
tokenMap[name] = toktype;
}
return true;
}
TokenQueue_t calculator::toRPN(const char* expr,
TokenMap vars, const char* delim,
const char** rest, OppMap_t opPrecedence,
rWordMap_t rWordMap) {
rpnBuilder data(vars, opPrecedence);
char* nextChar;
static char c = '\0';
if (!delim) delim = &c;
while (*expr && isspace(*expr) && !strchr(delim, *expr)) ++expr;
if (*expr == '\0' || strchr(delim, *expr)) {
throw std::invalid_argument("Cannot build a calculator from an empty expression!");
}
// In one pass, ignore whitespace and parse the expression into RPN
// using Dijkstra's Shunting-yard algorithm.
while (*expr && (data.bracketLevel || !strchr(delim, *expr))) {
if (isdigit(*expr)) {
// If the token is a number, add it to the output queue.
int64_t _int = strtol(expr, &nextChar, 10);
// If the number was not a float:
if (!strchr(".eE", *nextChar)) {
data.handle_token(new Token<int64_t>(_int, INT));
} else {
double digit = strtod(expr, &nextChar);
data.handle_token(new Token<double>(digit, REAL));
}
expr = nextChar;
} else if (isvariablechar(*expr)) {
rWordMap_t::iterator it;
// If the token is a variable, resolve it and
// add the parsed number to the output queue.
std::stringstream ss;
ss << *expr;
++expr;
while (isvariablechar(*expr) || isdigit(*expr)) {
ss << *expr;
++expr;
}
std::string key = ss.str();
if (data.lastTokenWasOp == '.') {
data.handle_token(new Token<std::string>(key, STR));
} else if ((it=rWordMap.find(key)) != rWordMap.end()) {
// Parse reserved words:
try {
it->second(expr, &expr, &data);
} catch (...) {
rpnBuilder::cleanRPN(&data.rpn);
throw;
}
} else {
packToken* value = vars.find(key);
if (value) {
// Save a reference token:
TokenBase* copy = (*value)->clone();
data.handle_token(new RefToken(key, copy));
} else {
// Save the variable name:
data.handle_token(new Token<std::string>(key, VAR));
}
}
} else if (*expr == '\'' || *expr == '"') {
// If it is a string literal, parse it and
// add to the output queue.
char quote = *expr;
++expr;
std::stringstream ss;
while (*expr && *expr != quote && *expr != '\n') {
if (*expr == '\\') {
switch (expr[1]) {
case 'n':
expr+=2;
ss << '\n';
break;
case 't':
expr+=2;
ss << '\t';
break;
default:
if (strchr("\"'\n", expr[1])) ++expr;
ss << *expr;
++expr;
}
} else {
ss << *expr;
++expr;
}
}
if (*expr != quote) {
std::string squote = (quote == '"' ? "\"": "'");
rpnBuilder::cleanRPN(&data.rpn);
throw syntax_error("Expected quote (" + squote +
") at end of string declaration: " + squote + ss.str() + ".");
}
++expr;
data.handle_token(new Token<std::string>(ss.str(), STR));
} else {
// Otherwise, the variable is an operator or paranthesis.
// Check for syntax errors (excess of operators i.e. 10 + + -1):
if (data.lastTokenWasUnary) {
std::string op;
op.push_back(*expr);
rpnBuilder::cleanRPN(&data.rpn);
throw syntax_error("Expected operand after unary operator `" + data.opStack.top() +
"` but found: `" + op + "` instead.");
}
switch (*expr) {
case '(':
// If it is a function call:
if (data.lastTokenWasOp == false) {
// This counts as a bracket and as an operator:
data.handle_op("()");
// Add it as a bracket to the op stack:
}
data.open_bracket("(");
++expr;
break;
case '[':
if (data.lastTokenWasOp == false) {
// If it is an operator:
data.handle_op("[]");
} else {
// If it is the list constructor:
// Add the list constructor to the rpn:
data.handle_token(new CppFunction(&TokenList::default_constructor, "list"));
// We make the program see it as a normal function call:
data.handle_op("()");
}
// Add it as a bracket to the op stack:
data.open_bracket("[");
++expr;
break;
case '{':
// Add a map constructor call to the rpn:
data.handle_token(new CppFunction(&TokenMap::default_constructor, "map"));
// We make the program see it as a normal function call:
data.handle_op("()");
data.open_bracket("{");
++expr;
break;
case ')':
data.close_bracket("(");
++expr;
break;
case ']':
data.close_bracket("[");
++expr;
break;
case '}':
data.close_bracket("{");
++expr;
break;
default:
{
// Then the token is an operator
std::stringstream ss;
ss << *expr;
++expr;
while (*expr && ispunct(*expr) && !strchr("+-'\"()_", *expr)) {
ss << *expr;
++expr;
}
std::string op = ss.str();
rWordMap_t::iterator it;
if ((it=rWordMap.find(op)) != rWordMap.end()) {
// Parse reserved operators:
try {
it->second(expr, &expr, &data);
} catch (...) {
rpnBuilder::cleanRPN(&data.rpn);
throw;
}
} else {
data.handle_op(op);
}
}
}
}
// Ignore spaces but stop on delimiter if not inside brackets.
while (*expr && isspace(*expr)
&& (data.bracketLevel || !strchr(delim, *expr))) ++expr;
}
// Check for syntax errors (excess of operators i.e. 10 + + -1):
if (data.lastTokenWasUnary) {
rpnBuilder::cleanRPN(&data.rpn);
throw syntax_error("Expected operand after unary operator `" + data.opStack.top() + "`");
}
while (!data.opStack.empty()) {
data.rpn.push(new Token<std::string>(data.opStack.top(), OP));
data.opStack.pop();
}
// In case one of the custom parsers left an empty expression:
if (data.rpn.size() == 0) data.rpn.push(new TokenNone());
if (rest) *rest = expr;
return data.rpn;
}
