void CharNgrams::addTokens( Vector<IToken*> * tokens )
{
bool ret = false;
if ( tokens )
{
int count = (int)(tokens->count());
for ( int i=0; i<count; i++ )
{
IToken * token = tokens->get(i);
string str = token->getTokenString();
int len = (int)(str.length());
for ( int j=0;j<len; j++ )
{
IToken * newToken = token->clone( str[j] );
addToken( newToken );
delete newToken;
}
printf("addTokens %s.\n", tokens->get(i)->getTokenString().c_str() );
//addToken( tokens->get( i ) );
}
ret = true;
}
}
//---------------------------------------------------------------------------
const IValue& ParserXBase::ParseFromRPN() const
{
ptr_val_type *pStack = &m_vStackBuffer[0];
const ptr_tok_type *pRPN = &(m_rpn.GetData()[0]);
int sidx = -1;
std::size_t lenRPN = m_rpn.GetSize();
for (std::size_t i=0; i<lenRPN; ++i)
{
IToken *pTok = pRPN[i].Get();
ECmdCode eCode = pTok->GetCode();
switch (eCode)
{
case cmSCRIPT_NEWLINE:
sidx = -1; //-= static_cast<TokenNewline*>(pTok)->GetStackOffset();
m_nFinalResultIdx = 0;
continue;
case cmVAR:
{
sidx++;
assert(sidx<(int)m_vStackBuffer.size());
pStack[sidx].Reset(static_cast<IValue*>(pTok));
}
continue;
case cmVAL:
{
sidx++;
assert(sidx<(int)m_vStackBuffer.size());
ptr_val_type &val = pStack[sidx];
if (val->GetCode()==cmVAR)
val.Reset(m_cache.CreateFromCache());
*val = *(static_cast<IValue*>(pTok));
}
continue;
case cmIC:
{
IOprtIndex *pIdxOprt = static_cast<IOprtIndex*>(pTok);
int nArgs = pIdxOprt->GetArgsPresent();
sidx -= nArgs - 1;
assert(sidx>=0);
ptr_val_type &idx = pStack[sidx]; // Pointer to the first index
ptr_val_type &val = pStack[--sidx]; // Pointer to the variable or value beeing indexed
pIdxOprt->At(val, &idx, nArgs);
/*
// apply the index operator
ptr_val_type &idx = pStack[sidx--];
ptr_val_type &val = pStack[sidx];
MUP_ASSERT(val->GetCode()==cmVAR);
int i;
try
{
i = idx->GetInteger();
if (i<0)
Error(ecINDEX_OUT_OF_BOUNDS, pTok->GetExprPos(), val.Get());
}
catch(ParserError &exc)
{
if (exc.GetCode()==ecTYPE_CONFLICT)
Error(ecTYPE_CONFLICT_IDX, pTok->GetExprPos(), val.Get());
else
throw;
}
val.Reset(new Variable( &(val->At(i)) ) );
*/
}
continue;
case cmOPRT_POSTFIX:
case cmFUNC:
case cmOPRT_BIN:
case cmOPRT_INFIX:
{
ICallback *pFun = static_cast<ICallback*>(pTok);
int nArgs = pFun->GetArgsPresent();
sidx -= nArgs - 1;
assert(sidx>=0);
ptr_val_type &val = pStack[sidx];
try
{
if (val->GetCode()==cmVAR)
{
ptr_val_type buf(m_cache.CreateFromCache());
pFun->Eval(buf, &val, nArgs);
val = buf;
}
else
pFun->Eval(val, &val, nArgs);
}
catch(ParserError &exc)
{
ErrorContext err;
err.Expr = m_pTokenReader->GetExpr();
err.Ident = pFun->GetIdent();
err.Errc = ecEVAL;
err.Pos = pFun->GetExprPos();
err.Hint = exc.GetMsg();
throw ParserError(err);
}
catch(MatrixError & /*exc*/)
{
ErrorContext err;
err.Expr = m_pTokenReader->GetExpr();
err.Ident = pFun->GetIdent();
err.Errc = ecEVAL;
err.Pos = pFun->GetExprPos();
err.Hint = _T("Matrix dimension mismatch");
throw ParserError(err);
}
}
continue;
case cmIF:
MUP_ASSERT(sidx>=0);
if (pStack[sidx--]->GetBool()==false)
i+=static_cast<TokenIfThenElse*>(pTok)->GetOffset();
continue;
case cmELSE:
case cmJMP:
i += static_cast<TokenIfThenElse*>(pTok)->GetOffset();
continue;
case cmENDIF:
continue;
default:
Error(ecINTERNAL_ERROR);
} // switch token
} // for all RPN tokens
return *pStack[m_nFinalResultIdx];
}
//---------------------------------------------------------------------------
const IValue& ParserXBase::ParseFromRPN() const
{
ptr_val_type *pStack = &m_vStackBuffer[0];
if (m_rpn.GetSize()==0)
{
// Passiert bei leeren strings oder solchen, die nur Leerzeichen enthalten
ErrorContext err;
err.Expr = m_pTokenReader->GetExpr();
err.Errc = ecUNEXPECTED_EOF;
err.Pos = 0;
throw ParserError(err);
}
const ptr_tok_type *pRPN = &(m_rpn.GetData()[0]);
int sidx = -1;
std::size_t lenRPN = m_rpn.GetSize();
for (std::size_t i=0; i<lenRPN; ++i)
{
IToken *pTok = pRPN[i].Get();
ECmdCode eCode = pTok->GetCode();
switch (eCode)
{
case cmSCRIPT_NEWLINE:
sidx = -1;
continue;
case cmVAL:
{
IValue *pVal = static_cast<IValue*>(pTok);
sidx++;
assert(sidx<(int)m_vStackBuffer.size());
if (pVal->IsVariable())
{
pStack[sidx].Reset(pVal);
}
else
{
ptr_val_type &val = pStack[sidx];
if (val->IsVariable())
val.Reset(m_cache.CreateFromCache());
*val = *(static_cast<IValue*>(pTok));
}
}
continue;
case cmIC:
{
IOprtIndex *pIdxOprt = static_cast<IOprtIndex*>(pTok);
int nArgs = pIdxOprt->GetArgsPresent();
sidx -= nArgs - 1;
assert(sidx>=0);
ptr_val_type &idx = pStack[sidx]; // Pointer to the first index
ptr_val_type &val = pStack[--sidx]; // Pointer to the variable or value beeing indexed
pIdxOprt->At(val, &idx, nArgs);
}
continue;
case cmOPRT_POSTFIX:
case cmFUNC:
case cmOPRT_BIN:
case cmOPRT_INFIX:
{
ICallback *pFun = static_cast<ICallback*>(pTok);
int nArgs = pFun->GetArgsPresent();
sidx -= nArgs - 1;
assert(sidx>=0);
ptr_val_type &val = pStack[sidx];
try
{
if (val->IsVariable())
{
ptr_val_type buf(m_cache.CreateFromCache());
pFun->Eval(buf, &val, nArgs);
val = buf;
}
else
pFun->Eval(val, &val, nArgs);
}
catch(ParserError &exc)
{
// <ibg 20130131> Not too happy about that:
// Multiarg functions may throw specific error codes when evaluating.
// These codes would be converted to ecEVAL here. I omit the conversion
// for certain handpicked errors. (The reason this catch block exists is
// that not all exceptions contain proper metadata when thrown out of
// a function.)
if (exc.GetCode()==ecTOO_FEW_PARAMS || exc.GetCode()==ecDOMAIN_ERROR || exc.GetCode()==ecOVERFLOW)
throw;
// </ibg>
ErrorContext err;
err.Expr = m_pTokenReader->GetExpr();
err.Ident = pFun->GetIdent();
err.Errc = ecEVAL;
err.Pos = pFun->GetExprPos();
err.Hint = exc.GetMsg();
throw ParserError(err);
}
catch(MatrixError & /*exc*/)
{
ErrorContext err;
err.Expr = m_pTokenReader->GetExpr();
err.Ident = pFun->GetIdent();
err.Errc = ecMATRIX_DIMENSION_MISMATCH;
err.Pos = pFun->GetExprPos();
throw ParserError(err);
}
}
continue;
case cmIF:
MUP_ASSERT(sidx>=0);
if (pStack[sidx--]->GetBool()==false)
i+=static_cast<TokenIfThenElse*>(pTok)->GetOffset();
continue;
case cmELSE:
case cmJMP:
i += static_cast<TokenIfThenElse*>(pTok)->GetOffset();
continue;
case cmENDIF:
continue;
default:
Error(ecINTERNAL_ERROR);
} // switch token
} // for all RPN tokens
return *pStack[0];
}
