void CharNgrams::addTokens( Vector<IToken*> * tokens ) { bool ret = false; if ( tokens ) { int count = (int)(tokens->count()); for ( int i=0; i<count; i++ ) { IToken * token = tokens->get(i); string str = token->getTokenString(); int len = (int)(str.length()); for ( int j=0;j<len; j++ ) { IToken * newToken = token->clone( str[j] ); addToken( newToken ); delete newToken; } printf("addTokens %s.\n", tokens->get(i)->getTokenString().c_str() ); //addToken( tokens->get( i ) ); } ret = true; } }
//--------------------------------------------------------------------------- const IValue& ParserXBase::ParseFromRPN() const { ptr_val_type *pStack = &m_vStackBuffer[0]; const ptr_tok_type *pRPN = &(m_rpn.GetData()[0]); int sidx = -1; std::size_t lenRPN = m_rpn.GetSize(); for (std::size_t i=0; i<lenRPN; ++i) { IToken *pTok = pRPN[i].Get(); ECmdCode eCode = pTok->GetCode(); switch (eCode) { case cmSCRIPT_NEWLINE: sidx = -1; //-= static_cast<TokenNewline*>(pTok)->GetStackOffset(); m_nFinalResultIdx = 0; continue; case cmVAR: { sidx++; assert(sidx<(int)m_vStackBuffer.size()); pStack[sidx].Reset(static_cast<IValue*>(pTok)); } continue; case cmVAL: { sidx++; assert(sidx<(int)m_vStackBuffer.size()); ptr_val_type &val = pStack[sidx]; if (val->GetCode()==cmVAR) val.Reset(m_cache.CreateFromCache()); *val = *(static_cast<IValue*>(pTok)); } continue; case cmIC: { IOprtIndex *pIdxOprt = static_cast<IOprtIndex*>(pTok); int nArgs = pIdxOprt->GetArgsPresent(); sidx -= nArgs - 1; assert(sidx>=0); ptr_val_type &idx = pStack[sidx]; // Pointer to the first index ptr_val_type &val = pStack[--sidx]; // Pointer to the variable or value beeing indexed pIdxOprt->At(val, &idx, nArgs); /* // apply the index operator ptr_val_type &idx = pStack[sidx--]; ptr_val_type &val = pStack[sidx]; MUP_ASSERT(val->GetCode()==cmVAR); int i; try { i = idx->GetInteger(); if (i<0) Error(ecINDEX_OUT_OF_BOUNDS, pTok->GetExprPos(), val.Get()); } catch(ParserError &exc) { if (exc.GetCode()==ecTYPE_CONFLICT) Error(ecTYPE_CONFLICT_IDX, pTok->GetExprPos(), val.Get()); else throw; } val.Reset(new Variable( &(val->At(i)) ) ); */ } continue; case cmOPRT_POSTFIX: case cmFUNC: case cmOPRT_BIN: case cmOPRT_INFIX: { ICallback *pFun = static_cast<ICallback*>(pTok); int nArgs = pFun->GetArgsPresent(); sidx -= nArgs - 1; assert(sidx>=0); ptr_val_type &val = pStack[sidx]; try { if (val->GetCode()==cmVAR) { ptr_val_type buf(m_cache.CreateFromCache()); pFun->Eval(buf, &val, nArgs); val = buf; } else pFun->Eval(val, &val, nArgs); } catch(ParserError &exc) { ErrorContext err; err.Expr = m_pTokenReader->GetExpr(); err.Ident = pFun->GetIdent(); err.Errc = ecEVAL; err.Pos = pFun->GetExprPos(); err.Hint = exc.GetMsg(); throw ParserError(err); } catch(MatrixError & /*exc*/) { ErrorContext err; err.Expr = m_pTokenReader->GetExpr(); err.Ident = pFun->GetIdent(); err.Errc = ecEVAL; err.Pos = pFun->GetExprPos(); err.Hint = _T("Matrix dimension mismatch"); throw ParserError(err); } } continue; case cmIF: MUP_ASSERT(sidx>=0); if (pStack[sidx--]->GetBool()==false) i+=static_cast<TokenIfThenElse*>(pTok)->GetOffset(); continue; case cmELSE: case cmJMP: i += static_cast<TokenIfThenElse*>(pTok)->GetOffset(); continue; case cmENDIF: continue; default: Error(ecINTERNAL_ERROR); } // switch token } // for all RPN tokens return *pStack[m_nFinalResultIdx]; }
//--------------------------------------------------------------------------- const IValue& ParserXBase::ParseFromRPN() const { ptr_val_type *pStack = &m_vStackBuffer[0]; if (m_rpn.GetSize()==0) { // Passiert bei leeren strings oder solchen, die nur Leerzeichen enthalten ErrorContext err; err.Expr = m_pTokenReader->GetExpr(); err.Errc = ecUNEXPECTED_EOF; err.Pos = 0; throw ParserError(err); } const ptr_tok_type *pRPN = &(m_rpn.GetData()[0]); int sidx = -1; std::size_t lenRPN = m_rpn.GetSize(); for (std::size_t i=0; i<lenRPN; ++i) { IToken *pTok = pRPN[i].Get(); ECmdCode eCode = pTok->GetCode(); switch (eCode) { case cmSCRIPT_NEWLINE: sidx = -1; continue; case cmVAL: { IValue *pVal = static_cast<IValue*>(pTok); sidx++; assert(sidx<(int)m_vStackBuffer.size()); if (pVal->IsVariable()) { pStack[sidx].Reset(pVal); } else { ptr_val_type &val = pStack[sidx]; if (val->IsVariable()) val.Reset(m_cache.CreateFromCache()); *val = *(static_cast<IValue*>(pTok)); } } continue; case cmIC: { IOprtIndex *pIdxOprt = static_cast<IOprtIndex*>(pTok); int nArgs = pIdxOprt->GetArgsPresent(); sidx -= nArgs - 1; assert(sidx>=0); ptr_val_type &idx = pStack[sidx]; // Pointer to the first index ptr_val_type &val = pStack[--sidx]; // Pointer to the variable or value beeing indexed pIdxOprt->At(val, &idx, nArgs); } continue; case cmOPRT_POSTFIX: case cmFUNC: case cmOPRT_BIN: case cmOPRT_INFIX: { ICallback *pFun = static_cast<ICallback*>(pTok); int nArgs = pFun->GetArgsPresent(); sidx -= nArgs - 1; assert(sidx>=0); ptr_val_type &val = pStack[sidx]; try { if (val->IsVariable()) { ptr_val_type buf(m_cache.CreateFromCache()); pFun->Eval(buf, &val, nArgs); val = buf; } else pFun->Eval(val, &val, nArgs); } catch(ParserError &exc) { // <ibg 20130131> Not too happy about that: // Multiarg functions may throw specific error codes when evaluating. // These codes would be converted to ecEVAL here. I omit the conversion // for certain handpicked errors. (The reason this catch block exists is // that not all exceptions contain proper metadata when thrown out of // a function.) if (exc.GetCode()==ecTOO_FEW_PARAMS || exc.GetCode()==ecDOMAIN_ERROR || exc.GetCode()==ecOVERFLOW) throw; // </ibg> ErrorContext err; err.Expr = m_pTokenReader->GetExpr(); err.Ident = pFun->GetIdent(); err.Errc = ecEVAL; err.Pos = pFun->GetExprPos(); err.Hint = exc.GetMsg(); throw ParserError(err); } catch(MatrixError & /*exc*/) { ErrorContext err; err.Expr = m_pTokenReader->GetExpr(); err.Ident = pFun->GetIdent(); err.Errc = ecMATRIX_DIMENSION_MISMATCH; err.Pos = pFun->GetExprPos(); throw ParserError(err); } } continue; case cmIF: MUP_ASSERT(sidx>=0); if (pStack[sidx--]->GetBool()==false) i+=static_cast<TokenIfThenElse*>(pTok)->GetOffset(); continue; case cmELSE: case cmJMP: i += static_cast<TokenIfThenElse*>(pTok)->GetOffset(); continue; case cmENDIF: continue; default: Error(ecINTERNAL_ERROR); } // switch token } // for all RPN tokens return *pStack[0]; }