Esempio n. 1
0
void CharNgrams::addTokens( Vector<IToken*> * tokens )
{
  bool ret = false;
  if ( tokens )
    {
      int count = (int)(tokens->count());
      for ( int i=0; i<count; i++ )
	{
	  IToken * token = tokens->get(i);
	  string str = token->getTokenString();
	  int len = (int)(str.length());
	  for ( int j=0;j<len; j++ )
	  {
	    IToken * newToken = token->clone( str[j] );
	    addToken( newToken );
	    delete newToken;
	  }
	  printf("addTokens %s.\n", tokens->get(i)->getTokenString().c_str() );
	  //addToken( tokens->get( i ) );
	}
      ret = true;
    }

}
Esempio n. 2
0
  //---------------------------------------------------------------------------
  const IValue& ParserXBase::ParseFromRPN() const
  {
    ptr_val_type *pStack = &m_vStackBuffer[0];
    const ptr_tok_type *pRPN = &(m_rpn.GetData()[0]);

    int sidx = -1;
    std::size_t lenRPN = m_rpn.GetSize();
    for (std::size_t i=0; i<lenRPN; ++i)
    {
      IToken *pTok = pRPN[i].Get();
      ECmdCode eCode = pTok->GetCode();

      switch (eCode)
      {
      case cmSCRIPT_NEWLINE:
           sidx = -1; //-= static_cast<TokenNewline*>(pTok)->GetStackOffset();
           m_nFinalResultIdx = 0;
           continue;

      case cmVAR:
           {
             sidx++;
             assert(sidx<(int)m_vStackBuffer.size());
             pStack[sidx].Reset(static_cast<IValue*>(pTok));
           }
           continue;

      case cmVAL:
           {
             sidx++;
             assert(sidx<(int)m_vStackBuffer.size());
             ptr_val_type &val = pStack[sidx];
             if (val->GetCode()==cmVAR)
               val.Reset(m_cache.CreateFromCache());

             *val = *(static_cast<IValue*>(pTok));
           }
           continue;

      case  cmIC:
            {
              IOprtIndex *pIdxOprt = static_cast<IOprtIndex*>(pTok);
              int nArgs = pIdxOprt->GetArgsPresent();
              sidx -= nArgs - 1;
              assert(sidx>=0);

              ptr_val_type &idx = pStack[sidx];     // Pointer to the first index
              ptr_val_type &val = pStack[--sidx];   // Pointer to the variable or value beeing indexed
              pIdxOprt->At(val, &idx, nArgs);




/*

              // apply the index operator
              ptr_val_type &idx = pStack[sidx--];
              ptr_val_type &val = pStack[sidx];
              MUP_ASSERT(val->GetCode()==cmVAR);
             
              int i;
              try
              {
                i = idx->GetInteger();
                if (i<0)
                  Error(ecINDEX_OUT_OF_BOUNDS, pTok->GetExprPos(), val.Get());
              }
              catch(ParserError &exc)
              {
                if (exc.GetCode()==ecTYPE_CONFLICT)
                  Error(ecTYPE_CONFLICT_IDX, pTok->GetExprPos(), val.Get());
                else 
                  throw;
              }
              val.Reset(new Variable( &(val->At(i)) ) );
*/
            }
            continue;

      case cmOPRT_POSTFIX:
      case cmFUNC:
      case cmOPRT_BIN:
      case cmOPRT_INFIX:
           {
             ICallback *pFun = static_cast<ICallback*>(pTok);
             int nArgs = pFun->GetArgsPresent();
             sidx -= nArgs - 1;
             assert(sidx>=0);

             ptr_val_type &val = pStack[sidx];
             try
             {
               if (val->GetCode()==cmVAR)
               {
                 ptr_val_type buf(m_cache.CreateFromCache());
                 pFun->Eval(buf, &val, nArgs);
                 val = buf;
               }
               else
                 pFun->Eval(val, &val, nArgs);
             }
             catch(ParserError &exc)
             {
               ErrorContext err;
               err.Expr = m_pTokenReader->GetExpr();
               err.Ident = pFun->GetIdent();
               err.Errc = ecEVAL;
               err.Pos = pFun->GetExprPos();
               err.Hint = exc.GetMsg();
               throw ParserError(err);
             }
             catch(MatrixError & /*exc*/)
             {
               ErrorContext err;
               err.Expr = m_pTokenReader->GetExpr();
               err.Ident = pFun->GetIdent();
               err.Errc = ecEVAL;
               err.Pos = pFun->GetExprPos();
               err.Hint = _T("Matrix dimension mismatch");
               throw ParserError(err);
             }
           }
           continue;

      case cmIF:
           MUP_ASSERT(sidx>=0);
           if (pStack[sidx--]->GetBool()==false)
             i+=static_cast<TokenIfThenElse*>(pTok)->GetOffset();
           continue;

      case cmELSE:
      case cmJMP:
           i += static_cast<TokenIfThenElse*>(pTok)->GetOffset();
           continue;

      case cmENDIF:
           continue;

			default:
					Error(ecINTERNAL_ERROR);
      } // switch token
    } // for all RPN tokens

    return *pStack[m_nFinalResultIdx];
  }
//---------------------------------------------------------------------------
const IValue& ParserXBase::ParseFromRPN() const
{
    ptr_val_type *pStack = &m_vStackBuffer[0];
    if (m_rpn.GetSize()==0)
    {
        // Passiert bei leeren strings oder solchen, die nur Leerzeichen enthalten
        ErrorContext err;
        err.Expr = m_pTokenReader->GetExpr();
        err.Errc = ecUNEXPECTED_EOF;
        err.Pos  = 0;
        throw ParserError(err);
    }

    const ptr_tok_type *pRPN = &(m_rpn.GetData()[0]);

    int sidx = -1;
    std::size_t lenRPN = m_rpn.GetSize();
    for (std::size_t i=0; i<lenRPN; ++i)
    {
        IToken *pTok = pRPN[i].Get();
        ECmdCode eCode = pTok->GetCode();

        switch (eCode)
        {
        case cmSCRIPT_NEWLINE:
            sidx = -1;
            continue;

        case cmVAL:
        {
            IValue *pVal = static_cast<IValue*>(pTok);

            sidx++;
            assert(sidx<(int)m_vStackBuffer.size());
            if (pVal->IsVariable())
            {
                pStack[sidx].Reset(pVal);
            }
            else
            {
                ptr_val_type &val = pStack[sidx];
                if (val->IsVariable())
                    val.Reset(m_cache.CreateFromCache());

                *val = *(static_cast<IValue*>(pTok));
            }
        }
        continue;

        case  cmIC:
        {
            IOprtIndex *pIdxOprt = static_cast<IOprtIndex*>(pTok);
            int nArgs = pIdxOprt->GetArgsPresent();
            sidx -= nArgs - 1;
            assert(sidx>=0);

            ptr_val_type &idx = pStack[sidx];     // Pointer to the first index
            ptr_val_type &val = pStack[--sidx];   // Pointer to the variable or value beeing indexed
            pIdxOprt->At(val, &idx, nArgs);
        }
        continue;

        case cmOPRT_POSTFIX:
        case cmFUNC:
        case cmOPRT_BIN:
        case cmOPRT_INFIX:
        {
            ICallback *pFun = static_cast<ICallback*>(pTok);
            int nArgs = pFun->GetArgsPresent();
            sidx -= nArgs - 1;
            assert(sidx>=0);

            ptr_val_type &val = pStack[sidx];
            try
            {
                if (val->IsVariable())
                {
                    ptr_val_type buf(m_cache.CreateFromCache());
                    pFun->Eval(buf, &val, nArgs);
                    val = buf;
                }
                else
                    pFun->Eval(val, &val, nArgs);
            }
            catch(ParserError &exc)
            {
                // <ibg 20130131> Not too happy about that:
                // Multiarg functions may throw specific error codes when evaluating.
                // These codes would be converted to ecEVAL here. I omit the conversion
                // for certain handpicked errors. (The reason this catch block exists is
                // that not all exceptions contain proper metadata when thrown out of
                // a function.)
                if (exc.GetCode()==ecTOO_FEW_PARAMS || exc.GetCode()==ecDOMAIN_ERROR || exc.GetCode()==ecOVERFLOW)
                    throw;
                // </ibg>

                ErrorContext err;
                err.Expr = m_pTokenReader->GetExpr();
                err.Ident = pFun->GetIdent();
                err.Errc = ecEVAL;
                err.Pos = pFun->GetExprPos();
                err.Hint = exc.GetMsg();
                throw ParserError(err);
            }
            catch(MatrixError & /*exc*/)
            {
                ErrorContext err;
                err.Expr = m_pTokenReader->GetExpr();
                err.Ident = pFun->GetIdent();
                err.Errc = ecMATRIX_DIMENSION_MISMATCH;
                err.Pos = pFun->GetExprPos();
                throw ParserError(err);
            }
        }
        continue;

        case cmIF:
            MUP_ASSERT(sidx>=0);
            if (pStack[sidx--]->GetBool()==false)
                i+=static_cast<TokenIfThenElse*>(pTok)->GetOffset();
            continue;

        case cmELSE:
        case cmJMP:
            i += static_cast<TokenIfThenElse*>(pTok)->GetOffset();
            continue;

        case cmENDIF:
            continue;

        default:
            Error(ecINTERNAL_ERROR);
        } // switch token
    } // for all RPN tokens

    return *pStack[0];
}