Esempio n. 1
0
			Token & ReadToken(TokenType type)
			{
				if (pos >= tokens.Count())
				{
					errors.Add(CompileError(TokenTypeToString(type) + String(L" expected but end of file encountered."), 0, CodePosition(0, 0, fileName)));
					throw 0;
				}
				else if (tokens[pos].Type != type)
				{
					errors.Add(CompileError(TokenTypeToString(type) + String(L" expected"), 20001, tokens[pos].Position));
					throw 20001;
				}
				return tokens[pos++];
			}
Esempio n. 2
0
string CPlmLine :: GetStr ()  const
{

	string Result = m_Word;
	if (m_bHomonym) Result = "  "+Result;
	Result += " ";
    Result += Format (" %i %i ", m_FilePosition, m_TokenLengthInFile);


	if (m_TokenType != OTHER_TOKEN_TYPE)
	{
		MainTokenTypeEnum t = m_TokenType;
		if (t == ROMAN_NUM)
			t = NUM;
		Result += " ";
		Result += TokenTypeToString(t);
		Result += " ";
	}

	if (   (m_TokenType == RLE)
		|| (m_TokenType == LLE)
	   )
	{
		if (m_Register == LowLow)
			Result +=  " aa ";
		else
		if (m_Register == UpLow)
			Result +=  " Aa ";
		else
		if (m_Register == UpUp)
			Result +=  " AA ";
	};

	
	Result += m_GraphDescr + " ";

	if (m_bQuoteMarks)
		Result +=  " #QUOTED ";

	if (m_bFI1) 
		Result +=  " FAM1 ";
	if (m_bFI2) 
		Result +=  " FAM2 ";
	if (m_bName) 
		Result +=  " NAM? ";
	if (m_bSent2) 
		Result +=  " SENT_END ";

	if (m_MorphSign != 0)
	{
		Result += m_MorphSign+m_CommonGramCode + " " + m_Lemma + " " + m_GramCodes + " ";
		Result +=  m_ParadigmId + " " + m_HomoWeight;
	};

	return  Result;
};
//==============================================================================
void ExpressionParser::ThrowInvalidTokenError_(const ExpressionParser::Token &token, const std::string& descriptionText)
{
  FlushOutput_();
  std::stringstream ss;
  ss << "unexpected token " << TokenTypeToString(token.type) << " : \""
     << token.value << "\" at " << token.line << "-" << token.column;
  if (!descriptionText.empty())
  {
    ss << ", " << descriptionText;
  }
  throw std::logic_error(ss.str());
}
Esempio n. 4
0
string CGrammarItem::GetDumpString() const 
{

	if (!m_bMeta && !m_Token.empty() && m_Attributes.empty() &&  m_MorphPattern.m_GrmAttribute.empty() ) 
		return Format("'%s'",m_ItemStrId.c_str());

	string Attributes;
	for (map<string, string>::const_iterator it = m_Attributes.begin(); it != m_Attributes.end(); it++)
		Attributes += Format ("%s=%s ",it->first.c_str(),it->second.c_str());

	if (!m_MorphPattern.m_GrmAttribute.empty())
		Attributes += Format ("grm=\"%s\" ", m_MorphPattern.m_GrmAttribute.c_str());

	if (!m_bCanHaveManyHomonyms)
		Attributes += "hom=\"no\" ";

	if (m_Register != AnyRegister)
		if (m_Register == LowLow)
			Attributes += "register=\"aa\" ";
		else
			if (m_Register == UpLow)
				Attributes += "register=\"Aa\" ";
			else
				Attributes += "register=\"AA\" ";

	if (m_TokenType != OTHER_TOKEN_TYPE)
		Attributes += Format("type=\"%s\" ", TokenTypeToString(m_TokenType).c_str());

	if (m_bSynMain)
		Attributes += "root ";


	Trim(Attributes);

	string Meta;
	if (m_bMeta)
		Meta = m_ItemStrId;
	else
		Meta = "TOKEN";


	return Format("[%s %s]",Meta.c_str(),Attributes.c_str());
};
Esempio n. 5
0
bool CPlmLine :: LoadPlmLineFromString (string LineStr, bool bStartLine, const CAgramtab* pRusGramTab) 
{
	m_bToDelete = false;
	m_bQuoteMarks = false;
	
	if (!bStartLine)
	{
		m_bHomonym = (LineStr[0] == ' ');

		Trim(LineStr);
		SetWord(::GetWordForm(LineStr) );
	}
	else
	{
		m_bHomonym = false;
		SetWord( "" );

	};


	long i = 1; // the first char can be a space (if this line contains a homonym)
	for (; i < LineStr.length(); i++)
		if (!isspace((BYTE) LineStr[i]) == 0)
			break;

	if (sscanf (LineStr.c_str()+i, "%i %i", &m_FilePosition, &m_TokenLengthInFile) != 2)
		return false;

	// pass all numbers
	for (; i < LineStr.length(); i++)
		 if (		(isdigit((BYTE) LineStr[i]) == 0)
				&&	(isspace((BYTE) LineStr[i]) == 0)
				&&	(((BYTE)LineStr[i]) != '-')
			)
			break;

	int MorphSignPos = GetMorphSignPosition(LineStr.c_str()+i);
	if (MorphSignPos == -1)
		MorphSignPos = LineStr.length();
	else
		MorphSignPos += i; // make MorphSignPos an absolute offset in LineStr

	m_GraphDescr = LineStr.substr (i, MorphSignPos - i);
	/* вставим пробел в начало, потому что часто ищут графету с пробелом в начале,
	например, " ЛЕ"*/
	m_GraphDescr = " " + m_GraphDescr;

	if (MorphSignPos != LineStr.length())
	{
		StringTokenizer tok(LineStr.c_str()+MorphSignPos," ");

		if (!tok() ) return false;
		string MorphSign  = tok.val();
		if (MorphSign.length() != 3) return false;
		m_MorphSign = MorphSign[0];
		m_CommonGramCode = MorphSign.substr(1);

		
		if (!tok() ) return false;
		m_Lemma = tok.val();
		if (m_Lemma.empty())  return false;
		
		if (!tok() ) return false;
		SetGramCodes ( tok.val(), pRusGramTab);

		if (!tok() ) return false;
		m_ParadigmId = tok.val();

		if (!tok() ) return false;
		m_HomoWeight = tok.val();

	};

	m_TokenType = OTHER_TOKEN_TYPE;
	for (int k=(int)RLE; k < OTHER_TOKEN_TYPE; k++)
		if (init_flag (m_GraphDescr, TokenTypeToString((MainTokenTypeEnum)k).c_str() ))
		{
            m_TokenType = (MainTokenTypeEnum)k;
			break;
		};

	if (init_flag (m_GraphDescr, "Aa"))
		m_Register = UpLow;
	else
	if (init_flag (m_GraphDescr, "AA"))
		m_Register = UpUp;
	else
	{
		init_flag (m_GraphDescr, "aa");
		m_Register = LowLow;		
	};

	m_bFirstUpperAlpha =  (m_Register == UpUp) || (m_Register == UpLow);

	m_bFI1 = init_flag (m_GraphDescr, "FAM1");
	m_bFI2 = init_flag (m_GraphDescr, "FAM2");
	m_bName = init_flag (m_GraphDescr, "NAM?");
	m_bSent2 = init_flag (m_GraphDescr, "SENT_END");
	int hyphen_occur = m_Word.find("-");
	m_bHyphenWord = (hyphen_occur != string::npos) && ( (m_TokenType == RLE) ||(m_TokenType == LLE));

	m_bOborot1 = (m_GraphDescr.find("EXPR1") != string::npos);
	m_bOborot2 = (m_GraphDescr.find("EXPR2") != string::npos);
	bool bRomanNumber = is_roman_number(m_Word.c_str(), m_Word.length());
	
	if ((hyphen_occur != string::npos) && (hyphen_occur!=0))
	{
		// "Павла I-го" 
		// "I-го" - одно слово
		bRomanNumber = is_roman_number(m_Word.c_str(), hyphen_occur);
	};
	if (bRomanNumber)
	{
		m_TokenType = ROMAN_NUM;
		m_CommonGramCode = "";
		m_MorphSign = 0;
		m_ParadigmId = "";
	};
	Trim(m_GraphDescr);
	return true;

};