TEST(scanner, token) {
StringTable table;
Pos start(1, 1);
Pos end(2, 2);
// test for bool literal token
auto boolToken = Token::makeBoolLiteral(true, start, end);
EXPECT_EQ(true, boolToken.getBoolValue());
EXPECT_EQ(TK_BOOL_CONST, boolToken.getTokenKind());
// test for string literal token
auto symbol = table.addString("zhang");
auto stringLiteralToken = Token::makeStrLiteral(symbol, start, end);
EXPECT_EQ(TK_STR_CONST, stringLiteralToken.getTokenKind());
EXPECT_TRUE(stringLiteralToken.getSymbol()->equalString("zhang"));
// test for identifier token
auto idToken = Token::makeIdentifier(symbol, start, end);
EXPECT_EQ(TK_ID, idToken.getTokenKind());
EXPECT_TRUE(idToken.getSymbol()->equalString("zhang"));
}
void CPBSAnalysis::processToken(TRSessionManager *sess, CTRAnalise*an, CTRToken *tk, CPBSSink* pbssink, std::map<short, int> &onts_doc, char * _textChunck, int offset) {
if (!tk)
return;
if (tk->obtemTipo() & TK_CONTR_P1) {
if (tk->obtemTokenLocucao())
processSepToken(sess, tk, tk->obtemTokenLocucao(), pbssink, _textChunck, offset);
// obtem o pr�ximo token da locu��o e trata
processToken(sess, an, (CTRToken*)tk->obtemTokenLocucao(), pbssink, onts_doc, _textChunck, offset);
return;
}
unsigned char doms[255], n_doms = 255, n_onts = 255;
unsigned short onts[255];
// Primeiro colocar a locu��o ou token
char* token = (char *)alloca(tk->obtemLen() + 1);
strncpy(token, _textChunck + tk->obtemPosInicial(), tk->obtemLen());
token[tk->obtemLen()] = '\0';
std::string word = token;
pbssink->PutToken(word, tk->obtemLen(), tk->obtemPosInicial() + offset, getTokenKind(tk));
OpenSemTag *tagloc = sess->obtemOSCData()->checkToken(tk);
if (tagloc) {
std::string name = tagloc->getName();
std::string value = tagloc->getValue(an, tk);
pbssink->PutFeature(name, value);
}
OpenSemTag *tag;
if ((tag = sess->obtemOSCData()->getTagLemma()) != NULL && tag->checkToken(tk)) {
std::string name = tag->getName();
std::string value = tag->getValue(an, tk);
pbssink->PutFeature(name, value);
}
if ((tag = sess->obtemOSCData()->getTagHead()) != NULL && tag->checkToken(tk)) {
std::string name = tag->getName();
std::string value = tag->getValue(an, tk);
pbssink->PutFeature(name, value);
}
if ((tag = sess->obtemOSCData()->getTagMrcat()) != NULL && tag->checkToken(tk)) {
std::string name = tag->getName();
std::string value = tag->getValue(an, tk);
pbssink->PutFeature(name, value);
}
if ((tag = sess->obtemOSCData()->getTagFullcat()) != NULL && tag->checkToken(tk)) {
std::string name = tag->getName();
std::string value = tag->getValue(an, tk);
pbssink->PutFeature(name, value);
}
if (sess->obtemOSCData()->getTagOnt1() || sess->obtemOSCData()->getTagOnt1Raw()) {
tk->obtemInfoOnt(an->obtemTRExec(), doms, n_doms, onts, NULL, n_onts);
std::map<short, int>::iterator itr;
for (int i = 0; i < n_onts; i++) {
if ((itr = onts_doc.find(onts[i])) != onts_doc.end())
itr->second++;
else
onts_doc.insert(std::pair<short, int>(onts[i], 1));
char stont[200];
if (sess->obtemOSCData()->getTagOnt1()) {
unsigned char id_nivel1, id_nivel2, id_nivel3, id_nivel4;
char* descs[4];
CTROntDataNos* nos_ont = an->obtemTRData()->obtemTROntData()->obtemTROntDataNos();
nos_ont->obtemNiveis(onts[i], id_nivel1, id_nivel2, id_nivel3, id_nivel4);
nos_ont->obtemDescricoesNiveis(descs, id_nivel1, id_nivel2, id_nivel3, id_nivel4);
sprintf(stont, "%d.%d.%d.%d - %s / %s / %s / %s", id_nivel1 + 1, id_nivel2 + 1, id_nivel3 + 1, id_nivel4 + 1, descs[0], descs[1], descs[2], descs[3]);
std::string name = sess->obtemOSCData()->getTagOnt1()->getName();
std::string value = stont;
pbssink->PutFeature(name, value);
}
if (sess->obtemOSCData()->getTagOnt1Raw()) {
unsigned char id_nivel1, id_nivel2, id_nivel3, id_nivel4;
CTROntDataNos* nos_ont = an->obtemTRData()->obtemTROntData()->obtemTROntDataNos();
nos_ont->obtemNiveis(onts[i], id_nivel1, id_nivel2, id_nivel3, id_nivel4);
sprintf(stont, "%02d.%02d.%02d.%02d", id_nivel1 + 1, id_nivel2 + 1, id_nivel3 + 1, id_nivel4 + 1);
std::string name = sess->obtemOSCData()->getTagOnt1Raw()->getName();
std::string value = stont;
pbssink->PutFeature(name, value);
}
}
}
std::string name = "pbt.endfeatures";
std::string value = "";
pbssink->PutFeature(name, value);
if (tk->obtemTokenLocucao())
processSepToken(sess, tk, tk->obtemTokenLocucao(), pbssink, _textChunck, offset);
// obtem o pr�ximo token da locu��o e trata
processToken(sess, an, (CTRToken*)tk->obtemTokenLocucao(), pbssink, onts_doc, _textChunck, offset);
}
