SyntaxTree* Parser::m_parse_orderedChoice(TArray<Parser*>* ps)
{
m_ErrorPos startPos = m_curErrPos();
int i = 0;
for (TArrayIterator<Parser*> itr(ps); itr.hasMore(); ++i) {
Parser* p = itr.next();
SyntaxTree* st = p->parse();
if (st->isError()) {
m_fail(startPos);
return st;
}
if (! st->isFail()) {
SyntaxTree::Childs* childs = new SyntaxTree::Childs(1);
childs->setContentsMemID("pcoc");
p->addSyntaxTreeToParent(childs, st);
SyntaxTree* ordst = createSyntaxTree(startPos.parsePos, childs);
ordst->chooseNum = i;
return ordst;
}
if (itr.hasMore())
m_back(startPos);
}
m_fail(startPos);
return m_PARSE_FAILED;
}
SyntaxTree* Parser::m_parse_sequence(TArray<Parser*>* ps)
{
m_ErrorPos startPos = m_curErrPos();
SyntaxTree::Childs* childs = new SyntaxTree::Childs(ps->size());
childs->setContentsMemID("pcsq");
int errorCutId = 0;
hyu32 errorCutPos = 0;
for (TArrayIterator<Parser*> itr(ps); itr.hasMore(); ) {
Parser* p = itr.next();
SyntaxTree* st = p->parse();
if (st->isErrorCut()) {
errorCutId = st->errorCutId;
errorCutPos = st->str.startPos;
delete st;
} else if (! st->isFail()) {
p->addSyntaxTreeToParent(childs, st);
} else {
if (st->isFailNotError() && errorCutId > 0) {
SyntaxTree* e = new SyntaxTree(errorCutPos, errorCutId, this);
if (! m_bUseMemoize)
delete st;
st = e;
}
m_fail(startPos);
if (! m_bUseMemoize) {
SyntaxTree::Childs::deleteRecursively(childs);
} else {
delete childs;
}
return st;
}
}
return createSyntaxTree(startPos.parsePos, childs);
}
SyntaxTree* Parser::m_parse_m2n(Parser* p, int m, int n)
{
int count = 0;
m_ErrorPos startPos = m_curErrPos();
SyntaxTree::Childs* childs = new SyntaxTree::Childs(m > 0 ? m : 1);
childs->setContentsMemID("pcmn");
for (;;) {
SyntaxTree* st = p->parse();
if (st->isError()) {
m_fail(startPos);
return st;
}
if (! st->isFail()) {
p->addSyntaxTreeToParent(childs, st);
if (++count >= MAX_MANY) count = MAX_MANY - 1;
if (count >= n)
break; // all ok
} else {
break;
}
}
if (count < m) {
m_fail(startPos);
if (! m_bUseMemoize) {
SyntaxTree::Childs::deleteRecursively(childs);
} else {
delete childs;
}
return m_PARSE_FAILED;
}
m_errors->clear();
return createSyntaxTree(startPos.parsePos, childs);
}
SyntaxTree* Parser::m_parse_op_postfix(Parser* my, Parser* exp, Parser* op, bool allowRepeat)
{
m_ErrorPos startPos = m_curErrPos();
SyntaxTree* ste = exp->parse();
if (ste->isFail()) {
m_fail(startPos);
return ste;
}
for (;;) {
m_ErrorPos midPos = m_curErrPos();
SyntaxTree* sto = op->parse();
if (sto->isError()) {
m_fail(startPos);
SyntaxTree::deleteRecursively(ste);
return sto;
}
if (sto->isFail()) {
m_back(midPos);
return ste;
}
SyntaxTree::Childs* childs = new SyntaxTree::Childs(2);
childs->setContentsMemID("pcpo");
exp->addSyntaxTreeToParent(childs, ste);
op->addSyntaxTreeToParent(childs, sto);
ste = my->createSyntaxTree(startPos.parsePos, childs);
if (! allowRepeat)
return ste;
}
}
ref_t ModuleScope :: resolveClosure(ref_t closureMessage, ref_t outputRef, ident_t ns)
{
ref_t signRef = 0;
module->resolveAction(getAction(closureMessage), signRef);
int paramCount = getParamCount(closureMessage);
IdentifierString closureName(module->resolveReference(closureTemplateReference));
if (signRef == 0) {
if (paramCount > 0) {
closureName.appendInt(paramCount);
}
if (isWeakReference(closureName)) {
return module->mapReference(closureName, true);
}
else return mapFullReference(closureName, true);
}
else {
ref_t signatures[ARG_COUNT];
size_t signLen = module->resolveSignature(signRef, signatures);
List<SNode> parameters;
SyntaxTree dummyTree;
SyntaxWriter dummyWriter(dummyTree);
dummyWriter.newNode(lxRoot);
for (size_t i = 0; i < signLen; i++) {
dummyWriter.appendNode(lxTarget, signatures[i]);
}
if (outputRef) {
dummyWriter.appendNode(lxTarget, outputRef);
}
// if the output signature is not provided - use the super class
else dummyWriter.appendNode(lxTarget, superReference);
dummyWriter.closeNode();
SNode paramNode = dummyTree.readRoot().firstChild();
while (paramNode != lxNone) {
parameters.add(paramNode);
paramNode = paramNode.nextNode();
}
closureName.append('#');
closureName.appendInt(paramCount + 1);
ref_t templateReference = 0;
if (isWeakReference(closureName)) {
templateReference = module->mapReference(closureName, true);
}
else templateReference = mapFullReference(closureName, true);
if (templateReference) {
return generateTemplate(templateReference, parameters, ns, false);
}
else return superReference;
}
}
/**
* @brief Parser::removePartial Destroys all of the nodes to the right of current
* @param S the tree
*/
void Parser::removePartial(SyntaxTree &S){
TNpair* c = S.getCurrent();
S.shiftToRoot();
bool f = false;
S = removePartial(S.getCurrent(),c,f);
S.shiftTo(c);
}
int main(int argc, char** argv) {
bool opTree = false;
string filename = "";
for (int i = 1; i < argc; i++) {
if (argv[i][0] == '-') {
switch (argv[i][1]) {
case 't': // Print the tree
opTree = true;
break;
}
} else
filename = string(argv[i]);
}
try {
SyntaxTree tree;
vector<char> bytecode;
ifstream ifs(filename.c_str());
VirtualMachine vm;
vm.compile(ifs, bytecode, tree);
if (opTree)
tree.dump(std::cout);
vm.run(&bytecode[0]);
} catch (std::exception &e) {
std::cerr << e.what() << "\n";
}
}
TEST_F(ExpressionTest, parseExpression3) {
static const std::string expression = "0 | (1 | 0 & 1) & 1";
static const std::string result = "|C&()|C&CCC";
auto operation = Expression().makeOperation(expression);
SyntaxTree visitor;
operation->accept(visitor);
EXPECT_STREQ(result.c_str(), visitor.result().c_str());
}
TEST_F(ExpressionTest, parseExpression4) {
static const std::string expression = "(0 & ((0 | 1) & 1)) & 0";
static const std::string result = "&()&C()&()|CCCC";
auto operation = Expression().makeOperation(expression);
SyntaxTree visitor;
operation->accept(visitor);
EXPECT_STREQ(result.c_str(), visitor.result().c_str());
}
SyntaxTree* Parser::m_parse_noTree(Parser* p)
{
SyntaxTree* st = p->parse();
if (st->isFail())
return st;
if (! m_bUseMemoize) {
SyntaxTree::deleteRecursively(st);
}
return m_NO_SYNTAX_TREE;
}
int main ()
{
SyntaxTree * treeS = new ( nothrow ) SyntaxTree ();
treeS->parse();
treeS->print();
cout << endl;
delete treeS;
return 0;
}
vector<DisambiguatedData> PredictedMorphologyTreeRecoverer::GetMorphology(
const SyntaxTree& realTree)
{
size_t size = static_cast<size_t>(realTree.GetSize());
vector<Token> tokens(size);
for (size_t nodeIndex = 0; nodeIndex < size; ++nodeIndex)
{
tokens[nodeIndex] = static_cast<Token>(realTree.GetNodes()[nodeIndex]);
}
return disambiguator->Disambiguate(tokens);
}
void Syntaxer::CleanupOnFailure() {
SyntaxTree tree;
for (int i = 0; i < (int)stack_->symbols.size(); ++i) {
if (stack_->symbols[i].SymbolType() == GrammarSymbolNonterminal) {
tree.SetRoot((SyntaxNode*)stack_->values[i]);
tree.Destroy();
}
}
stack_->clear();
}
SyntaxTree::Childs* Parser::m_parse_infix_common(Parser* exp, Parser* op, bool rep)
{
// Exp (Op Exp)* のパース
// rep が false なら Exp (Op Exp)? のパース
// 結果はSyntaxTreeの配列で [exp op exp op exp ....]
m_ErrorPos startPos = m_curErrPos();
SyntaxTree* st = exp->parse();
if (st->isError()) {
m_fail(startPos);
return m_INFIX_COMMON_FATAL_ERROR;
}
if (st->isFail()) {
m_fail(startPos);
return NULL;
}
SyntaxTree::Childs* childs = new SyntaxTree::Childs(3);
childs->setContentsMemID("pcif");
childs->add(st);
//exp->addSyntaxTreeToParent(childs, st);
for (;;) {
m_ErrorPos midPos = m_curErrPos();
SyntaxTree* sto = op->parse();
if (sto->isError()) {
m_fail(startPos);
if (! m_bUseMemoize) {
SyntaxTree::Childs::deleteRecursively(childs);
} else {
delete childs;
}
return m_INFIX_COMMON_FATAL_ERROR;
}
if (sto->isFail()) {
m_back(midPos);
break;
}
SyntaxTree* ste = exp->parse();
if (ste->isError()) {
m_fail(startPos);
if (! m_bUseMemoize) {
SyntaxTree::Childs::deleteRecursively(childs);
} else {
delete childs;
}
return m_INFIX_COMMON_FATAL_ERROR;
}
if (ste->isFail()) {
if (! m_bUseMemoize)
SyntaxTree::deleteRecursively(sto);
m_back(midPos);
break;
}
op->addSyntaxTreeToParent(childs, sto);
exp->addSyntaxTreeToParent(childs, ste);
if (!rep)
break;
}
return childs;
}
SyntaxTree* Parser::m_parse_andPredicate(Parser* p)
{
m_ErrorPos startPos = m_curErrPos();
SyntaxTree* st = p->parse();
if (st->isFail()) {
m_fail(startPos);
return m_PARSE_FAILED;
}
m_back(startPos);
SyntaxTree::deleteRecursively(st);
return m_NO_SYNTAX_TREE;
}
void ModuleScope :: generateTemplateProperty(SyntaxWriter& output, ref_t reference, List<SNode>& parameters)
{
SyntaxTree templateTree;
TemplateGenerator transformer(templateTree);
SyntaxWriter writer(templateTree);
writer.newNode(lxRoot);
transformer.generateTemplateProperty(writer, *this, reference, parameters);
writer.closeNode();
SyntaxTree::copyNode(output, templateTree.readRoot());
}
void ModuleScope :: importClassTemplate(SyntaxWriter& output, ref_t reference, List<SNode>& parameters)
{
SyntaxTree templateTree;
TemplateGenerator transformer(templateTree);
SyntaxWriter writer(templateTree);
writer.newNode(lxRoot);
transformer.generateTemplate(writer, *this, reference, parameters, false, true);
writer.closeNode();
transformer.importClass(output, templateTree.readRoot());
}
int main()
{
string formula;
while (getline(cin, formula))
{
SyntaxTree F;
F.BuildSyntaxTree(formula);
cout << fixed << setprecision(5);
cout << F.Count() << endl;
}
system("pause");
return 0;
}
SyntaxTree* UserParser::m_parse1(m_ErrorPos startPos)
{
const char* n = name();
SyntaxTree* st;
if (m_bUseMemoize) {
st = m_memo.getAt(startPos.parsePos);
if (st == m_PARSING) {
// left recursion
char pbuf[128];
gpInp->sprintSourceInfo(pbuf, 128, startPos.parsePos);
HMD_PRINTF("LEFT RECURSION DETECTED: %s %s\n", n, pbuf);
return m_FATAL_PARSER_ERROR;
} else if (st != m_NOT_PARSED_YET) {
if (st->isValidTree())
gpInp->setPos(st->str.endPos);
if (m_printIntermediateLevel > 1 ||
(m_printIntermediateLevel > 0 && !st->isFail())) {
char pbuf[128];
gpInp->sprintSourceInfo(pbuf, 128, startPos.parsePos);
HMD_PRINTF("%s %s -> memoized ", n, pbuf);
if (st->isValidTree()) {
char b[44];
gpInp->copySummary(b, 40, st->str);
HMD_PRINTF("pos=%d-%d '%s'\n", st->str.startPos, st->str.endPos, b);
} else {
if (st == m_NO_SYNTAX_TREE)
HMD_PRINTF("no syntax tree\n");
else if (st == m_PARSE_FAILED)
HMD_PRINTF("parse failed\n");
else if (st->isErrorCut())
HMD_PRINTF("ErrorCut(%d)\n",st->errorCutId);
else
HMD_PRINTF("(UNKNOWN BUG?)\n");
}
}
return st;
}
// not parsed yet
m_memo.setAt(startPos.parsePos, m_PARSING);
}
if (m_printIntermediateLevel > 1) {
char pbuf[128];
gpInp->sprintSourceInfo(pbuf, 128, startPos.parsePos);
HMD_PRINTF("try %s %s pos=%d:\n", n, pbuf, startPos.parsePos);
}
return NULL;
}
void
prtPatSet(PatSet *p, ostream &os, unsigned indent) {
if (p != NULL) {
PatSet::iterator pi;
for (pi = p->begin(); pi != p->end(); pi++) {
SyntaxTree *st = *pi;
if (st != NULL) {
os << endl;
st->print(os, 0, 1000000, 50, indent+2);
os << endl;
}
}
}
}
void LeftBinarize( SyntaxTree &tree, ParentNodes &parents )
{
for(ParentNodes::const_iterator p = parents.begin(); p != parents.end(); p++) {
const SplitPoints &point = *p;
if (point.size() > 3) {
const vector< SyntaxNode* >& topNodes
= tree.GetNodes( point[0], point[point.size()-1]-1);
string topLabel = topNodes[0]->GetLabel();
for(size_t i=2; i<point.size()-1; i++) {
// cerr << "LeftBin " << point[0] << "-" << (point[point.size()-1]-1) << ": " << point[0] << "-" << point[i]-1 << " ^" << topLabel << endl;
tree.AddNode( point[0], point[i]-1, "^" + topLabel );
}
}
}
}
/**
* @brief Parser::attachWords Attachs the words to the leaves of the tree
* @param S the tree
*/
void Parser::attachWords(SyntaxTree& S){
TNpair::TNvector L = S.getLeaves();
for(std::size_t i = 0; i < L.size(); ++i){
Word W = getNextWord(i);
removeAllOtherTypes(W,GPtoWT[L[i]->data()._d.first]);
L[i]->data()._d.second.setWord(W);
}
}
int main(int argc, char* argv[])
{
init( argc, argv ); // initialize from switches, set flags
// loop through all sentences
int i=0;
char inBuffer[LINE_MAX_LENGTH];
while(true) {
i++;
if (i%1000 == 0) cerr << "." << flush;
if (i%10000 == 0) cerr << ":" << flush;
if (i%100000 == 0) cerr << "!" << flush;
// get line from stdin
SAFE_GETLINE( cin, inBuffer, LINE_MAX_LENGTH, '\n', __FILE__);
if (cin.eof()) break;
// process into syntax tree representation
string inBufferString = string( inBuffer );
set< string > labelCollection; // set of labels, not used
map< string, int > topLabelCollection; // count of top labels, not used
SyntaxTree tree;
ProcessAndStripXMLTags( inBufferString, tree, labelCollection, topLabelCollection );
vector< string > inWords = tokenize( inBufferString.c_str() );
// output tree
// cerr << "BEFORE:" << endl << tree;
ParentNodes parents = tree.Parse();
// execute selected grammar relaxation schemes
if (leftBinarizeFlag)
LeftBinarize( tree, parents );
if (rightBinarizeFlag)
RightBinarize( tree, parents );
if (SAMTLevel>0)
SAMT( tree, parents );
// output tree
// cerr << "AFTER:" << endl << tree;
store( tree, inWords );
}
}
void Parser::m_reduceSyntaxTreeToParent(SyntaxTree::Childs* arr, SyntaxTree* st)
{
if (! st->isValidTree())
return;
for (TArrayIterator<SyntaxTree*> itr(st->childs); itr.hasMore(); ) {
SyntaxTree* t = itr.next();
if (t->isValidTree()) {
t->parser->addSyntaxTreeToParent(arr, t);
}
}
if (! m_bUseMemoize) {
// no more need
if (st->childs != NULL)
delete st->childs;
delete st;
}
}
SyntaxTree* Parser::m_parse_notPredicate(Parser* p)
{
m_ErrorPos startPos = m_curErrPos();
SyntaxTree* st = p->parse();
if (st->isFatalError()) {
m_fail(startPos);
return st;
}
m_back(startPos);
if (! st->isFail()) {
if (! m_bUseMemoize) {
SyntaxTree::deleteRecursively(st);
}
return m_PARSE_FAILED;
}
return m_NO_SYNTAX_TREE;
}
void Compiler::checkComparisonConsistency(const SyntaxTree& tree) const {
if (tree.left()->type == SyntaxTree::TYPE_NIL ||
tree.right()->type == SyntaxTree::TYPE_NIL)
error(tree.sourceLineNumber, "cannot compare disequality of a nil value.");
else if (tree.left()->type == SyntaxTree::TYPE_BOOLEAN ||
tree.right()->type == SyntaxTree::TYPE_BOOLEAN)
error(tree.sourceLineNumber, "cannot compare disequality of a boolean.");
else if ((tree.left()->type == SyntaxTree::TYPE_NUMBER ||
tree.left()->type == SyntaxTree::TYPE_STRING) && (
tree.right()->type == SyntaxTree::TYPE_NUMBER ||
tree.right()->type == SyntaxTree::TYPE_STRING) &&
tree.left()->type != tree.right()->type)
error(tree.sourceLineNumber, "disequality involves operands of different type.");
}
SyntaxTree* Parser::m_parse_op_prefix(Parser* my, Parser* exp, Parser* op, bool allowRepeat)
{
m_ErrorPos startPos = m_curErrPos();
SyntaxTree::Childs* firstChilds = new SyntaxTree::Childs(2);
firstChilds->setContentsMemID("pcpr");
SyntaxTree::Childs* lastChilds = firstChilds;
for (;;) {
m_ErrorPos midPos = m_curErrPos();
SyntaxTree* sto = op->parse();
if (sto->isError()) {
m_fail(startPos);
SyntaxTree::Childs::deleteRecursively(firstChilds);
return sto;
}
if (sto->isFail()) {
m_back(midPos);
break;
} else {
if (lastChilds->size() > 0) {
SyntaxTree::Childs* newChilds = new SyntaxTree::Childs(2);
newChilds->setContentsMemID("pcpr");
lastChilds->add(my->createSyntaxTree(m_curPos(), newChilds));
lastChilds = newChilds;
}
op->addSyntaxTreeToParent(lastChilds, sto);
if (! allowRepeat)
break;
}
}
SyntaxTree* ste = exp->parse();
if (ste->isFail()) {
m_fail(startPos);
SyntaxTree::Childs::deleteRecursively(firstChilds);
return ste;
}
if (lastChilds->size() == 0) {
// no op
SyntaxTree::Childs::deleteRecursively(firstChilds);
return ste;
}
lastChilds->add(ste);
return my->createSyntaxTree(startPos.parsePos, firstChilds);
}
void AlignedSentenceSyntax::Populate(bool isSyntax, int mixedSyntaxType, const Parameter ¶ms,
string line, Phrase &phrase, SyntaxTree &tree)
{
// parse source and target string
if (isSyntax) {
line = "<xml><tree label=\"X\">" + line + "</tree></xml>";
XMLParse(phrase, tree, line, params);
if (mixedSyntaxType != 0) {
// mixed syntax. Always add [X] where there isn't 1
tree.SetHieroLabel(params.hieroNonTerm);
if (mixedSyntaxType == 2) {
tree.AddToAll(params.hieroNonTerm);
}
}
} else {
PopulateWordVec(phrase, line);
tree.SetHieroLabel(params.hieroNonTerm);
}
}
void SyntaxTree :: saveNode(Node node, _Memory* dump, bool inclusingNode)
{
SyntaxTree tree;
SyntaxWriter writer(tree);
writer.newNode(lxRoot);
if (inclusingNode) {
if (node.strArgument >= 0) {
writer.newNode(node.type, node.identifier());
}
else writer.newNode(node.type, node.argument);
copyNode(writer, node);
writer.closeNode();
}
else copyNode(writer, node);
writer.closeNode();
tree.save(dump);
}
SyntaxTree* OperatorParser::parse(void)
{
HMD_ASSERT(m_parser);
m_ErrorPos startPos = m_curErrPos();
SyntaxTree* st = m_parser->parse();
const char* n = name();
if (st->isError()) {
m_fail(startPos);
if (! st->isFatalError()) {
char pbuf[128];
gpInp->sprintSourceInfo(pbuf, 128, startPos.parsePos);
HMD_PRINTF("%s %s -> ERROR %d\n", n, pbuf, st->errorCutId);
}
return m_FATAL_PARSER_ERROR;
}
if (st->isFail()) {
if (m_printIntermediateLevel > 2) {
char pbuf[128];
gpInp->sprintSourceInfo(pbuf, 128, startPos.parsePos);
HMD_PRINTF("%s %s -> fail\n", n, pbuf);
}
m_fail(startPos);
return m_PARSE_FAILED;
}
if (m_printIntermediateLevel > 1) {
char pbuf[128];
gpInp->sprintSourceInfo(pbuf, 128, startPos.parsePos);
if (st->isValidTree()) {
char b[44];
gpInp->copySummary(b, 40, st->str);
HMD_PRINTF("%s %s -> '%s'\n", n, pbuf, b);
} else {
HMD_PRINTF("%s %s -> (notree)\n", n, pbuf);
}
}
#if 0
if (st->isValidTree()) {
if (st->numChild() == 1) {
// no operator, so that this node is not needed
SyntaxTree* tmp = st->replace(0, NULL);
if (! m_bUseMemoize)
delete st;
return tmp;
}
}
#endif
return st;
}