parser_state * PyParser_New(grammar *g, int start) { parser_state *ps; DBP("Attempting to create new PyParser\n"); if (!g->g_accel) PyGrammar_AddAccelerators(g); DBP("Added grammar accelerators\n"); ps = (parser_state *)PyMem_MALLOC(sizeof(parser_state)); DBP("Created parser state\n"); if (ps == NULL) return NULL; ps->p_grammar = g; #ifdef PY_PARSER_REQUIRES_FUTURE_KEYWORD ps->p_flags = 0; #endif ps->p_tree = PyNode_New(start); if (ps->p_tree == NULL) { PyMem_FREE(ps); return NULL; } s_reset(&ps->p_stack); (void) s_push(&ps->p_stack, PyGrammar_FindDFA(g, start), ps->p_tree); DBP("Created PyParser!!\n"); return ps; }
parser_state * PyParser_New(grammar *g, int start) { parser_state *ps; if (!g->g_accel) PyGrammar_AddAccelerators(g); ps = PyMem_NEW(parser_state, 1); if (ps == NULL) return NULL; ps->p_grammar = g; ps->p_generators = 0; ps->p_tree = PyNode_New(start); if (ps->p_tree == NULL) { PyMem_DEL(ps); return NULL; } s_reset(&ps->p_stack); (void) s_push(&ps->p_stack, PyGrammar_FindDFA(g, start), ps->p_tree); return ps; }
int PyParser_AddToken(parser_state *ps, int type, char *str, int lineno, int col_offset, int *expected_ret) { int ilabel; int err; D(printf("Token %s/'%s' ... ", _PyParser_TokenNames[type], str)); /* Find out which label this token is */ ilabel = classify(ps, type, str); if (ilabel < 0) return E_SYNTAX; /* Loop until the token is shifted or an error occurred */ for (;;) { /* Fetch the current dfa and state */ dfa *d = ps->p_stack.s_top->s_dfa; state *s = &d->d_state[ps->p_stack.s_top->s_state]; D(printf(" DFA '%s', state %d:", d->d_name, ps->p_stack.s_top->s_state)); /* Check accelerator */ if (s->s_lower <= ilabel && ilabel < s->s_upper) { int x = s->s_accel[ilabel - s->s_lower]; if (x != -1) { if (x & (1<<7)) { /* Push non-terminal */ int nt = (x >> 8) + NT_OFFSET; int arrow = x & ((1<<7)-1); dfa *d1 = PyGrammar_FindDFA( ps->p_grammar, nt); if ((err = push(&ps->p_stack, nt, d1, arrow, lineno, col_offset)) > 0) { D(printf(" MemError: push\n")); return err; } D(printf(" Push ...\n")); continue; } /* Shift the token */ if ((err = shift(&ps->p_stack, type, str, x, lineno, col_offset)) > 0) { D(printf(" MemError: shift.\n")); return err; } D(printf(" Shift.\n")); /* Pop while we are in an accept-only state */ while (s = &d->d_state [ps->p_stack.s_top->s_state], s->s_accept && s->s_narcs == 1) { D(printf(" DFA '%s', state %d: " "Direct pop.\n", d->d_name, ps->p_stack.s_top->s_state)); #ifdef PY_PARSER_REQUIRES_FUTURE_KEYWORD #if 0 if (d->d_name[0] == 'i' && strcmp(d->d_name, "import_stmt") == 0) future_hack(ps); #endif #endif s_pop(&ps->p_stack); if (s_empty(&ps->p_stack)) { D(printf(" ACCEPT.\n")); return E_DONE; } d = ps->p_stack.s_top->s_dfa; } return E_OK; } }
static void calcfirstset(grammar *g, dfa *d) { int i, j; state *s; arc *a; int nsyms; int *sym; int nbits; static bitset dummy; bitset result; int type; dfa *d1; label *l0; if (Py_DebugFlag) printf("Calculate FIRST set for '%s'\n", d->d_name); if (dummy == NULL) dummy = newbitset(1); if (d->d_first == dummy) { fprintf(stderr, "Left-recursion for '%s'\n", d->d_name); return; } if (d->d_first != NULL) { fprintf(stderr, "Re-calculating FIRST set for '%s' ???\n", d->d_name); } d->d_first = dummy; l0 = g->g_ll.ll_label; nbits = g->g_ll.ll_nlabels; result = newbitset(nbits); sym = (int *)PyObject_MALLOC(sizeof(int)); if (sym == NULL) Py_FatalError("no mem for new sym in calcfirstset"); nsyms = 1; sym[0] = findlabel(&g->g_ll, d->d_type, (char *)NULL); s = &d->d_state[d->d_initial]; for (i = 0; i < s->s_narcs; i++) { a = &s->s_arc[i]; for (j = 0; j < nsyms; j++) { if (sym[j] == a->a_lbl) break; } if (j >= nsyms) { /* New label */ sym = (int *)PyObject_REALLOC(sym, sizeof(int) * (nsyms + 1)); if (sym == NULL) Py_FatalError( "no mem to resize sym in calcfirstset"); sym[nsyms++] = a->a_lbl; type = l0[a->a_lbl].lb_type; if (ISNONTERMINAL(type)) { d1 = PyGrammar_FindDFA(g, type); if (d1->d_first == dummy) { fprintf(stderr, "Left-recursion below '%s'\n", d->d_name); } else { if (d1->d_first == NULL) calcfirstset(g, d1); mergebitset(result, d1->d_first, nbits); } } else if (ISTERMINAL(type)) { addbit(result, a->a_lbl); } } } d->d_first = result; if (Py_DebugFlag) { printf("FIRST set for '%s': {", d->d_name); for (i = 0; i < nbits; i++) { if (testbit(result, i)) printf(" %s", PyGrammar_LabelRepr(&l0[i])); } printf(" }\n"); } PyObject_FREE(sym); }
int PyParser_AddToken(register parser_state *ps, register int type, char *str, int lineno, int *expected_ret) { register int ilabel; int err; D(printf("Token %s/'%s' ... ", _PyParser_TokenNames[type], str)); /* Find out which label this token is */ ilabel = classify(ps, type, str); if (ilabel < 0) return E_SYNTAX; /* Loop until the token is shifted or an error occurred */ for (;;) { /* Fetch the current dfa and state */ register dfa *d = ps->p_stack.s_top->s_dfa; register state *s = &d->d_state[ps->p_stack.s_top->s_state]; D(printf(" DFA '%s', state %d:", d->d_name, ps->p_stack.s_top->s_state)); /* Check accelerator */ if (s->s_lower <= ilabel && ilabel < s->s_upper) { register int x = s->s_accel[ilabel - s->s_lower]; if (x != -1) { if (x & (1<<7)) { /* Push non-terminal */ int nt = (x >> 8) + NT_OFFSET; int arrow = x & ((1<<7)-1); dfa *d1 = PyGrammar_FindDFA( ps->p_grammar, nt); if ((err = push(&ps->p_stack, nt, d1, arrow, lineno)) > 0) { D(printf(" MemError: push\n")); return err; } D(printf(" Push ...\n")); continue; } /* Shift the token */ if ((err = shift(&ps->p_stack, type, str, x, lineno)) > 0) { D(printf(" MemError: shift.\n")); return err; } D(printf(" Shift.\n")); /* Pop while we are in an accept-only state */ while (s = &d->d_state [ps->p_stack.s_top->s_state], s->s_accept && s->s_narcs == 1) { D(printf(" DFA '%s', state %d: " "Direct pop.\n", d->d_name, ps->p_stack.s_top->s_state)); if (d->d_name[0] == 'i' && strcmp(d->d_name, "import_stmt") == 0) future_hack(ps); s_pop(&ps->p_stack); if (s_empty(&ps->p_stack)) { D(printf(" ACCEPT.\n")); return E_DONE; } d = ps->p_stack.s_top->s_dfa; } return E_OK; } } if (s->s_accept) { if (d->d_name[0] == 'i' && strcmp(d->d_name, "import_stmt") == 0) future_hack(ps); /* Pop this dfa and try again */ s_pop(&ps->p_stack); D(printf(" Pop ...\n")); if (s_empty(&ps->p_stack)) { D(printf(" Error: bottom of stack.\n")); return E_SYNTAX; } continue; } /* Stuck, report syntax error */ D(printf(" Error.\n")); if (expected_ret) { if (s->s_lower == s->s_upper - 1) { /* Only one possible expected token */ *expected_ret = ps->p_grammar-> g_ll.ll_label[s->s_lower].lb_type; } else *expected_ret = -1; } return E_SYNTAX; }