//=========================================================================== // DoKeyword // The next token is expected to be a keyword. //=========================================================================== int DoKeyword(void) { ReadToken(); if(ISTOKEN("syntax")) { ReadToken(); if(ISTOKEN("simple")) { syntax = STX_SIMPLE; Message("Using simple syntax."); } else { Message("Unknown syntax '%s'.", token); return false; } } else if(ISTOKEN("group")) { ReadToken(); group = strtol(token, 0, 0); if(group < 1 || group > NUM_GROUPS) { Message("Illegal group number %i (1..%i allowed).", group, NUM_GROUPS); return false; } Message("Switching to group %i.", group); group--; // Make it an index number, though. } return true; }
/** * <JA> * @brief オプション文字列を分解して追加格納する. * * @param buf [in] 文字列 * @param argv [i/o] オプション列へのポインタ * @param argc [i/o] オプション列の数へのポインタ * @param maxnum [i/o] オプション列の割付最大数 * </JA> * <EN> * @brief Divide option string into option arguments and append to array. * * @param buf [in] option string * @param argv [i/o] pointer to option array * @param argc [i/o] pointer to the length of option array * @param maxnum [i/o] pointer to the allocated length of option array * </EN> */ static void add_to_arglist(char *buf, char ***argv_ret, int *argc_ret, int *maxnum_ret) { char *p = buf; char cpy[BUFLEN]; char *dst, *dst_from; char **argv = *argv_ret; int argc = *argc_ret; int maxnum = *maxnum_ret; dst = cpy; while (1) { while (*p != '\0' && ISTOKEN(*p)) p++; if (*p == '\0') break; dst_from = dst; while (*p != '\0' && (!ISTOKEN(*p))) { #if !defined(_WIN32) if (*p == '\\') { /* escape by '\' */ if (*(++p) == '\0') break; *(dst++) = *(p++); } else { #endif if (*p == '"') { /* quote by "" */ p++; while (*p != '\0' && *p != '"') *(dst++) = *(p++); if (*p == '\0') break; p++; } else if (*p == '\'') { /* quote by '' */ p++; while (*p != '\0' && *p != '\'') *(dst++) = *(p++); if (*p == '\0') break; p++; } else if (*p == '#') { /* comment out by '#' */ *p = '\0'; break; } else { /* other */ *(dst++) = *(p++); } #if !defined(_WIN32) } #endif } if (dst != dst_from) { *dst = '\0'; dst++; if ( argc >= maxnum) { maxnum += 20; argv = (char **)myrealloc(argv, sizeof(char *) * maxnum); } argv[argc++] = strcpy((char*)mymalloc(strlen(dst_from)+1), dst_from); } } *argv_ret = argv; *argc_ret = argc; *maxnum_ret = maxnum; }
action *get_shifts(int stateno) { register action *actions, *temp; register shifts *sp; register short *to_state; register int i, k; register int symbol; actions = 0; sp = shift_table[stateno]; if (sp) { to_state = sp->shift; for (i = sp->nshifts - 1; i >= 0; i--) { k = to_state[i]; symbol = accessing_symbol[k]; if (ISTOKEN(symbol)) { temp = NEW(action); temp->next = actions; temp->symbol = symbol; temp->number = k; temp->prec = symbol_prec[symbol]; temp->action_code = SHIFT; temp->assoc = symbol_assoc[symbol]; actions = temp; } } } return (actions); }
static action * get_shifts(int stateno) { action *actions, *temp; shifts *sp; Value_t *to_state2; Value_t i, k; Value_t symbol; actions = 0; sp = shift_table[stateno]; if (sp) { to_state2 = sp->shift; for (i = (Value_t) (sp->nshifts - 1); i >= 0; i--) { k = to_state2[i]; symbol = accessing_symbol[k]; if (ISTOKEN(symbol)) { temp = NEW(action); temp->next = actions; temp->symbol = symbol; temp->number = k; temp->prec = symbol_prec[symbol]; temp->action_code = SHIFT; temp->assoc = symbol_assoc[symbol]; actions = temp; } } } return (actions); }
/** * Generic function to extract tokens from strings, with quotation handling. * The usage is as the same as strtok. * * @param str [i/o] source string, or NULL when this is a continuous call from previous call. Will be truncated in this function. * @param delim [in] string to specify the delimiters. * @param left_paren [in] left brace * @param right_paren [in] right brace * @param mode [in] if 1, just move to the beginning of next token * * @return pointer to the next extracted token. */ char * mystrtok_quotation(char *str, char *delim, int left_paren, int right_paren, int mode) { static char *buf; /* target string buffer */ static char *pos; /* current pointer position */ char *p; char *from; int c; if (str != NULL) { pos = buf = str; } /* find start point */ p = pos; while (*p != '\0' && ISTOKEN(*p)) p++; if (*p == '\0') return NULL; /* no token left */ /* if mode == 1, exit here */ if (mode == 1) { pos = p; return p; } /* copy to ret_buf until end point is found */ c = *p; if (c == left_paren) { p++; if (*p == '\0') return NULL; from = p; while ((c = *p) != '\0' && ((c != right_paren) || (*(p+1) != '\0' && !ISTOKEN(*(p+1))))) p++; /* if quotation not terminated, allow the rest as one token */ /* if (*p == '\0') return NULL; */ } else { from = p; while ((c = *p) != '\0' && (!ISTOKEN(c))) p++; } if (*p != '\0') { *p = '\0'; p++; } pos = p; return from; }
//=========================================================================== // DoCompile // Returns true if the compilation was a success. //=========================================================================== int DoCompile(void) { while(!endOfSource) { ReadToken(); if(ISTOKEN("")) break; // Keywords. if(ISTOKEN("%")) { RET_FAIL( DoKeyword() ); } else { // It must be a texture definition. RET_FAIL( DoTexture() ); } } return true; }
static void set_goto_map(void) { shifts *sp; int i; int symbol; int k; Value_t *temp_base; Value_t *temp_map; Value_t state2; Value_t state1; goto_base = NEW2(nvars + 1, Value_t); temp_base = NEW2(nvars + 1, Value_t); goto_map = goto_base - ntokens; temp_map = temp_base - ntokens; ngotos = 0; for (sp = first_shift; sp; sp = sp->next) { for (i = sp->nshifts - 1; i >= 0; i--) { symbol = accessing_symbol[sp->shift[i]]; if (ISTOKEN(symbol)) break; if (ngotos == MAXYYINT) fatal("too many gotos"); ngotos++; goto_map[symbol]++; } } k = 0; for (i = ntokens; i < nsyms; i++) { temp_map[i] = (Value_t)k; k += goto_map[i]; } for (i = ntokens; i < nsyms; i++) goto_map[i] = temp_map[i]; goto_map[nsyms] = (Value_t)ngotos; temp_map[nsyms] = (Value_t)ngotos; from_state = NEW2(ngotos, Value_t); to_state = NEW2(ngotos, Value_t); for (sp = first_shift; sp; sp = sp->next) { state1 = sp->number; for (i = sp->nshifts - 1; i >= 0; i--) { state2 = sp->shift[i]; symbol = accessing_symbol[state2]; if (ISTOKEN(symbol)) break; k = temp_map[symbol]++; from_state[k] = state1; to_state[k] = state2; } } FREE(temp_base); }
void set_goto_map() { register shifts *sp; register int i; register int symbol; register int k; register short *temp_map; register int state2; register int state1; goto_map = NEW2(nvars + 1, short) - ntokens; temp_map = NEW2(nvars + 1, short) - ntokens; ngotos = 0; for (sp = first_shift; sp; sp = sp->next) { for (i = sp->nshifts - 1; i >= 0; i--) { symbol = accessing_symbol[sp->shift[i]]; if (ISTOKEN(symbol)) break; if (ngotos == MAXSHORT) fatal("too many gotos"); ngotos++; goto_map[symbol]++; } } k = 0; for (i = ntokens; i < nsyms; i++) { temp_map[i] = k; k += goto_map[i]; } for (i = ntokens; i < nsyms; i++) goto_map[i] = temp_map[i]; goto_map[nsyms] = ngotos; temp_map[nsyms] = ngotos; from_state = NEW2(ngotos, short); to_state = NEW2(ngotos, short); for (sp = first_shift; sp; sp = sp->next) { state1 = sp->number; for (i = sp->nshifts - 1; i >= 0; i--) { state2 = sp->shift[i]; symbol = accessing_symbol[state2]; if (ISTOKEN(symbol)) break; k = temp_map[symbol]++; from_state[k] = state1; to_state[k] = state2; } } FREE(temp_map + ntokens); }
static void inaccessable_symbols (void) { bitset Vp, Vs, Pp; /* Find out which productions are reachable and which symbols are used. Starting with an empty set of productions and a set of symbols which only has the start symbol in it, iterate over all productions until the set of productions remains unchanged for an iteration. For each production which has a LHS in the set of reachable symbols, add the production to the set of reachable productions, and add all of the nonterminals in the RHS of the production to the set of reachable symbols. Consider only the (partially) reduced grammar which has only nonterminals in N and productions in P. The result is the set P of productions in the reduced grammar, and the set V of symbols in the reduced grammar. Although this algorithm also computes the set of terminals which are reachable, no terminal will be deleted from the grammar. Some terminals might not be in the grammar but might be generated by semantic routines, and so the user might want them available with specified numbers. (Is this true?) However, the nonreachable terminals are printed (if running in verbose mode) so that the user can know. */ Vp = bitset_create (nsyms, BITSET_FIXED); Pp = bitset_create (nrules, BITSET_FIXED); /* If the start symbol isn't useful, then nothing will be useful. */ if (bitset_test (N, accept->number - ntokens)) { bitset_set (V, accept->number); while (1) { rule_number r; bitset_copy (Vp, V); for (r = 0; r < nrules; r++) { if (!bitset_test (Pp, r) && bitset_test (P, r) && bitset_test (V, rules[r].lhs->number)) { item_number *rhsp; for (rhsp = rules[r].rhs; *rhsp >= 0; rhsp++) if (ISTOKEN (*rhsp) || bitset_test (N, *rhsp - ntokens)) bitset_set (Vp, *rhsp); bitset_set (Pp, r); } } if (bitset_equal_p (V, Vp)) break; Vs = Vp; Vp = V; V = Vs; } } bitset_free (V); V = Vp; /* Tokens 0, 1, and 2 are internal to Bison. Consider them useful. */ bitset_set (V, endtoken->number); /* end-of-input token */ bitset_set (V, errtoken->number); /* error token */ bitset_set (V, undeftoken->number); /* some undefined token */ bitset_free (P); P = Pp; nuseful_productions = bitset_count (P); nuseless_productions = nrules - nuseful_productions; nuseful_nonterminals = 0; { symbol_number i; for (i = ntokens; i < nsyms; i++) if (bitset_test (V, i)) nuseful_nonterminals++; } nuseless_nonterminals = nvars - nuseful_nonterminals; /* A token that was used in %prec should not be warned about. */ { rule_number r; for (r = 0; r < nrules; ++r) if (rules[r].precsym != 0) bitset_set (V1, rules[r].precsym->number); } }
/** * Read and parse an HTK Config file, and set the specified option values. * * @param HTKconffile [in] HTK Config file path name * @param para [out] MFCC parameter to set * * @return TRUE on success, FALSE on failure. */ boolean htk_config_file_parse(char *HTKconffile, Value *para) { FILE *fp; char buf[512]; char *p, *d, *a; float srate; boolean skipped; jlog("Stat: para: parsing HTK Config file: %s\n", HTKconffile); /* convert the content into argument list c_argv[1..c_argc-1] */ /* c_argv[0] will be the original conffile name */ if ((fp = fopen(HTKconffile, "r")) == NULL) { jlog("Error: para: failed to open HTK Config file: %s\n", HTKconffile); return FALSE; } srate = 0.0; while (getl_fp(buf, 512, fp) != NULL) { p = buf; if (*p == 35) { /* skip comment line */ continue; } /* parse the input line to get directive and argument */ while (*p != '\0' && ISTOKEN(*p)) p++; if (*p == '\0') continue; d = p; while (*p != '\0' && (!ISTOKEN(*p)) && *p != '=') p++; if (*p == '\0') continue; *p = '\0'; p++; while (*p != '\0' && ((ISTOKEN(*p)) || *p == '=')) p++; if (*p == '\0') continue; a = p; while (*p != '\0' && (!ISTOKEN(*p))) p++; *p = '\0'; /* process arguments */ skipped = FALSE; if (strmatch(d, "SOURCERATE")) { /* -smpPeriod */ srate = atof(a); } else if (strmatch(d, "TARGETRATE")) { /* -fshift */ para->frameshift = atof(a); } else if (strmatch(d, "WINDOWSIZE")) { /* -fsize */ para->framesize = atof(a); } else if (strmatch(d, "ZMEANSOURCE")) { /* -zmeansource */ para->zmeanframe = (a[0] == 'T') ? TRUE : FALSE; } else if (strmatch(d, "USEPOWER")) { /* -usepower */ para->usepower = (a[0] == 'T') ? TRUE : FALSE; } else if (strmatch(d, "PREEMCOEF")) { /* -preemph */ para->preEmph = atof(a); } else if (strmatch(d, "USEHAMMING")) { /* (fixed to T) */ if (a[0] != 'T') { jlog("Error: para: USEHAMMING should be T\n", HTKconffile); return FALSE; } } else if (strmatch(d, "NUMCHANS")) { /* -fbank */ para->fbank_num = atoi(a); } else if (strmatch(d, "CEPLIFTER")) { /* -ceplif */ para->lifter = atoi(a); } else if (strmatch(d, "DELTAWINDOW")) { /* -delwin */ para->delWin = atoi(a); } else if (strmatch(d, "ACCWINDOW")) { /* -accwin */ para->accWin = atoi(a); } else if (strmatch(d, "LOFREQ")) { /* -lofreq */ para->lopass = atof(a); } else if (strmatch(d, "HIFREQ")) { /* -hifreq */ para->hipass = atof(a); } else if (strmatch(d, "RAWENERGY")) { /* -rawe */ para->raw_e = (a[0] == 'T') ? TRUE : FALSE; } else if (strmatch(d, "ENORMALISE")) { /* -enormal */ para->enormal = (a[0] == 'T') ? TRUE : FALSE; } else if (strmatch(d, "ESCALE")) { /* -escale */ para->escale = atof(a); } else if (strmatch(d, "SILFLOOR")) { /* -silfloor */ para->silFloor = atof(a); } else if (strmatch(d, "WARPFREQ")) { /* -vtln (1) */ para->vtln_alpha = atof(a); } else if (strmatch(d, "WARPLCUTOFF")) { /* -vtln (2) */ para->vtln_lower = atof(a); } else if (strmatch(d, "WARPUCUTOFF")) { /* -vtln (3) */ para->vtln_upper = atof(a); } else if (strmatch(d, "TARGETKIND")) { jlog("Warning: para: TARGETKIND skipped (will be determined by AM header)\n"); skipped = TRUE; } else if (strmatch(d, "NUMCEPS")) { jlog("Warning: para: NUMCEPS skipped (will be determined by AM header)\n"); skipped = TRUE; } else { jlog("Warning: para: \"%s\" ignored (not supported, or irrelevant)\n", d); skipped = TRUE; } if (!skipped) { jlog("Stat: para: %s=%s\n", d, a); } } if (srate == 0.0) { jlog("Warning: no SOURCERATE found\n"); jlog("Warning: assume source waveform sample rate to 625 (16kHz)\n"); srate = 625; } para->smp_period = srate; para->smp_freq = period2freq(para->smp_period); para->frameshift /= srate; para->framesize /= srate; if (fclose(fp) == -1) { jlog("Error: para: failed to close file\n"); return FALSE; } para->loaded = 1; return TRUE; }
void nullable_compute (void) { rule_number ruleno; symbol_number *s1; symbol_number *s2; rule_list *p; symbol_number *squeue = xnmalloc (nvars, sizeof *squeue); size_t *rcount = xcalloc (nrules, sizeof *rcount); /* RITEM contains all the rules, including useless productions. Hence we must allocate room for useless nonterminals too. */ rule_list **rsets = xcalloc (nvars, sizeof *rsets); /* This is said to be more elements than we actually use. Supposedly NRITEMS - NRULES is enough. But why take the risk? */ rule_list *relts = xnmalloc (nritems + nvars + 1, sizeof *relts); nullable = xcalloc (nvars, sizeof *nullable); s1 = s2 = squeue; p = relts; for (ruleno = 0; ruleno < nrules; ++ruleno) if (rules[ruleno].useful) { rule *rules_ruleno = &rules[ruleno]; if (rules_ruleno->rhs[0] >= 0) { /* This rule has a non empty RHS. */ item_number *rp = NULL; bool any_tokens = false; for (rp = rules_ruleno->rhs; *rp >= 0; ++rp) if (ISTOKEN (*rp)) any_tokens = true; /* This rule has only nonterminals: schedule it for the second pass. */ if (!any_tokens) for (rp = rules_ruleno->rhs; *rp >= 0; ++rp) { rcount[ruleno]++; p->next = rsets[*rp - ntokens]; p->value = rules_ruleno; rsets[*rp - ntokens] = p; p++; } } else { /* This rule has an empty RHS. */ aver (item_number_as_rule_number (rules_ruleno->rhs[0]) == ruleno); if (rules_ruleno->useful && ! nullable[rules_ruleno->lhs->number - ntokens]) { nullable[rules_ruleno->lhs->number - ntokens] = true; *s2++ = rules_ruleno->lhs->number; } } } while (s1 < s2) for (p = rsets[*s1++ - ntokens]; p; p = p->next) { rule *r = p->value; if (--rcount[r->number] == 0) if (r->useful && ! nullable[r->lhs->number - ntokens]) { nullable[r->lhs->number - ntokens] = true; *s2++ = r->lhs->number; } } free (squeue); free (rcount); free (rsets); free (relts); if (trace_flag & trace_sets) nullable_print (stderr); }
void set_nullable() { register short *r; register short *s1; register short *s2; register int ruleno; register int symbol; register shorts *p; short *squeue; short *rcount; shorts **rsets; shorts *relts; char any_tokens; short *r1; #ifdef TRACE fprintf(stderr, "Entering set_nullable"); #endif nullable = NEW2(nvars, char) - ntokens; squeue = NEW2(nvars, short); s1 = s2 = squeue; rcount = NEW2(nrules + 1, short); rsets = NEW2(nvars, shorts *) - ntokens; relts = NEW2(nitems + nvars + 1, shorts); p = relts; r = ritem; while (*r) { if (*r < 0) { symbol = rlhs[-(*r++)]; if (!nullable[symbol]) { nullable[symbol] = 1; *s2++ = symbol; } } else { r1 = r; any_tokens = 0; for (symbol = *r++; symbol > 0; symbol = *r++) { if (ISTOKEN(symbol)) any_tokens = 1; } if (!any_tokens) { ruleno = -symbol; r = r1; for (symbol = *r++; symbol > 0; symbol = *r++) { rcount[ruleno]++; p->next = rsets[symbol]; p->value = ruleno; rsets[symbol] = p; p++; } } } } while (s1 < s2) { p = rsets[*s1++]; while (p) { ruleno = p->value; p = p->next; if (--rcount[ruleno] == 0) { symbol = rlhs[ruleno]; if (!nullable[symbol]) { nullable[symbol] = 1; *s2++ = symbol; } } } } FREE(squeue); FREE(rcount); FREE(rsets + ntokens); FREE(relts); }
char * lua_format(const char * buf, int * len){ FormatState * fs = malloc(sizeof(FormatState)); FormatStateOut * fso = malloc(sizeof(FormatStateOut)); memset(fs, 0, sizeof(FormatState)); memset(fso, 0, sizeof(FormatStateOut)); fs->buf = buf; fs->curpos = -1; fs->len = *len; fs->varybuf = malloc(*len); memcpy(fs->varybuf, buf, *len); fso->outbuf = malloc(*len * 2); memset(fso->outbuf, 0, *len * 2); while (1){ const string * token = lex(fs); if (!token){ break; } if (token->len == 0){ continue; } if (token->type == Newline ){ if (token->s[0] == '\n') { writetoken(fso, "\n", 1); for (int i = 0; i < fs->level; ++i) { writetoken(fso, "\t", 1); } } } else if (token->type == Blank) { // } else if (token->type == Punct) { if (ISTOKEN("(") || ISTOKEN("{") || ISTOKEN("[")) { ++fs->level; writetoken(fso, token->s, token->len); } else if (ISTOKEN(")") || ISTOKEN("}") || ISTOKEN("]")) { --fs->level; if (isemptyline(fso)) { backtoken(fso, 1); } writetoken(fso, token->s, token->len); } else if (ISTOKEN(",")) { writetoken(fso, token->s, token->len); writetoken(fso, " ", 1); } else if (ISTOKEN(".") || ISTOKEN(":") || ISTOKEN("#") || ISTOKEN(",")) { writetoken(fso, token->s, token->len); } else { writetoken(fso, " ", 1); writetoken(fso, token->s, token->len); writetoken(fso, " ", 1); } } else { if (ISTOKEN("local") || ISTOKEN("if") || ISTOKEN("for") || ISTOKEN("while") || ISTOKEN("return")) { writetoken(fso, token->s, token->len); writetoken(fso, " ", 1); } else if (ISTOKEN("function")) { writetoken(fso, token->s, token->len); writetoken(fso, " ", 1); ++fs->level; } else if (ISTOKEN("then") || ISTOKEN("do") ) { writetoken(fso, " ", 1); writetoken(fso, token->s, token->len); writetoken(fso, " ", 1); ++fs->level; } else if (ISTOKEN("end")) { if (isemptyline(fso)) { backtoken(fso, 1); } else { writetoken(fso, " ", 1); } writetoken(fso, token->s, token->len); --fs->level; } else if (ISTOKEN("else")) { if (isemptyline(fso)) { backtoken(fso, 1); } else { writetoken(fso, " ", 1); } writetoken(fso, token->s, token->len); writetoken(fso, " ", 1); } else if (ISTOKEN("elseif")) { if (isemptyline(fso)) { backtoken(fso, 1); } else { writetoken(fso, " ", 1); } writetoken(fso, token->s, token->len); writetoken(fso, " ", 1); --fs->level; } else if (ISTOKEN("and") || ISTOKEN("or") || ISTOKEN("in")) { writetoken(fso, " ", 1); writetoken(fso, token->s, token->len); writetoken(fso, " ", 1); }else { writetoken(fso, token->s, token->len); } } } *len = fso->curpos; fso->outbuf[*len] = 0; return fso->outbuf; }
//=========================================================================== // DoTexture // Called after the name of the texture has been read (in 'token'). //=========================================================================== int DoTexture(void) { def_t *def; int i; mappatch_t *pat = NULL; // Check that it's a valid texture name (and convert to upper case). strupr(token); if(strlen(token) > 8) { Message("Too long texture name '%s'.", token); return false; } else if(strlen(token) <= 2) { Message("Warning: Short texture name '%s'.", token); } // Get the definition and let's start filling up those infos! def = GetTexture(token); while(!endOfSource) { ReadToken(); if(ISTOKEN(";")) break; // End of definition. // Flags. if(ISTOKEN("masked")) { // Masked flag (ever needed?). def->tex.flags |= 1; } else if(ISTOKEN("flags")) { // Custom flags. ReadToken(); def->tex.flags = strtol(token, 0, 0); } else if(ISTOKEN("misc")) { // Custom integer. ReadToken(); def->tex.reserved = strtol(token, 0, 0); } else if(pat && ISTOKEN("arg1")) { // Custom data for stepdir. ReadToken(); pat->reserved1 = (short) strtol(token, 0, 0); } else if(pat && ISTOKEN("arg2")) { // Custom data for 'colormap'. ReadToken(); pat->reserved2 = (short) strtol(token, 0, 0); } else if(isdigit(token[0]) || (pat && token[0] == '-')) { i = strtol(token, 0, 0); if(pat) pat->originX = i; else def->tex.width = i; ReadToken(); if(!ISTOKEN(",")) { Message("Expected a comma after %s.", pat? "patch origin X" : "texture width"); return false; } ReadToken(); i = strtol(token, 0, 0); if(pat) pat->originY = i; else def->tex.height = i; } else if(ISTOKEN("@")) { // A patch definition follows. // Allocate a new patch entry from the def. i = def->tex.patchCount++; if(i == MAX_PATCHES) { Message("Too many patches (maximum is %i).", MAX_PATCHES); return false; } pat = def->tex.patches + i; // Initialize. memset(pat, 0, sizeof(*pat)); pat->reserved1 = 1; // stepdir defaults to one. // The name of the patch comes first. ReadToken(); strupr(token); if(strlen(token) > 8) { Message("Too long patch name '%s'.", token); return false; } pat->patch = PatchNumber(token); } else { Message("Bad token '%s'.", token); return false; } } return true; }