main() { dataBlockT db; string line, cmd, token, var; int pos; printf("List structure test program\n"); printf("Type \"help\" for help\n"); db = NewDataBlock(); while (TRUE) { printf(">"); line = GetLine(); SetScannerString(db->scanner, line); cmd = ConvertToLowerCase(ReadToken(db->scanner)); if (IsVariableName(cmd)) { var = cmd; token = ReadToken(db->scanner); if (StringEqual(token, "")) { DisplayStringList(GetValue(db, var)); } else if (StringEqual(token, "=")) { ParseOperation(db, var); } else { Error("Unexpected token %s", token); } } else if (StringEqual(cmd, "help")) { HelpCmd(); } else if (StringEqual(cmd, "quit")) { break; } else { SaveToken(db->scanner, cmd); ParseOperation(db, NULL); } } }
const State* Transition(const Token& tok, FormatTokensVisitor& visitor) const override { switch (tok.Type) { case DELIMITER: return this; case OPERATION: return ParseOperation(tok, visitor); case CONSTANT: case MASK: return ParseValue(tok, visitor); default: return State::Error(); } }
int TRI_ParseQueryFulltextIndex (TRI_fulltext_query_t* const query, const char* const queryString, bool* isSubstringQuery) { char* ptr; size_t i; *isSubstringQuery = false; ptr = (char*) queryString; if (*ptr == '\0') { return TRI_ERROR_BAD_PARAMETER; } i = 0; while (*ptr) { char* start; char* end; char* split; TRI_fulltext_query_operation_e operation; TRI_fulltext_query_match_e match; char c; c = *ptr; // ignore whitespace if (c == ' ' || c == '\t' || c == '\r' || c == '\n' || c == '\f' || c == '\b' || c == ',') { ++ptr; continue; } // defaults operation = TRI_FULLTEXT_AND; match = TRI_FULLTEXT_COMPLETE; // word begin // get operation if (c == '+' || c == '-' || c == '|') { operation = ParseOperation(c); ++ptr; } split = NULL; start = ptr; while (*ptr) { c = *ptr; if (c == ' ' || c == '\t' || c == '\r' || c == '\n' || c == '\f' || c == '\b' || c == ',') { // end of word break; } else if (split == NULL && c == ':') { split = ptr + 1; } ++ptr; } end = ptr; if ((end - start == 0) || (split != NULL && split - start == 0) || (split != NULL && end - split == 0)) { // invalid string return TRI_ERROR_BAD_PARAMETER; } // get command if (split != NULL) { if (TRI_CaseEqualString2(start, "prefix:", strlen("prefix:"))) { match = TRI_FULLTEXT_PREFIX; } // 2013-01-17: deactivated substring matching as there is no implementation for it // else if (TRI_CaseEqualString2(start, "substring:", strlen("substring:"))) { // match = TRI_FULLTEXT_SUBSTRING; // } else if (TRI_CaseEqualString2(start, "complete:", strlen("complete:"))) { match = TRI_FULLTEXT_COMPLETE; } start = split; } assert(end >= start); if (! TRI_SetQueryFulltextIndex(query, (size_t) i, start, (size_t) (end - start), match, operation)) { // normalisation failed return TRI_ERROR_OUT_OF_MEMORY; } ++i; if (i >= TRI_FULLTEXT_SEARCH_MAX_WORDS) { break; } } if (i == 0) { // no words to search... return TRI_ERROR_BAD_PARAMETER; } return TRI_ERROR_NO_ERROR; }
/* * File ::= { Declaration } * Declaration ::= Node * | Operation * | OperationCase * | Option * | Enum * | Literal * | Header * | Output * | Common * | Include * Literal ::= { LiteralFlag } ( LITERAL_DEFNS | LITERAL_END ) * LiteralFlag ::= %both | %decls | %end * Header ::= %header STRING * Output ::= %output STRING * Common ::= %common * Include ::= %include [ %readonly ] STRING */ void TreeCCParse(TreeCCContext *context) { /* Fetch the first token from the input stream */ if(!TreeCCNextToken(context->input)) { return; } /* Parse lines of input until the input is exhausted */ do { switch(context->input->token) { case TREECC_TOKEN_EOF: { /* Shouldn't happen, but ignore it if it does */ } break; case TREECC_TOKEN_IDENTIFIER: { /* Parse an operation case definition */ ParseOperationCase(context); } continue; /* Skip the call to TreeCCNextToken */ case TREECC_TOKEN_LITERAL_DEFNS: case TREECC_TOKEN_LITERAL_END: case TREECC_TOKEN_BOTH: case TREECC_TOKEN_DECLS: case TREECC_TOKEN_END: { /* Parse a literal definition block for this file */ int flags = 0; while(context->input->token == TREECC_TOKEN_BOTH || context->input->token == TREECC_TOKEN_DECLS || context->input->token == TREECC_TOKEN_END) { if(context->input->token == TREECC_TOKEN_BOTH) { flags |= TREECC_LITERAL_CODE | TREECC_LITERAL_DECLS; } else if(context->input->token == TREECC_TOKEN_DECLS) { flags |= TREECC_LITERAL_DECLS; } else { flags |= TREECC_LITERAL_END; } TreeCCNextToken(context->input); } if((flags & TREECC_LITERAL_DECLS) == 0) { flags |= TREECC_LITERAL_CODE; } if(context->input->token == TREECC_TOKEN_LITERAL_DEFNS) { TreeCCAddLiteralDefn(context, TreeCCValue(context->input), flags); } else if(context->input->token == TREECC_TOKEN_LITERAL_END) { TreeCCAddLiteralDefn(context, TreeCCValue(context->input), flags | TREECC_LITERAL_END); } else { TreeCCError(context->input, "literal definition block expected"); continue; } } break; case TREECC_TOKEN_NODE: { /* Parse a node definition */ ParseNode(context); } continue; /* Skip the call to TreeCCNextToken */ case TREECC_TOKEN_OPERATION: { /* Parse an operation definition */ ParseOperation(context); } continue; /* Skip the call to TreeCCNextToken */ case TREECC_TOKEN_OPTION: { /* Parse an option declaration */ ParseOption(context); } continue; /* Skip the call to TreeCCNextToken */ case TREECC_TOKEN_HEADER: { /* Parse a header filename specification */ TreeCCNextToken(context->input); if(context->input->token == TREECC_TOKEN_STRING) { TreeCCStream *stream = TreeCCStreamCreate(context, context->input->text, context->input->text, 1); context->headerStream = stream; stream->readOnly |= context->input->readOnly; if(!(context->commonHeader)) { context->commonHeader = stream; } } else { TreeCCError(context->input, "header filename expected"); continue; } } break; case TREECC_TOKEN_OUTPUT: { /* Parse an output filename specification */ TreeCCNextToken(context->input); if(context->input->token == TREECC_TOKEN_STRING) { TreeCCStream *stream = TreeCCStreamCreate(context, context->input->text, context->input->text, 0); context->sourceStream = stream; stream->readOnly |= context->input->readOnly; if(!(context->commonSource)) { context->commonSource = stream; } } else { TreeCCError(context->input, "output filename expected"); continue; } } break; case TREECC_TOKEN_OUTDIR: { /* Parse an output directory specification */ TreeCCNextToken(context->input); if(context->input->token == TREECC_TOKEN_STRING) { context->outputDirectory = TreeCCResolvePathname(context->input->filename, context->input->text); } else { TreeCCError(context->input, "output filename expected"); continue; } } break; case TREECC_TOKEN_ENUM: { /* Parse an enumerated type definition */ ParseEnum(context); } continue; /* Skip the call to TreeCCNextToken */ case TREECC_TOKEN_COMMON: { /* Put the common housekeeping code in the current header and source streams */ context->commonHeader = context->headerStream; context->commonSource = context->sourceStream; } break; case TREECC_TOKEN_INCLUDE: { /* Include another file at this point */ int readOnly = context->input->readOnly; TreeCCNextToken(context->input); if(context->input->token == TREECC_TOKEN_READONLY) { readOnly = 1; TreeCCNextToken(context->input); } if(context->input->token == TREECC_TOKEN_STRING) { char *includeFile = TreeCCResolvePathname(context->input->filename, context->input->text); FILE *file = fopen(includeFile, "r"); if(file != NULL) { /* Parse the contents of the included file */ TreeCCInput *newInput = (TreeCCInput *)malloc(sizeof(TreeCCInput)); TreeCCInput *origInput = context->input; if(!newInput) { TreeCCOutOfMemory(context->input); } TreeCCOpen(newInput, context->input->progname, file, includeFile); context->input = newInput; TreeCCParse(context); context->input = origInput; TreeCCClose(newInput, 1); free(newInput); } else { TreeCCError(context->input, "cannot open \"%s\"", includeFile); free(includeFile); } } else { TreeCCError(context->input, "include filename expected"); } } break; case TREECC_TOKEN_LITERAL_CODE: case TREECC_TOKEN_LPAREN: case TREECC_TOKEN_RPAREN: case TREECC_TOKEN_LBRACE: case TREECC_TOKEN_RBRACE: case TREECC_TOKEN_LSQUARE: case TREECC_TOKEN_RSQUARE: case TREECC_TOKEN_COMMA: case TREECC_TOKEN_EQUALS: case TREECC_TOKEN_STAR: case TREECC_TOKEN_REF: case TREECC_TOKEN_SEMI: case TREECC_TOKEN_COLON_COLON: case TREECC_TOKEN_STRING: case TREECC_TOKEN_UNKNOWN: case TREECC_TOKEN_ABSTRACT: case TREECC_TOKEN_TYPEDEF: case TREECC_TOKEN_NOCREATE: case TREECC_TOKEN_VIRTUAL: case TREECC_TOKEN_INLINE: case TREECC_TOKEN_SPLIT: case TREECC_TOKEN_READONLY: { /* This token is not valid here */ TreeCCError(context->input, "declaration expected"); do { /* Attempt to re-synchronise on the next declaration */ TreeCCNextToken(context->input); } while(!IsStartDecl(context->input->token)); } continue; /* Skip the call to TreeCCNextToken */ } /* Get the next token from the input stream */ TreeCCNextToken(context->input); } while(context->input->token != TREECC_TOKEN_EOF); }