static struct rqlParse *rqlParseOr(struct tokenizer *tkz) /* Parse out and or or. */ { struct rqlParse *p = rqlParseAnd(tkz); struct rqlParse *parent = NULL; for (;;) { char *tok = tokenizerNext(tkz); if (tok == NULL || !sameString(tok, "or")) { tokenizerReuse(tkz); return p; } else { if (parent == NULL) { p = rqlParseCoerce(p, rqlTypeBoolean); AllocVar(parent); parent->op = rqlOpOr; parent->type = rqlTypeBoolean; parent->children = p; p = parent; } struct rqlParse *r = rqlParseCoerce(rqlParseAnd(tkz), rqlTypeBoolean); slAddTail(&parent->children, r); } } }
static struct rqlParse *rqlParseIndex(struct tokenizer *tkz) /* Handle the [] in this[6]. Convert it into tree: * rqlOpArrayIx * rqlParseAtom * rqlParseAtom */ { struct rqlParse *collection = rqlParseAtom(tkz); struct rqlParse *p = collection; char *tok = tokenizerNext(tkz); if (tok == NULL) tokenizerReuse(tkz); else if (tok[0] == '[') { // struct rqlParse *index = rqlParseExpression(tkz); struct rqlParse *index = rqlParseAtom(tkz); index = rqlParseCoerce(index, rqlTypeInt); skipOverRequired(tkz, "]"); AllocVar(p); p->op = rqlOpArrayIx; p->type = rqlTypeString; p->children = collection; collection->next = index; } else tokenizerReuse(tkz); return p; }
// todo implement substitution Object * parsePattern(Collector * c, Tokenizer * tt) { if (tokenizerNext(tt) != TOK_NUMBER) goto fail; double dcol = tt->c.number; if (tokenizerNext(tt) != TOK_COMMA) goto fail; if (tokenizerNext(tt) != TOK_NUMBER) goto fail; double drow = tt->c.number; if (tokenizerNext(tt) != TOK_RBRACKET) goto fail; if (!checkInteger(dcol)) goto fail; if (!checkInteger(drow)) goto fail; Object * pattern = (Object *) newPattern(c, (int) dcol, (int) drow); if (!pattern) goto fail; return pattern; fail: return NULL; }
Object * parseRange(Collector * c, Tokenizer * tt, Coord * ref1) { Coord * ref2 = NULL; TokenType t = tokenizerNext(tt); if (t != TOK_CELLID) goto fail; ref2 = parseReference(c, tt); if (!ref2) goto fail; return (Object *) newRange(c, ref1, ref2); fail: return NULL; }
static boolean eatMatchingTok(struct tokenizer *tkz, char *s) /* If next token matches s then eat it and return TRUE */ { char *tok = tokenizerNext(tkz); if (tok != NULL && sameString(tok, s)) return TRUE; else { tokenizerReuse(tkz); return FALSE; } }
Object * parseFormula(Collector * c, Tokenizer * tt) { TokenType t = tokenizerNext(tt); switch (t) { case TOK_NUMBER: return parseNumber(c, tt); case TOK_STRING: return parseString(c, tt); case TOK_EQUALS: return parseValue(c, tt); default: return (Object *) newStringN(c, tt->be - tt->bs, tt->bs); } }
static struct rqlParse *rqlParseAtom(struct tokenizer *tkz) /* Return low level (symbol or literal) */ { char *tok = tokenizerMustHaveNext(tkz); struct rqlParse *p; AllocVar(p); char c = tok[0]; if (c == '\'' || c == '"') { p->op = rqlOpLiteral; p->type = rqlTypeString; int len = strlen(tok+1); p->val.s = cloneStringZ(tok+1, len-1); } else if (isalpha(c) || c == '_') { p->op = rqlOpSymbol; p->type = rqlTypeString; /* String until promoted at least. */ p->val.s = cloneString(tok); } else if (isdigit(c)) { p->op = rqlOpLiteral; p->type = rqlTypeInt; p->val.i = sqlUnsigned(tok); if ((tok = tokenizerNext(tkz)) != NULL) { if (tok[0] == '.') { char buf[32]; tok = tokenizerMustHaveNext(tkz); safef(buf, sizeof(buf), "%d.%s", p->val.i, tok); p->type = rqlTypeDouble; p->val.x = sqlDouble(buf); } else tokenizerReuse(tkz); } } else if (c == '(') { p = rqlParseExpression(tkz); skipOverRequired(tkz, ")"); } else { errAbort("Unexpected %s line %d of %s", tok, tkz->lf->lineIx, tkz->lf->fileName); } return p; }
List * parseValueList(Collector * c, Tokenizer * tt) { List * args = newList(c); if (!args) goto fail; TokenType t = tokenizerNext(tt); if (t != TOK_LPAREN) goto fail; char * sp = tt->bp; t = tokenizerNext(tt); if (t == TOK_RPAREN) return args; tokenizerSet(tt, sp); do { Object * arg = parseValue(c, tt); if (!arg) goto fail; if (!listAppend(args, arg)) goto fail; t = tokenizerNext(tt); if (t == TOK_RPAREN) { return args; } } while (t == TOK_COMMA); fail: return NULL; }
Object * parseReferenceOrRange(Collector * c, Tokenizer * tt) { Coord * ref = NULL; ref = parseReference(c, tt); if (!ref) goto fail; char * sp = tt->bp; TokenType t = tokenizerNext(tt); if (t == TOK_COLON) { return parseRange(c, tt, ref); } else { tokenizerSet(tt, sp); } return (Object *) ref; fail: return NULL; }
Object * parseValue(Collector * c, Tokenizer * tt) { TokenType t = tokenizerNext(tt); switch (t) { case TOK_CELLID: return parseReferenceOrRange(c, tt); case TOK_NUMBER: return parseNumber(c, tt); case TOK_STRING: return parseString(c, tt); case TOK_ID: return parseCall(c, tt); case TOK_LBRACKET: return parsePattern(c, tt); default: return NULL; } }
static struct asObject *asParseTokens(struct tokenizer *tkz) /* Parse file into a list of objects. */ { struct asObject *objList = NULL; struct asObject *obj; while (tokenizerNext(tkz)) { obj = asParseTableDef(tkz); if (findObType(objList, obj->name)) tokenizerErrAbort(tkz, "Duplicate definition of %s", obj->name); slAddTail(&objList, obj); } for (obj = objList; obj != NULL; obj = obj->next) asLinkEmbeddedObjects(obj, objList); return objList; }
static struct rqlParse *rqlParseNot(struct tokenizer *tkz) /* parse out a logical not. */ { char *tok = tokenizerNext(tkz); if (sameString(tok, "not")) { struct rqlParse *p = rqlParseCoerce(rqlParseCmp(tkz), rqlTypeBoolean); struct rqlParse *n; AllocVar(n); n->op = rqlOpNot; n->type = rqlTypeBoolean; n->children = p; return n; } else { tokenizerReuse(tkz); return rqlParseCmp(tkz); } }
struct token *readAllTokens(char *fileName, struct lm *lm) /* Get list of all tokens from file. */ { struct tokenizer *tkz; struct token *tokList = NULL, *tok; char *s; tkz = tokenizerNew(fileName); tkz->uncommentC = tkz->uncommentShell = tkz->leaveQuotes = TRUE; while ((s = tokenizerNext(tkz)) != NULL) { lmAllocVar(lm, tok); tok->string = lmCloneString(lm, s); tok->fileName = fileName; tok->lineIx = tkz->lf->lineIx; slAddHead(&tokList, tok); } slReverse(&tokList); return tokList; }
static char *rqlParseFieldSpec(struct tokenizer *tkz, struct dyString *buf) /* Return a field spec, which may contain * and ?. Put results in buf, and * return buf->string. */ { boolean firstTime = TRUE; dyStringClear(buf); for (;;) { char *tok = tokenizerNext(tkz); if (tok == NULL) break; char c = *tok; if (c == '?' || c == '*' || isalpha(c) || c == '_' || c == '/' || c == '.') { if (firstTime) dyStringAppend(buf, tok); else { if (tkz->leadingSpaces == 0) dyStringAppend(buf, tok); else { tokenizerReuse(tkz); break; } } } else { tokenizerReuse(tkz); break; } firstTime = FALSE; } if (buf->stringSize == 0) errAbort("Expecting field name line %d of %s", tkz->lf->lineIx, tkz->lf->fileName); return buf->string; }
void tokenizerMustHaveNext(struct tokenizer *tkz) /* Get next token, which must be there. */ { if (tokenizerNext(tkz) == NULL) errAbort("Unexpected end of file"); }
struct rqlStatement *rqlStatementParse(struct lineFile *lf) /* Parse an RQL statement out of text */ { struct tokenizer *tkz = tokenizerOnLineFile(lf); tkz->uncommentShell = TRUE; tkz->uncommentC = TRUE; tkz->leaveQuotes = TRUE; struct rqlStatement *rql; AllocVar(rql); rql->command = cloneString(tokenizerMustHaveNext(tkz)); if (sameString(rql->command, "select")) { struct dyString *buf = dyStringNew(0); struct slName *list = NULL; char *tok = rqlParseFieldSpec(tkz, buf); /* Look for count(*) as special case. */ boolean countOnly = FALSE; if (sameString(tok, "count")) { char *paren = tokenizerNext(tkz); if (paren[0] == '(') { while ((paren = tokenizerMustHaveNext(tkz)) != NULL) { if (paren[0] == ')') break; } countOnly = TRUE; freez(&rql->command); rql->command = cloneString("count"); } else { tokenizerReuse(tkz); } } if (!countOnly) { list = slNameNew(tok); for (;;) { /* Parse out comma-separated field list. */ char *comma = tokenizerNext(tkz); if (comma == NULL || comma[0] != ',') { tokenizerReuse(tkz); break; } slNameAddHead(&list, rqlParseFieldSpec(tkz, buf)); } slReverse(&list); rql->fieldList = list; } dyStringFree(&buf); } else if (sameString(rql->command, "count")) { /* No parameters to count. */ } else errAbort("Unknown RQL command '%s line %d of %s\n", rql->command, lf->lineIx, lf->fileName); char *from = tokenizerNext(tkz); if (from != NULL) { if (sameString(from, "from")) { for (;;) { struct dyString *buf = dyStringNew(0); char *table = rqlParseFieldSpec(tkz, buf); slNameAddTail(&rql->tableList, table); char *comma = tokenizerNext(tkz); if (comma == NULL) break; if (comma[0] != ',') { tokenizerReuse(tkz); break; } dyStringFree(&buf); } } else { errAbort("missing 'from' clause in %s\n", rql->command); } } /* Parse where clause. */ char *where = tokenizerNext(tkz); if (where != NULL) { if (!sameString(where, "where")) { tokenizerReuse(tkz); } else { rql->whereClause = rqlParseExpression(tkz); rqlParseVarsUsed(rql->whereClause, &rql->whereVarList); } } /* Parse limit clause. */ char *limit = tokenizerNext(tkz); rql->limit = -1; if (limit != NULL) { if (!sameString(limit, "limit")) errAbort("Unknown clause '%s' line %d of %s", limit, lf->lineIx, lf->fileName); char *count = tokenizerMustHaveNext(tkz); if (!isdigit(count[0])) errAbort("Expecting number after limit, got %s line %d of %s", count, lf->lineIx, lf->fileName); rql->limit = atoi(count); } /* Check that are at end of statement. */ char *extra = tokenizerNext(tkz); if (extra != NULL) errAbort("Extra stuff starting with '%s' past end of statement line %d of %s", extra, lf->lineIx, lf->fileName); return rql; }
static struct rqlParse *rqlParseCmp(struct tokenizer *tkz) /* Parse out comparison. */ { struct rqlParse *l = rqlParseUnaryMinus(tkz); struct rqlParse *p = l; char *tok = tokenizerNext(tkz); boolean forceString = FALSE; boolean needNot = FALSE; if (tok != NULL) { enum rqlOp op = rqlOpUnknown; if (sameString(tok, "=")) { op = rqlOpEq; } else if (sameString(tok, "!")) { op = rqlOpNe; skipOverRequired(tkz, "="); } else if (sameString(tok, ">")) { if (eatMatchingTok(tkz, "=")) op = rqlOpGe; else op = rqlOpGt; } else if (sameString(tok, "<")) { if (eatMatchingTok(tkz, "=")) op = rqlOpGe; else op = rqlOpLe; } else if (sameString(tok, "not")) { forceString = TRUE; op = rqlOpLike; needNot = TRUE; skipOverRequired(tkz, "like"); } else if (sameString(tok, "like")) { forceString = TRUE; op = rqlOpLike; } else { tokenizerReuse(tkz); return p; } struct rqlParse *r = rqlParseUnaryMinus(tkz); AllocVar(p); p->op = op; p->type = rqlTypeBoolean; /* Now force children to be the same type, inserting casts if need be. */ if (forceString) { if (l->type != rqlTypeString || r->type != rqlTypeString) { errAbort("Expecting string type around comparison line %d of %s", tkz->lf->lineIx, tkz->lf->fileName); } } else { enum rqlType childType = commonTypeForBop(l->type, r->type); l = rqlParseCoerce(l, childType); r = rqlParseCoerce(r, childType); } /* Now hang children onto node. */ p->children = l; l->next = r; /* Put in a not around self if need be. */ if (needNot) { struct rqlParse *n; AllocVar(n); n->op = rqlOpNot; n->type = rqlTypeBoolean; n->children = p; p = n; } } return p; }