// --------------------------------------------------------------------------- // XMLStringTokenizer: Management methods // --------------------------------------------------------------------------- XMLCh* XMLStringTokenizer::nextToken() { if (fOffset >= fStringLen) { return 0; } bool tokFound = false; XMLSize_t startIndex = fOffset; XMLSize_t endIndex = fOffset; for (; endIndex < fStringLen; endIndex++) { if (isDelimeter(fString[endIndex])) { if (tokFound) { break; } startIndex++; continue; } tokFound = true; } fOffset = endIndex; if (tokFound) { XMLCh* tokStr = (XMLCh*) fMemoryManager->allocate ( (endIndex - startIndex + 1) * sizeof(XMLCh) );//new XMLCh[(endIndex - startIndex) + 1]; XMLString::subString(tokStr, fString, startIndex, endIndex, fMemoryManager); fTokens->addElement(tokStr); return tokStr; } return 0; }
/** * Get next token in the current string expr. * Uses the ExpParser data expr, e, token, t, token_type and err */ void ExpParser::getToken() { token_type = NOTHING; char* t; // points to a character in token t = token; // let t point to the first character in token *t = '\0'; // set token empty //printf("\tgetToken e:{%c}, ascii=%i, col=%i\n", *e, *e, e-expr); // skip over whitespaces while (*e == ' ' || *e == '\t') // space or tab { e++; } // check for end of expression if (*e == '\0') { // token is still empty token_type = DELIMETER; return; } // check for minus if (*e == '-') { token_type = DELIMETER; *t = *e; e++; t++; *t = '\0'; // add a null character at the end of token return; } // check for parentheses if (*e == '(' || *e == ')') { token_type = DELIMETER; *t = *e; e++; t++; *t = '\0'; return; } // check for operators (delimeters) if (isDelimeter(*e)) { token_type = DELIMETER; while (isDelimeter(*e)) { *t = *e; e++; t++; } *t = '\0'; // add a null character at the end of token return; } // check for a value if (isDigitDot(*e)) { token_type = NUMBER; while (isDigitDot(*e)) { *t = *e; e++; t++; } // check for scientific notation like "2.3e-4" or "1.23e50" if (toupper(*e) == 'E') { *t = *e; e++; t++; if (*e == '+' || *e == '-') { *t = *e; e++; t++; } while (isDigit(*e)) { *t = *e; e++; t++; } } *t = '\0'; return; } // check for variables or functions if (isAlpha(*e)) { while (isAlpha(*e) || isDigit(*e)) //while (isNotDelimeter(*e)) { *t = *e; e++; t++; } *t = '\0'; // add a null character at the end of token // check if this is a variable or a function. // a function has a parentesis '(' open after the name char* e2 = NULL; e2 = e; // skip whitespaces while (*e2 == ' ' || *e2 == '\t') // space or tab { e2++; } if (*e2 == '(') { token_type = FUNCTION; } else { token_type = VARIABLE; } return; } // something unknown is found, wrong characters -> a syntax error token_type = UNKNOWN; while (*e != '\0') { *t = *e; e++; t++; } *t = '\0'; throw Error(row(), col(), 1, token); return; }