void matchNumeric() const { givenACodeSampleToTokenize nonNumeric("abc", true); ASSERT_EQUALS(false, Token::Match(nonNumeric.tokens(), "%num%")); givenACodeSampleToTokenize binary("101010b", true); ASSERT_EQUALS(true, Token::Match(binary.tokens(), "%num%")); givenACodeSampleToTokenize octal("0123", true); ASSERT_EQUALS(true, Token::Match(octal.tokens(), "%num%")); givenACodeSampleToTokenize decimal("4567", true); ASSERT_EQUALS(true, Token::Match(decimal.tokens(), "%num%")); givenACodeSampleToTokenize hexadecimal("0xDEADBEEF", true); ASSERT_EQUALS(true, Token::Match(hexadecimal.tokens(), "%num%")); givenACodeSampleToTokenize floatingPoint("0.0f", true); ASSERT_EQUALS(true, Token::Match(floatingPoint.tokens(), "%num%")); givenACodeSampleToTokenize doublePrecision("0.0d", true); ASSERT_EQUALS(true, Token::Match(doublePrecision.tokens(), "%num%")); givenACodeSampleToTokenize signedLong("0L", true); ASSERT_EQUALS(true, Token::Match(signedLong.tokens(), "%num%")); givenACodeSampleToTokenize negativeSignedLong("-0L", true); ASSERT_EQUALS(true, Token::Match(negativeSignedLong.tokens(), "- %num%")); givenACodeSampleToTokenize positiveSignedLong("+0L", true); ASSERT_EQUALS(true, Token::Match(positiveSignedLong.tokens(), "+ %num%")); givenACodeSampleToTokenize unsignedInt("0U", true); ASSERT_EQUALS(true, Token::Match(unsignedInt.tokens(), "%num%")); givenACodeSampleToTokenize unsignedLong("0UL", true); ASSERT_EQUALS(true, Token::Match(unsignedLong.tokens(), "%num%")); givenACodeSampleToTokenize unsignedLongLong("0ULL", true); ASSERT_EQUALS(true, Token::Match(unsignedLongLong.tokens(), "%num%")); givenACodeSampleToTokenize positive("+666", true); ASSERT_EQUALS(true, Token::Match(positive.tokens(), "+ %num%")); givenACodeSampleToTokenize negative("-42", true); ASSERT_EQUALS(true, Token::Match(negative.tokens(), "- %num%")); givenACodeSampleToTokenize negativeNull("-.0", true); ASSERT_EQUALS(true, Token::Match(negativeNull.tokens(), "- %num%")); givenACodeSampleToTokenize positiveNull("+.0", true); ASSERT_EQUALS(true, Token::Match(positiveNull.tokens(), "+ %num%")); }
void matchNumeric() { givenACodeSampleToTokenize nonNumeric("abc"); ASSERT_EQUALS(false, Token::Match(nonNumeric.tokens(), "%num%")); givenACodeSampleToTokenize binary("101010b"); ASSERT_EQUALS(true, Token::Match(binary.tokens(), "%num%")); givenACodeSampleToTokenize octal("0123"); ASSERT_EQUALS(true, Token::Match(octal.tokens(), "%num%")); givenACodeSampleToTokenize decimal("4567"); ASSERT_EQUALS(true, Token::Match(decimal.tokens(), "%num%")); givenACodeSampleToTokenize hexadecimal("0xDEADBEEF"); ASSERT_EQUALS(true, Token::Match(hexadecimal.tokens(), "%num%")); givenACodeSampleToTokenize floatingPoint("0.0f"); ASSERT_EQUALS(true, Token::Match(hexadecimal.tokens(), "%num%")); givenACodeSampleToTokenize doublePrecision("0.0d"); ASSERT_EQUALS(true, Token::Match(hexadecimal.tokens(), "%num%")); givenACodeSampleToTokenize unsignedInt("0U"); ASSERT_EQUALS(true, Token::Match(hexadecimal.tokens(), "%num%")); givenACodeSampleToTokenize unsignedLong("0UL"); ASSERT_EQUALS(true, Token::Match(hexadecimal.tokens(), "%num%")); givenACodeSampleToTokenize unsignedLongLong("0ULL"); ASSERT_EQUALS(true, Token::Match(hexadecimal.tokens(), "%num%")); givenACodeSampleToTokenize positive("+666"); ASSERT_EQUALS(true, Token::Match(positive.tokens(), "+ %num%")); givenACodeSampleToTokenize negative("-42"); ASSERT_EQUALS(true, Token::Match(negative.tokens(), "- %num%")); }
PrologToken getToken(const std::string& str, unsigned int& idx, const std::string& fileName, int& line, int& col) { top: const char normalOperatorChars[] = "+-*/\\^<>=`~:.?@#&"; if (idx == str.size()) { // End of string return endOfFile(fileName, line, col); } int oldLine = line, oldCol = col; #define STAR_P (str[idx]) #define P_SUB_1 (idx + 1 == str.size() ? 0 : str[idx + 1]) #define INC_INDEX do { \ if (STAR_P == '\n') { \ ++line; \ col = 1; \ } else { \ ++col; \ } \ ++idx; \ } while (false) if (islower(STAR_P)) { // Normal atoms unsigned int oldIdx = idx; do {INC_INDEX;} while (isalnum(STAR_P) || STAR_P == '_'); bool startOfTerm = false; if (STAR_P == '(') {startOfTerm = true; INC_INDEX;} return atom(str.substr(oldIdx, idx - oldIdx - startOfTerm), startOfTerm, fileName, oldLine, oldCol); } else if (STAR_P == '_' && !isalnum(P_SUB_1)) { // Wildcard INC_INDEX; return wildcard(fileName, oldLine, oldCol); } else if (isupper(STAR_P) || STAR_P == '_' || STAR_P == '$') { // Variables unsigned int oldIdx = idx; do {INC_INDEX;} while (isalnum(STAR_P) || STAR_P == '_' || STAR_P == '$'); return variable(str.substr(oldIdx, idx - oldIdx), fileName, oldLine, oldCol); } else if (isdigit(STAR_P) || STAR_P == '-' && isdigit(P_SUB_1)) { // Numbers int isNegative = 1; if (STAR_P == '-') {isNegative = -1; INC_INDEX;} unsigned int oldIdx = idx; do {INC_INDEX;} while (isdigit(STAR_P)); std::string tokenString; if (STAR_P == '.' && isdigit(P_SUB_1)) { // Floating point INC_INDEX; while (isdigit(STAR_P)) INC_INDEX; if (STAR_P == 'e' || STAR_P == 'E') { INC_INDEX; if (STAR_P == '-') INC_INDEX; while (isdigit(STAR_P)) INC_INDEX; } return floatingPoint(isNegative * stringToDouble(str.substr(oldIdx, idx - oldIdx), fileName, oldLine, oldCol), fileName, oldLine, oldCol); } else if (STAR_P == '\'') { // Base or character definition int base = stringToInt(str.substr(oldIdx, idx - oldIdx), 10); if (base == 0 && P_SUB_1) { // Character INC_INDEX; return integer(isNegative * (STAR_P), fileName, oldLine, oldCol); } else if (base >= 2 && base <= 36) { INC_INDEX; oldIdx = idx; while (isalnum(STAR_P)) INC_INDEX; return integer(isNegative * stringToInt(str.substr(oldIdx, idx - oldIdx), base), fileName, oldLine, oldCol); } else { std::cerr << "Invalid number base " << base << std::endl; abort(); } } else { // Normal integer return integer(isNegative * stringToInt(str.substr(oldIdx, idx - oldIdx), 10), fileName, oldLine, oldCol); } } else if (STAR_P == '.' && P_SUB_1 && !strchr(normalOperatorChars, P_SUB_1)) { INC_INDEX; return period(fileName, oldLine, oldCol); } else if (strchr(normalOperatorChars, STAR_P)) { // Operators unsigned int oldIdx = idx; do {INC_INDEX;} while (strchr(normalOperatorChars, STAR_P)); bool startOfTerm = false; if (STAR_P == '(') {startOfTerm = true; INC_INDEX;} return atom(str.substr(oldIdx, idx - oldIdx - startOfTerm), startOfTerm, fileName, oldLine, oldCol); } else if (STAR_P == '!' || STAR_P == ';') { // Single-character operators char c = STAR_P; INC_INDEX; bool startOfTerm = false; if (STAR_P == '(') {startOfTerm = true; INC_INDEX;} return atom(std::string(1, c), startOfTerm, fileName, oldLine, oldCol); } else if (STAR_P == '\'') { // Quoted atoms std::string result; INC_INDEX; unsigned int oldIdx = idx; get_more_chars_atom: while (STAR_P != '\'') INC_INDEX; result += str.substr(oldIdx, idx - oldIdx); if (P_SUB_1 == '\'') { // Escaped single quote result += "'"; INC_INDEX; INC_INDEX; goto get_more_chars_atom; } else { // End of atom INC_INDEX; bool startOfTerm = false; if (STAR_P == '(') {startOfTerm = true; INC_INDEX;} return atom(result, startOfTerm, fileName, oldLine, oldCol); } } else if (STAR_P == '"') { // Strings std::string result; INC_INDEX; unsigned int oldIdx = idx; get_more_chars_string: while (STAR_P && STAR_P != '"' && STAR_P != '\\') INC_INDEX; result += str.substr(oldIdx, idx - oldIdx); if (STAR_P == '"') { if (P_SUB_1 == '"') { // Escaped double quote result += "\""; INC_INDEX; INC_INDEX; oldIdx = idx; goto get_more_chars_string; } else { // End of string INC_INDEX; return stringToken(result, fileName, oldLine, oldCol); } } else if (STAR_P == '\\') { std::cerr << "Cannot handle backslash escapes in strings yet" << std::endl; abort(); } else if (!STAR_P) { std::cerr << "Unterminated string constant" << std::endl; abort(); } } else if (STAR_P == '(') { INC_INDEX; return leftParen(fileName, oldLine, oldCol); } else if (STAR_P == ')') { INC_INDEX; return rightParen(fileName, oldLine, oldCol); } else if (STAR_P == '[') { if (P_SUB_1 == ']') { INC_INDEX; INC_INDEX; bool startOfTerm = false; if (STAR_P == '(') {startOfTerm = true; INC_INDEX;} return atom("[]", startOfTerm, fileName, oldLine, oldCol); } else { INC_INDEX; return leftBracket(fileName, oldLine, oldCol); } } else if (STAR_P == ']') { INC_INDEX; return rightBracket(fileName, oldLine, oldCol); } else if (STAR_P == '|') { INC_INDEX; return verticalBar(fileName, oldLine, oldCol); } else if (STAR_P == ',') { INC_INDEX; return comma(fileName, oldLine, oldCol); } else if (isspace(STAR_P)) { // White space INC_INDEX; goto top; } else if (STAR_P == '%') { // Comment do {INC_INDEX;} while (STAR_P && STAR_P != '\n' && STAR_P != '\r'); if (!STAR_P) { std::cerr << "Unterminated comment" << std::endl; abort(); } goto top; } else { std::cerr << "Bad character '" << STAR_P << "'" << std::endl; abort(); } #undef STAR_P #undef P_SUB_1 #undef INC_INDEX std::cerr << "Should not get here" << std::endl; abort(); return endOfFile(fileName, line, col); // Should not get here }