// Scanner Test
void
MimeSnifferTest::ScannerTest() {
#if TEST_R5
Outputf("(no tests actually performed for R5 version)\n");
#else // TEST_R5
// tests:
// Internal TokenStream and CharStream classes
// Define some useful macros for dynamically allocating
// various Token classes
#define T(type) (new Token(type, -1))
#define S(str) (new StringToken(str, -1))
#define I(val) (new IntToken(val, -1))
#define F(val) (new FloatToken(val, -1))
struct test_case {
const char *rule;
int tokenCount;
Token *tokens[256];
} testCases[] = {
{ "'Hey'[]:", 4,
{ S("Hey"),
T(LeftBracket),
T(RightBracket),
T(Colon)
}
},
{ "1", 1, { I(1) } },
{ "1.0", 1, { F(1.0) } },
{ "1.0 (\"ABCD\")", 4, { F(1.0), T(LeftParen), S("ABCD"), T(RightParen) } },
{ "1.0 ('ABCD')", 4, { F(1.0), T(LeftParen), S("ABCD"), T(RightParen) } },
{ " 1.0 ('ABCD') ", 4, { F(1.0), T(LeftParen), S("ABCD"), T(RightParen) } },
{ "0.8 [0:3] ('ABCDEFG' | 'abcdefghij')", 11,
{ F(0.8),
T(LeftBracket),
I(0),
T(Colon),
I(3),
T(RightBracket),
T(LeftParen),
S("ABCDEFG"),
T(Divider),
S("abcdefghij"),
T(RightParen)
}
},
{ "0.5([10]'ABCD'|[17]'abcd'|[13]'EFGH')", 17,
{ F(0.5),
T(LeftParen),
T(LeftBracket),
I(10),
T(RightBracket),
S("ABCD"),
T(Divider),
T(LeftBracket),
I(17),
T(RightBracket),
S("abcd"),
T(Divider),
T(LeftBracket),
I(13),
T(RightBracket),
S("EFGH"),
T(RightParen)
}
},
{ "0.5 \n [0:3] \t ('ABCD' \n | 'abcd' | 'EFGH')", 13,
{ F(0.5),
T(LeftBracket),
I(0),
T(Colon),
I(3),
T(RightBracket),
T(LeftParen),
S("ABCD"),
T(Divider),
S("abcd"),
T(Divider),
S("EFGH"),
T(RightParen)
}
},
{ "0.8 [ 0 : 3 ] ('ABCDEFG' | 'abcdefghij')", 11,
{ F(0.8),
T(LeftBracket),
I(0),
T(Colon),
I(3),
T(RightBracket),
T(LeftParen),
S("ABCDEFG"),
T(Divider),
S("abcdefghij"),
T(RightParen)
}
},
{ "0.8 [0:3] ('ABCDEFG' & 'abcdefg')", 11,
{ F(0.8),
T(LeftBracket),
I(0),
T(Colon),
I(3),
T(RightBracket),
T(LeftParen),
S("ABCDEFG"),
T(Ampersand),
S("abcdefg"),
T(RightParen)
}
},
{ "1.0 ('ABCD') | ('EFGH')", 8,
{ F(1.0),
T(LeftParen),
S("ABCD"),
T(RightParen),
T(Divider),
T(LeftParen),
S("EFGH"),
T(RightParen)
}
},
{ "1.0 [0:3] ('ABCD') | [2:4] ('EFGH')", 18,
{ F(1.0),
T(LeftBracket),
I(0),
T(Colon),
I(3),
T(RightBracket),
T(LeftParen),
S("ABCD"),
T(RightParen),
T(Divider),
T(LeftBracket),
I(2),
T(Colon),
I(4),
T(RightBracket),
T(LeftParen),
S("EFGH"),
T(RightParen)
}
},
{ "0.8 [0:4] (\\077Mkj0x34 & 'abcdefgh')", 11,
{ F(0.8),
T(LeftBracket),
I(0),
T(Colon),
I(4),
T(RightBracket),
T(LeftParen),
S("\077Mkj0x34"),
T(Ampersand),
S("abcdefgh"),
T(RightParen)
}
},
{ "0.8 [0:4] (\\077Mkj\\x34 & 'abcdefgh')", 11,
{ F(0.8),
T(LeftBracket),
I(0),
T(Colon),
I(4),
T(RightBracket),
T(LeftParen),
S("\077Mkj\x34"),
T(Ampersand),
S("abcdefgh"),
T(RightParen)
}
},
{ "0.8 [0:3] (\\077034 & 'abcd')", 11,
{ F(0.8),
T(LeftBracket),
I(0),
T(Colon),
I(3),
T(RightBracket),
T(LeftParen),
S("\077034"),
T(Ampersand),
S("abcd"),
T(RightParen)
}
},
{ "0.8 [0:3] (\\077\\034 & 'ab')", 11,
{ F(0.8),
T(LeftBracket),
I(0),
T(Colon),
I(3),
T(RightBracket),
T(LeftParen),
S("\077\034"),
T(Ampersand),
S("ab"),
T(RightParen)
}
},
{ "0.8 [0:3] (\\77\\034 & 'ab')", 11,
{ F(0.8),
T(LeftBracket),
I(0),
T(Colon),
I(3),
T(RightBracket),
T(LeftParen),
S("\077\034"),
T(Ampersand),
S("ab"),
T(RightParen)
}
},
{ "0.8 [0:3] (\\7 & 'a')", 11,
{ F(0.8),
T(LeftBracket),
I(0),
T(Colon),
I(3),
T(RightBracket),
T(LeftParen),
S("\007"),
T(Ampersand),
S("a"),
T(RightParen)
}
},
{ "0.8 [0:3] (\"\\17\" & 'a')", 11,
{ F(0.8),
T(LeftBracket),
I(0),
T(Colon),
I(3),
T(RightBracket),
T(LeftParen),
S("\017"),
T(Ampersand),
S("a"),
T(RightParen)
}
},
{ "0.8 [0:3] ('\\17' & 'a')", 11,
{ F(0.8),
T(LeftBracket),
I(0),
T(Colon),
I(3),
T(RightBracket),
T(LeftParen),
S("\017"),
T(Ampersand),
S("a"),
T(RightParen)
}
},
{ "0.8 [0:3] (\\g & 'a')", 11,
{ F(0.8),
T(LeftBracket),
I(0),
T(Colon),
I(3),
T(RightBracket),
T(LeftParen),
S("g"),
T(Ampersand),
S("a"),
T(RightParen)
}
},
{ "0.8 [0:3] (\\g&\\b)", 11,
{ F(0.8),
T(LeftBracket),
I(0),
T(Colon),
I(3),
T(RightBracket),
T(LeftParen),
S("g"),
T(Ampersand),
S("\b"),
T(RightParen)
}
},
{ "0.8 [0:3] (\\g\\&b & 'abc')", 11,
{ F(0.8),
T(LeftBracket),
I(0),
T(Colon),
I(3),
T(RightBracket),
T(LeftParen),
S("g&b"),
T(Ampersand),
S("abc"),
T(RightParen)
}
},
{ "0.8 [0:3] (0x3457 & 'ab')", 11,
{ F(0.8),
T(LeftBracket),
I(0),
T(Colon),
I(3),
T(RightBracket),
T(LeftParen),
S("\x34\x57"),
T(Ampersand),
S("ab"),
T(RightParen)
}
},
{ "0.8 [0:3] (\\x34\\x57 & 'ab')", 11,
{ F(0.8),
T(LeftBracket),
I(0),
T(Colon),
I(3),
T(RightBracket),
T(LeftParen),
S("\x34\x57"),
T(Ampersand),
S("ab"),
T(RightParen)
}
},
{ "0.8 [0:3] (0xA4b7 & 'ab')", 11,
{ F(0.8),
T(LeftBracket),
I(0),
T(Colon),
I(3),
T(RightBracket),
T(LeftParen),
S("\xA4\xb7"),
T(Ampersand),
S("ab"),
T(RightParen)
}
},
{ "0.8 [0:3] (\\xA4\\xb7 & 'ab')", 11,
{ F(0.8),
T(LeftBracket),
I(0),
T(Colon),
I(3),
T(RightBracket),
T(LeftParen),
S("\xA4\xb7"),
T(Ampersand),
S("ab"),
T(RightParen)
}
},
{ "0.8 [0:3] (\"\\xA4\\xb7\" & 'ab')", 11,
{ F(0.8),
T(LeftBracket),
I(0),
T(Colon),
I(3),
T(RightBracket),
T(LeftParen),
S("\xA4\xb7"),
T(Ampersand),
S("ab"),
T(RightParen)
}
},
{ "0.8 [0:3] (\'\\xA4\\xb7\' & 'ab')", 11,
{ F(0.8),
T(LeftBracket),
I(0),
T(Colon),
I(3),
T(RightBracket),
T(LeftParen),
S("\xA4\xb7"),
T(Ampersand),
S("ab"),
T(RightParen)
}
},
{ "0.8 [0:3] ('ab\"' & 'abc')", 11,
{ F(0.8),
T(LeftBracket),
I(0),
T(Colon),
I(3),
T(RightBracket),
T(LeftParen),
S("ab\""),
T(Ampersand),
S("abc"),
T(RightParen)
}
},
{ "0.8 [0:3] (\"ab\\\"\" & 'abc')", 11,
{ F(0.8),
T(LeftBracket),
I(0),
T(Colon),
I(3),
T(RightBracket),
T(LeftParen),
S("ab\""),
T(Ampersand),
S("abc"),
T(RightParen)
}
},
{ "0.8 [0:3] (\"ab\\A\" & 'abc')", 11,
{ F(0.8),
T(LeftBracket),
I(0),
T(Colon),
I(3),
T(RightBracket),
T(LeftParen),
S("abA"),
T(Ampersand),
S("abc"),
T(RightParen)
}
},
{ "0.8 [0:3] (\"ab'\" & 'abc')", 11,
{ F(0.8),
T(LeftBracket),
I(0),
T(Colon),
I(3),
T(RightBracket),
T(LeftParen),
S("ab'"),
T(Ampersand),
S("abc"),
T(RightParen)
}
},
{ "0.8 [0:3] (\"ab\\\\\" & 'abc')", 11,
{ F(0.8),
T(LeftBracket),
I(0),
T(Colon),
I(3),
T(RightBracket),
T(LeftParen),
S("ab\\"),
T(Ampersand),
S("abc"),
T(RightParen)
}
},
{ "0.8 [-5:-3] (\"abc\" & 'abc')", 11,
{ F(0.8),
T(LeftBracket),
I(-5),
T(Colon),
I(-3),
T(RightBracket),
T(LeftParen),
S("abc"),
T(Ampersand),
S("abc"),
T(RightParen)
}
},
{ "0.8 [5:3] (\"abc\" & 'abc')", 11,
{ F(0.8),
T(LeftBracket),
I(5),
T(Colon),
I(3),
T(RightBracket),
T(LeftParen),
S("abc"),
T(Ampersand),
S("abc"),
T(RightParen)
}
},
{ "1.2 ('ABCD')", 4,
{ F(1.2),
T(LeftParen),
S("ABCD"),
T(RightParen)
}
},
{ ".2 ('ABCD')", 4,
{ F(0.2),
T(LeftParen),
S("ABCD"),
T(RightParen)
}
},
{ "0. ('ABCD')", 4,
{ F(0.0),
T(LeftParen),
S("ABCD"),
T(RightParen)
}
},
// Signed integers
{ "-1 ('ABCD')", 4,
{ I(-1),
T(LeftParen),
S("ABCD"),
T(RightParen)
}
},
{ "+1 ('ABCD')", 4,
{ I(1),
T(LeftParen),
S("ABCD"),
T(RightParen)
}
},
// Unsigned extended floats
{ "1E25 ('ABCD')", 4,
{ F(1e25),
T(LeftParen),
S("ABCD"),
T(RightParen)
}
},
{ "1e25 ('ABCD')", 4,
{ F(1e25),
T(LeftParen),
S("ABCD"),
T(RightParen)
}
},
{ "1E+25 ('ABCD')", 4,
{ F(1e25),
T(LeftParen),
S("ABCD"),
T(RightParen)
}
},
{ "1e+25 ('ABCD')", 4,
{ F(1e25),
T(LeftParen),
S("ABCD"),
T(RightParen)
}
},
{ "1E-25 ('ABCD')", 4,
{ F(1e-25),
T(LeftParen),
S("ABCD"),
T(RightParen)
}
},
{ "1e-25 ('ABCD')", 4,
{ F(1e-25),
T(LeftParen),
S("ABCD"),
T(RightParen)
}
},
// Positive signed extended floats
{ "+1E25 ('ABCD')", 4,
{ F(1e25),
T(LeftParen),
S("ABCD"),
T(RightParen)
}
},
{ "+1e25 ('ABCD')", 4,
{ F(1e25),
T(LeftParen),
S("ABCD"),
T(RightParen)
}
},
{ "+1E+25 ('ABCD')", 4,
{ F(1e25),
T(LeftParen),
S("ABCD"),
T(RightParen)
}
},
{ "+1e+25 ('ABCD')", 4,
{ F(1e25),
T(LeftParen),
S("ABCD"),
T(RightParen)
}
},
{ "+1E-25 ('ABCD')", 4,
{ F(1e-25),
T(LeftParen),
S("ABCD"),
T(RightParen)
}
},
{ "+1e-25 ('ABCD')", 4,
{ F(1e-25),
T(LeftParen),
S("ABCD"),
T(RightParen)
}
},
// Negative signed extended floats
{ "-1E25 ('ABCD')", 4,
{ F(-1e25),
T(LeftParen),
S("ABCD"),
T(RightParen)
}
},
{ "-1e25 ('ABCD')", 4,
{ F(-1e25),
T(LeftParen),
S("ABCD"),
T(RightParen)
}
},
{ "-1E+25 ('ABCD')", 4,
{ F(-1e25),
T(LeftParen),
S("ABCD"),
T(RightParen)
}
},
{ "-1e+25 ('ABCD')", 4,
{ F(-1e25),
T(LeftParen),
S("ABCD"),
T(RightParen)
}
},
{ "-1E-25 ('ABCD')", 4,
{ F(-1e-25),
T(LeftParen),
S("ABCD"),
T(RightParen)
}
},
{ "-1e-25 ('ABCD')", 4,
{ F(-1e-25),
T(LeftParen),
S("ABCD"),
T(RightParen)
}
},
// Miscellaneous extended floats
{ ".1E-25 ('ABCD')", 4,
{ F(0.1e-25),
T(LeftParen),
S("ABCD"),
T(RightParen)
}
},
{ "-.1e-25 ('ABCD')", 4,
{ F(-0.1e-25),
T(LeftParen),
S("ABCD"),
T(RightParen)
}
},
// Signed floats
{ "-1.0 ('ABCD')", 4,
{ F(-1.0),
T(LeftParen),
S("ABCD"),
T(RightParen)
}
},
{ "+1.0 ('ABCD')", 4,
{ F(1.0),
T(LeftParen),
S("ABCD"),
T(RightParen)
}
},
// The uber test
{ "0 -0 +0 1 -2 +3 0. -0. +0. 1. -2. +3. 0.0 -0.1 +0.2 1.0 -2.1 +3.2 "
"0.e0 0.e-1 0.e+2 1.e1 2.e-2 3.e+3 -1.e1 -2.e-2 -3.e+3 +1.e1 +2.e-2 +3.e+3 "
"0.012345 1.23456 ( ) [ ] | & : -i "
" \"abcxyzABCXYZ_ ( ) [ ] | & : -i \t\n \\\" ' \\012\\0\\377\\x00\\x12\\xab\\xCD\\xeF\\x1A\\xb2 \" "
" 'abcxyzABCXYZ_ ( ) [ ] | & : -i \t\n \" \\' \\012\\0\\377\\x00\\x12\\xab\\xCD\\xeF\\x1A\\xb2 ' "
" \\000abc_xyz123\"'\"'456 \\xA1a1 \\!\\?\\\\ "
" 0x00 0x12 0xabCD 0xaBcD 0x0123456789aBcDeFfEdCbA", 50,
{ I(0), I(0), I(0), I(1), I(-2), I(3), F(0.0), F(0.0), F(0.0),
F(1.0), F(-2.0), F(3.0), F(0.0), F(-0.1), F(0.2), F(1.0), F(-2.1), F(3.2),
F(0.0), F(0.0e-1), F(0.0e2), F(1.0e1), F(2.0e-2), F(3.0e3),
F(-1.0e1), F(-2.0e-2), F(-3.0e3), F(1.0e1), F(2.0e-2), F(3.0e3),
F(0.012345), F(1.23456), T(LeftParen), T(RightParen), T(LeftBracket),
T(RightBracket), T(Divider), T(Ampersand), T(Colon), T(CaseInsensitiveFlag),
S(std::string("abcxyzABCXYZ_ ( ) [ ] | & : -i \t\n \" ' \012\0\377\x00\x12\xab\xCD\xeF\x1A\xb2 ", 49)),
S(std::string("abcxyzABCXYZ_ ( ) [ ] | & : -i \t\n \" ' \012\0\377\x00\x12\xab\xCD\xeF\x1A\xb2 ", 49)),
S(std::string("\000abc_xyz123\"'\"'456", 18)),
S("\241a1"),
S("!?\\"),
S(std::string("\x00", 1)), S("\x12"), S("\xAB\xCD"), S("\xAB\xCD"),
S("\x01\x23\x45\x67\x89\xAB\xCD\xEF\xFE\xDC\xBA")
}
},
};
// Undefine our nasty macros
#undef T
#undef S
#undef I
#undef F
const int testCaseCount = sizeof(testCases) / sizeof(test_case);
for (int i = 0; i < testCaseCount; i++) {
NextSubTest();
// cout << endl << testCases[i].rule << endl;
TokenStream stream;
try {
stream.SetTo(testCases[i].rule);
CHK(stream.InitCheck() == B_OK);
for (int j = 0; j < testCases[i].tokenCount; j++) {
const Token *token = stream.Get();
CHK(token);
/*
cout << tokenTypeToString(token->Type()) << endl;
if (token->Type() == CharacterString)
cout << " token1 == " << token->String() << endl;
if (testCases[i].tokens[j]->Type() == CharacterString)
cout << " token2 == " << (testCases[i].tokens[j])->String() << endl;
if (token->Type() == CharacterString)
{
const std::string &str = token->String();
printf("parser: ");
for (int i = 0; i < str.length(); i++)
printf("%x ", str[i]);
printf("\n");
}
if (testCases[i].tokens[j]->Type() == CharacterString)
{
const std::string &str = (testCases[i].tokens[j])->String();
printf("tester: ");
for (int i = 0; i < str.length(); i++)
printf("%x ", str[i]);
printf("\n");
}
switch (token->Type()) {
case CharacterString:
cout << " string == " << token->String() << endl;
break;
case Integer:
cout << " int == " << token->Int() << endl;
break;
case FloatingPoint:
cout << " float == " << token->Float() << endl;
break;
}
*/
CHK(*token == *(testCases[i].tokens[j]));
delete testCases[i].tokens[j];
}
CHK(stream.IsEmpty());
} catch (Err *e) {
CppUnit::Exception *err = new CppUnit::Exception(e->Msg());
delete e;
throw *err;
}
}
#endif // !TEST_R5
}
