void Scanner::SetScannerBehindT() { buffer->SetPos(t->pos); NextCh(); line = t->line; col = t->col; charPos = t->charPos; for (size_t i = 0; i < tlen; i++) NextCh(); }
void Scanner::Init() { EOL = '\n'; eofSym = 0; maxT = 16; noSym = 16; int i; for (i = 65; i <= 90; ++i) start.set(i, 1); for (i = 97; i <= 122; ++i) start.set(i, 1); for (i = 48; i <= 57; ++i) start.set(i, 2); start.set(58, 19); start.set(59, 11); start.set(43, 13); start.set(45, 14); start.set(42, 15); start.set(94, 16); start.set(40, 17); start.set(41, 18); start.set(Buffer::EoF, -1); keywords.set(L"display", 3); keywords.set(L"halt", 8); keywords.set(L"mod", 12); tvalLength = 128; tval = new wchar_t[tvalLength]; // text of current token // HEAP_BLOCK_SIZE byte heap + pointer to next heap block heap = malloc(HEAP_BLOCK_SIZE + sizeof(void*)); firstHeap = heap; heapEnd = (void**) (((char*) heap) + HEAP_BLOCK_SIZE); *heapEnd = 0; heapTop = heap; if (sizeof(Token) > HEAP_BLOCK_SIZE) { wprintf(L"--- Too small HEAP_BLOCK_SIZE\n"); exit(1); } pos = -1; line = 1; col = 0; oldEols = 0; NextCh(); if (ch == 0xEF) { // check optional byte order mark for UTF-8 NextCh(); int ch1 = ch; NextCh(); int ch2 = ch; if (ch1 != 0xBB || ch2 != 0xBF) { wprintf(L"Illegal byte order mark at start of file"); exit(1); } Buffer *oldBuf = buffer; buffer = new UTF8Buffer(buffer); col = 0; delete oldBuf; oldBuf = NULL; NextCh(); } pt = tokens = CreateToken(); // first token is a dummy }
bool Scanner::Comment0() { int level = 1, pos0 = pos, line0 = line, col0 = col, charPos0 = charPos; NextCh(); if (ch == L'+') { NextCh(); for(;;) { if (ch == L'+') { NextCh(); if (ch == L'/') { level--; if (level == 0) { oldEols = line - line0; NextCh(); return true; } NextCh(); } } else if (ch == L'/') { NextCh(); if (ch == L'+') { level++; NextCh(); } } else if (ch == buffer->EoF) return false; else NextCh(); } } else { buffer->SetPos(pos0); NextCh(); line = line0; col = col0; charPos = charPos0; } return false; }
void Scanner::Init() { EOL = '\n'; eofSym = 0; maxT = 9; noSym = 9; int i; for (i = 48; i <= 57; ++i) start.set(i, 1); start.set(99, 2); start.set(40, 6); start.set(41, 7); start.set(43, 8); start.set(45, 9); start.set(42, 10); start.set(47, 11); start.set(Buffer::EoF, -1); tvalLength = 128; tval = new wchar_t[tvalLength]; // text of current token // COCO_HEAP_BLOCK_SIZE byte heap + pointer to next heap block heap = malloc(COCO_HEAP_BLOCK_SIZE + sizeof(void*)); firstHeap = heap; heapEnd = (void**) (((char*) heap) + COCO_HEAP_BLOCK_SIZE); *heapEnd = 0; heapTop = heap; if (sizeof(Token) > COCO_HEAP_BLOCK_SIZE) { wprintf(L"--- Too small COCO_HEAP_BLOCK_SIZE\n"); exit(1); } pos = -1; line = 1; col = 0; charPos = -1; oldEols = 0; NextCh(); if (ch == 0xEF) { // check optional byte order mark for UTF-8 NextCh(); int ch1 = ch; NextCh(); int ch2 = ch; if (ch1 != 0xBB || ch2 != 0xBF) { wprintf(L"Illegal byte order mark at start of file"); exit(1); } Buffer *oldBuf = buffer; buffer = new UTF8Buffer(buffer); col = 0; charPos = -1; delete oldBuf; oldBuf = NULL; NextCh(); } pt = tokens = CreateToken(); // first token is a dummy }
void CRScanner::Reset() { CurrLine = 1; LineStart = 0; BuffPos = -1; CurrCol = 0; ComEols = 0; NextSym.Init(); NextCh(); }
void CParser::DigRep() { // changes the current character to the first character that // isn't a decimal while ((ch >= '0') && (ch <= '9')) NextCh(); }
void ResetText(){ /*if(Path == NULL){ fprintf(stderr, "Call format: O <source code path>\n"); exit(1); } else */if((fInput = fopen(/*Path*/"Input.txt", "rb")) == NULL){ ResetError = TRUE; Message = "Source code file not found!"; } else{ ResetError = FALSE; fseek(fInput, 0, SEEK_END); len = ftell(fInput); if(len == -1L){ fprintf(stderr, "Input file fseek error."); exit(1); } bytes = malloc(len); fseek(fInput, 0, SEEK_SET); fread(bytes, 1, len, fInput); Message = "OK"; Pos = 0; Line = 1; fclose(fInput); NextCh(); } }
void ScanPastEOL() { int ch; do { ch = NextCh(); } while (ch != '\n' && ch != 0); }
Token* Scanner::NextToken() { while (ch == ' ' || ch == 10 || ch == 13 ) NextCh(); int recKind = noSym; int recEnd = pos; t = CreateToken(); t->pos = pos; t->col = col; t->line = line; t->charPos = charPos; int state = start.state(ch); tlen = 0; AddCh(); switch (state) { case -1: { t->kind = eofSym; break; } // NextCh already done case 0: { case_0: if (recKind != noSym) { tlen = recEnd - t->pos; SetScannerBehindT(); } t->kind = recKind; break; } // NextCh already done case 1: case_1: recEnd = pos; recKind = 1; if ((ch >= L'0' && ch <= L'9')) {AddCh(); goto case_1;} else {t->kind = 1; break;} case 2: if (ch == L'a') {AddCh(); goto case_3;} else {goto case_0;} case 3: case_3: if (ch == L'l') {AddCh(); goto case_4;} else {goto case_0;} case 4: case_4: if (ch == L'c') {AddCh(); goto case_5;} else {goto case_0;} case 5: case_5: {t->kind = 2; break;} case 6: {t->kind = 3; break;} case 7: {t->kind = 4; break;} case 8: {t->kind = 5; break;} case 9: {t->kind = 6; break;} case 10: {t->kind = 7; break;} case 11: {t->kind = 8; break;} } AppendVal(t); return t; }
// Skips spaces in input void SkipSpaces() { int c; do { c = NextCh(); } while(c != '\n' && isspace(c) && c != 0); unNextCh(); }
// Gets the next non space character int NextNonSpace(int skipnl) { int ch; do { ch = NextCh(); } while((ch != '\n' || skipnl) && isspace(ch) && ch!=0); return (ch); }
void CParser::CharRep() { // changes the current character to the first character that // isn't an alphanumeric character while (((ch >= '0') && (ch <= '9')) || ((ch >= 'a') && (ch <= 'z')) || ((ch >= 'A') && (ch <= 'Z')) || (ch == '.') || (ch == '_')) NextCh(); }
static void AddCh(CJcScanner* scanner){ char *newBuf; if (scanner->tlen >= scanner->tvalLength){ scanner->tvalLength *= 2; newBuf = (char*)g_oInterface.Malloc(scanner->tvalLength); MemoryCopy(newBuf, scanner->tval, scanner->tlen*sizeof(char)); g_oInterface.Free(scanner->tval); scanner->tval = newBuf; } scanner->tval[scanner->tlen++] = scanner->ch; NextCh(scanner); }
Token* Scanner::NextToken() { while (ch == ' ' || ch == 10 || ch == 13 ) NextCh(); int recKind = noSym; int recEnd = pos; t = CreateToken(); t->pos = pos; t->col = col; t->line = line; int state = start.state(ch); tlen = 0; AddCh(); switch (state) { case -1: { t->kind = eofSym; break; } // NextCh already done case 0: { case_0: if (recKind != noSym) { tlen = recEnd - t->pos; SetScannerBehindT(); } t->kind = recKind; break; } // NextCh already done case 1: case_1: recEnd = pos; recKind = 1; if ((ch >= L'0' && ch <= L'9') || (ch >= L'A' && ch <= L'Z') || (ch >= L'a' && ch <= L'z')) {AddCh(); goto case_1;} else {t->kind = 1; wchar_t *literal = coco_string_create(tval, 0, tlen); t->kind = keywords.get(literal, t->kind); coco_string_delete(literal); break;} case 2: case_2: recEnd = pos; recKind = 2; if ((ch >= L'0' && ch <= L'9')) {AddCh(); goto case_2;} else {t->kind = 2; break;} case 3: if (ch == L'=') {AddCh(); goto case_4;} else {goto case_0;} case 4: case_4: {t->kind = 3; break;} case 5: {t->kind = 4; break;} case 6: {t->kind = 5; break;} case 7: {t->kind = 6; break;} case 8: {t->kind = 8; break;} case 9: {t->kind = 9; break;} } AppendVal(t); return t; }
void Scanner::AddCh() { if (tlen >= tvalLength) { tvalLength *= 2; wchar_t *newBuf = new wchar_t[tvalLength]; memcpy(newBuf, tval, tlen*sizeof(wchar_t)); delete [] tval; tval = newBuf; } if (ch != Buffer::EoF) { tval[tlen++] = valCh; NextCh(); } }
void UnassembleCommand(void) { int count = 8; // Default instruction count to display Sb(); if (IsDwDebugNumeric(NextCh())) {Uaddr = ReadInstructionAddress("U"); Wl();} Sb(); if (IsDwDebugNumeric(NextCh())) {count = ReadNumber(0);} Sb(); if (!DwEoln()) {Wsl("Unrecognised parameters on unassemble command.");} int firstByte = Uaddr; int limitByte = min(firstByte + count*4, FlashSize()); // Allow for up to 2 words per instruction int length = limitByte - firstByte; if (length <= 0) {Fail("Nothing to disassemble.");} u8 buf[length+2]; DwReadFlash(firstByte, length, buf); buf[length] = 0; buf[length+1] = 0; while (1) { Uaddr += DisassembleInstruction(Uaddr, &buf[Uaddr-firstByte]); count--; if (count <= 0 || Uaddr >= FlashSize()) {return;} Wl(); } }
void CParser::TermChar(char c) { // generates an error if the next char isn't the specified char c, // otherwise, skip the char if (ch == c) { NextCh(); } else { STR_String str; str.Format("Warning: %c expected\ncontinuing without it", c); trace(str); } }
void NextCh(){ if( (Ch = bytes[++Index]) == EOF ){ Ch = chEOT; } else if(Ch == '\n'){ Line++; Pos = 0; Ch = chEOL; } else if(Ch == '\r'){ NextCh(); } else if(Ch == '\t'){ while(++Pos % TABSIZE); } else{ Pos++; } }
void SearchForDefined() { char *ptr, *id, *sptr; int c; SDef tdef, *p; ptr = inptr; while(1) { if (PeekCh() == 0) // Stop at end of current input break; SkipSpaces(); sptr = inptr; id = GetIdentifier(); if (id) { if (strcmp(id, "defined") == 0) { c = NextNonSpace(0); if (c != '(') { err(20); break; } id = GetIdentifier(); if (id == NULL) { err(21); break; } c = NextNonSpace(0); if (c != ')') err(22); tdef.name = id; p = (SDef *)htFind(&HashInfo, &tdef); SubMacro((char *)(p ? "1" : "0"), inptr-sptr); } } else NextCh(); } inptr = ptr; }
Token* Scanner::NextToken() { while (ch == ' ' || (ch >= 9 && ch <= 10) || ch == 13 ) NextCh(); int recKind = noSym; int recEnd = pos; t = CreateToken(); t->pos = pos; t->col = col; t->line = line; t->charPos = charPos; int state = start.state(ch); tlen = 0; AddCh(); switch (state) { case -1: { t->kind = eofSym; break; } // NextCh already done case 0: { case_0: if (recKind != noSym) { tlen = recEnd - t->pos; SetScannerBehindT(); } t->kind = recKind; break; } // NextCh already done case 1: case_1: if ((ch >= L'0' && ch <= L'9')) {AddCh(); goto case_2;} else {goto case_0;} case 2: case_2: recEnd = pos; recKind = 2; if ((ch >= L'0' && ch <= L'9')) {AddCh(); goto case_2;} else {t->kind = 2; break;} case 3: if ((ch >= L'A' && ch <= L'Z') || (ch >= L'a' && ch <= L'z')) {AddCh(); goto case_4;} else {goto case_0;} case 4: case_4: if ((ch >= L'A' && ch <= L'Z') || (ch >= L'a' && ch <= L'z')) {AddCh(); goto case_4;} else if (ch == L'"') {AddCh(); goto case_5;} else {goto case_0;} case 5: case_5: {t->kind = 3; break;} case 6: case_6: recEnd = pos; recKind = 4; if ((ch >= L'0' && ch <= L'9') || (ch >= L'A' && ch <= L'Z') || ch == L'_' || (ch >= L'a' && ch <= L'z')) {AddCh(); goto case_6;} else {t->kind = 4; wchar_t *literal = coco_string_create(tval, 0, tlen); t->kind = keywords.get(literal, t->kind); coco_string_delete(literal); break;} case 7: case_7: recEnd = pos; recKind = 1; if ((ch >= L'0' && ch <= L'9')) {AddCh(); goto case_7;} else if (ch == L'.') {AddCh(); goto case_1;} else {t->kind = 1; break;} case 8: {t->kind = 6; break;} case 9: {t->kind = 7; break;} case 10: {t->kind = 9; break;} case 11: {t->kind = 10; break;} case 12: {t->kind = 11; break;} case 13: if (ch == L'.') {AddCh(); goto case_14;} else {goto case_0;} case 14: case_14: {t->kind = 12; break;} case 15: {t->kind = 13; break;} case 16: {t->kind = 17; break;} case 17: {t->kind = 18; break;} case 18: {t->kind = 22; break;} case 19: {t->kind = 23; break;} case 20: {t->kind = 31; break;} case 21: {t->kind = 33; break;} case 22: case_22: {t->kind = 53; break;} case 23: case_23: {t->kind = 54; break;} case 24: case_24: {t->kind = 55; break;} case 25: case_25: {t->kind = 56; break;} case 26: {t->kind = 61; break;} case 27: {t->kind = 62; break;} case 28: {t->kind = 63; break;} case 29: recEnd = pos; recKind = 26; if (ch == L'=') {AddCh(); goto case_22;} else {t->kind = 26; break;} case 30: recEnd = pos; recKind = 57; if (ch == L'>') {AddCh(); goto case_23;} else if (ch == L'=') {AddCh(); goto case_24;} else {t->kind = 57; break;} case 31: recEnd = pos; recKind = 58; if (ch == L'=') {AddCh(); goto case_25;} else {t->kind = 58; break;} } AppendVal(t); return t; }
TokenRef Scanner::NextToken() { while (ch == ' ' || (ch >= 9 && ch <= 10) || ch == 13 || ch == ' ' ) NextCh(); if ((ch == '/' && Comment0()) || (ch == '/' && Comment1())) return NextToken(); int recKind = noSym; size_t recEnd = pos; t = CreateToken(); t->pos = pos; t->col = col; t->line = line; t->charPos = charPos; int state = start.state(ch); tlen = 0; AddCh(); switch (state) { case -1: { t->kind = eofSym; break; } // NextCh already done case 0: { case_0: if (recKind != noSym) { tlen = recEnd - t->pos; SetScannerBehindT(); } t->kind = recKind; break; } // NextCh already done case 1: case_1: recEnd = pos; recKind = 1; if ((ch >= '0' && ch <= '9') || (ch >= 'A' && ch <= 'Z') || ch == '_' || (ch >= 'a' && ch <= 'z')) {AddCh(); goto case_1;} else {t->kind = 1; char *literal = coco_string_create(tval, 0, tlen); t->kind = keywords.get(literal, t->kind); coco_string_delete(literal); break;} case 2: case_2: if ((ch >= '0' && ch <= '9')) {AddCh(); goto case_3;} else {goto case_0;} case 3: case_3: recEnd = pos; recKind = 3; if ((ch >= '0' && ch <= '9')) {AddCh(); goto case_3;} else {t->kind = 3; break;} case 4: case_4: if ((ch >= '0' && ch <= '9') || (ch >= 'a' && ch <= 'f')) {AddCh(); goto case_5;} else {goto case_0;} case 5: case_5: if ((ch >= '0' && ch <= '9') || (ch >= 'a' && ch <= 'f')) {AddCh(); goto case_6;} else {goto case_0;} case 6: case_6: if ((ch >= '0' && ch <= '9') || (ch >= 'a' && ch <= 'f')) {AddCh(); goto case_7;} else {goto case_0;} case 7: case_7: if ((ch >= '0' && ch <= '9') || (ch >= 'a' && ch <= 'f')) {AddCh(); goto case_8;} else {goto case_0;} case 8: case_8: if ((ch >= '0' && ch <= '9') || (ch >= 'a' && ch <= 'f')) {AddCh(); goto case_9;} else {goto case_0;} case 9: case_9: recEnd = pos; recKind = 4; if ((ch >= '0' && ch <= '9') || (ch >= 'a' && ch <= 'f')) {AddCh(); goto case_10;} else {t->kind = 4; break;} case 10: case_10: if ((ch >= '0' && ch <= '9') || (ch >= 'a' && ch <= 'f')) {AddCh(); goto case_11;} else {goto case_0;} case 11: case_11: {t->kind = 4; break;} case 12: if ((ch >= 'A' && ch <= 'Z') || ch == '_' || (ch >= 'a' && ch <= 'z')) {AddCh(); goto case_13;} else {goto case_0;} case 13: case_13: recEnd = pos; recKind = 5; if ((ch >= '0' && ch <= '9') || (ch >= 'A' && ch <= 'Z') || ch == '_' || (ch >= 'a' && ch <= 'z')) {AddCh(); goto case_13;} else {t->kind = 5; break;} case 14: case_14: if (ch <= '!' || (ch >= '#' && ch <= '[') || (ch >= ']' && ch <= 65535)) {AddCh(); goto case_14;} else if (ch == '"') {AddCh(); goto case_15;} else if (ch == 92) {AddCh(); goto case_17;} else {goto case_0;} case 15: case_15: {t->kind = 6; break;} case 16: case_16: recEnd = pos; recKind = 2; if ((ch >= '0' && ch <= '9')) {AddCh(); goto case_16;} else if (ch == '.') {AddCh(); goto case_2;} else {t->kind = 2; break;} case 17: case_17: if (ch <= '!' || (ch >= '#' && ch <= '[') || (ch >= ']' && ch <= 65535)) {AddCh(); goto case_14;} else if (ch == 92) {AddCh(); goto case_17;} else if (ch == '"') {AddCh(); goto case_18;} else {goto case_0;} case 18: case_18: recEnd = pos; recKind = 6; if (ch <= '!' || (ch >= '#' && ch <= '[') || (ch >= ']' && ch <= 65535)) {AddCh(); goto case_14;} else if (ch == '"') {AddCh(); goto case_15;} else if (ch == 92) {AddCh(); goto case_17;} else {t->kind = 6; break;} case 19: {t->kind = 11; break;} case 20: {t->kind = 12; break;} case 21: {t->kind = 15; break;} case 22: {t->kind = 16; break;} case 23: {t->kind = 20; break;} case 24: {t->kind = 31; break;} case 25: {t->kind = 32; break;} case 26: {t->kind = 36; break;} case 27: {t->kind = 43; break;} case 28: {t->kind = 45; break;} case 29: {t->kind = 47; break;} case 30: {t->kind = 90; break;} case 31: {t->kind = 91; break;} case 32: {t->kind = 93; break;} case 33: recEnd = pos; recKind = 49; if ((ch >= '0' && ch <= '9') || (ch >= 'a' && ch <= 'f')) {AddCh(); goto case_4;} else {t->kind = 49; break;} } AppendVal(t); return t; }
Token* Scanner::NextToken() { while (ch == ' ' || false ) NextCh(); t = CreateToken(); t->pos = pos; t->col = col; t->line = line; int state = start.state(ch); tlen = 0; AddCh(); switch (state) { case -1: { t->kind = eofSym; break; } // NextCh already done case 0: { t->kind = noSym; break; } // NextCh already done case 1: case_1: if ((ch >= L'0' && ch <= L'9') || (ch >= L'a' && ch <= L'z')) {AddCh(); goto case_1;} else {t->kind = 1; wchar_t *literal = coco_string_create_lower(tval, 0, tlen); t->kind = keywords.get(literal, t->kind); coco_string_delete(literal); break;} case 2: if (ch <= L'&' || (ch >= L'(' && ch <= 65535)) {AddCh(); goto case_3;} else {t->kind = noSym; break;} case 3: case_3: if (ch <= L'&' || (ch >= L'(' && ch <= 65535)) {AddCh(); goto case_3;} else if (ch == 39) {AddCh(); goto case_4;} else {t->kind = noSym; break;} case 4: case_4: {t->kind = 2; break;} case 5: case_5: if ((ch >= L'0' && ch <= L'9')) {AddCh(); goto case_5;} else {t->kind = 3; break;} case 6: if (ch == L'=') {AddCh(); goto case_7;} else {t->kind = noSym; break;} case 7: case_7: {t->kind = 4; break;} case 8: if (ch == L'=') {AddCh(); goto case_7;} else {t->kind = 4; break;} case 9: if (ch == L'=') {AddCh(); goto case_7;} else {t->kind = 4; break;} case 10: {t->kind = 5; break;} case 11: {t->kind = 8; break;} case 12: {t->kind = 9; break;} case 13: {t->kind = 10; break;} case 14: case_14: {t->kind = 20; break;} case 15: case_15: {t->kind = 21; break;} case 16: {t->kind = 26; break;} case 17: if (ch == L'=') {AddCh(); goto case_7;} else {t->kind = 31; break;} case 18: if ((ch >= L'0' && ch <= L'9') || (ch >= L'a' && ch <= L'g') || (ch >= L'i' && ch <= L'z')) {AddCh(); goto case_1;} else if (ch == L'h') {AddCh(); goto case_20;} else {t->kind = 1; wchar_t *literal = coco_string_create_lower(tval, 0, tlen); t->kind = keywords.get(literal, t->kind); coco_string_delete(literal); break;} case 19: if ((ch >= L'0' && ch <= L'9') || (ch >= L'b' && ch <= L'z')) {AddCh(); goto case_1;} else if (ch == L'a') {AddCh(); goto case_21;} else {t->kind = 1; wchar_t *literal = coco_string_create_lower(tval, 0, tlen); t->kind = keywords.get(literal, t->kind); coco_string_delete(literal); break;} case 20: case_20: if ((ch >= L'0' && ch <= L'9') || (ch >= L'b' && ch <= L'z')) {AddCh(); goto case_1;} else if (ch == L'a') {AddCh(); goto case_22;} else {t->kind = 1; wchar_t *literal = coco_string_create_lower(tval, 0, tlen); t->kind = keywords.get(literal, t->kind); coco_string_delete(literal); break;} case 21: case_21: if ((ch >= L'0' && ch <= L'9') || (ch >= L'a' && ch <= L'q') || (ch >= L's' && ch <= L'z')) {AddCh(); goto case_1;} else if (ch == L'r') {AddCh(); goto case_23;} else {t->kind = 1; wchar_t *literal = coco_string_create_lower(tval, 0, tlen); t->kind = keywords.get(literal, t->kind); coco_string_delete(literal); break;} case 22: case_22: if ((ch >= L'0' && ch <= L'9') || (ch >= L'a' && ch <= L'q') || (ch >= L's' && ch <= L'z')) {AddCh(); goto case_1;} else if (ch == L'r') {AddCh(); goto case_24;} else {t->kind = 1; wchar_t *literal = coco_string_create_lower(tval, 0, tlen); t->kind = keywords.get(literal, t->kind); coco_string_delete(literal); break;} case 23: case_23: if ((ch >= L'0' && ch <= L'9') || (ch >= L'a' && ch <= L'b') || (ch >= L'd' && ch <= L'z')) {AddCh(); goto case_1;} else if (ch == L'c') {AddCh(); goto case_25;} else {t->kind = 1; wchar_t *literal = coco_string_create_lower(tval, 0, tlen); t->kind = keywords.get(literal, t->kind); coco_string_delete(literal); break;} case 24: case_24: if ((ch >= L'0' && ch <= L'9') || (ch >= L'a' && ch <= L'z')) {AddCh(); goto case_1;} else if (ch == L'(') {AddCh(); goto case_14;} else {t->kind = 1; wchar_t *literal = coco_string_create_lower(tval, 0, tlen); t->kind = keywords.get(literal, t->kind); coco_string_delete(literal); break;} case 25: case_25: if ((ch >= L'0' && ch <= L'9') || (ch >= L'a' && ch <= L'g') || (ch >= L'i' && ch <= L'z')) {AddCh(); goto case_1;} else if (ch == L'h') {AddCh(); goto case_26;} else {t->kind = 1; wchar_t *literal = coco_string_create_lower(tval, 0, tlen); t->kind = keywords.get(literal, t->kind); coco_string_delete(literal); break;} case 26: case_26: if ((ch >= L'0' && ch <= L'9') || (ch >= L'b' && ch <= L'z')) {AddCh(); goto case_1;} else if (ch == L'a') {AddCh(); goto case_27;} else {t->kind = 1; wchar_t *literal = coco_string_create_lower(tval, 0, tlen); t->kind = keywords.get(literal, t->kind); coco_string_delete(literal); break;} case 27: case_27: if ((ch >= L'0' && ch <= L'9') || (ch >= L'a' && ch <= L'q') || (ch >= L's' && ch <= L'z')) {AddCh(); goto case_1;} else if (ch == L'r') {AddCh(); goto case_28;} else {t->kind = 1; wchar_t *literal = coco_string_create_lower(tval, 0, tlen); t->kind = keywords.get(literal, t->kind); coco_string_delete(literal); break;} case 28: case_28: if ((ch >= L'0' && ch <= L'9') || (ch >= L'a' && ch <= L'z')) {AddCh(); goto case_1;} else if (ch == L'(') {AddCh(); goto case_15;} else {t->kind = 1; wchar_t *literal = coco_string_create_lower(tval, 0, tlen); t->kind = keywords.get(literal, t->kind); coco_string_delete(literal); break;} } AppendVal(t); return t; }
void Scanner::Init() { EOL = '\n'; eofSym = 0; maxT = 94; noSym = 94; int i; for (i = 65; i <= 90; ++i) start.set(i, 1); for (i = 95; i <= 95; ++i) start.set(i, 1); for (i = 97; i <= 122; ++i) start.set(i, 1); for (i = 48; i <= 57; ++i) start.set(i, 16); start.set(35, 33); start.set(64, 12); start.set(34, 14); start.set(123, 19); start.set(125, 20); start.set(61, 21); start.set(59, 22); start.set(44, 23); start.set(91, 24); start.set(93, 25); start.set(45, 26); start.set(58, 27); start.set(60, 28); start.set(46, 29); start.set(40, 30); start.set(41, 31); start.set(33, 32); start.set(Buffer::EoF, -1); keywords.set("OSS", 7); keywords.set("END", 8); keywords.set("FLAG", 9); keywords.set("IF", 10); keywords.set("ELIF", 13); keywords.set("ELSE", 14); keywords.set("ORDER", 17); keywords.set("WAYS", 18); keywords.set("GROUP", 19); keywords.set("SYMBOL", 21); keywords.set("POLYGON", 22); keywords.set("RECTANGLE", 23); keywords.set("x", 24); keywords.set("CIRCLE", 25); keywords.set("CONST", 26); keywords.set("COLOR", 27); keywords.set("MAG", 28); keywords.set("UINT", 29); keywords.set("STYLE", 30); keywords.set("FEATURE", 33); keywords.set("PATH", 34); keywords.set("TYPE", 35); keywords.set("ONEWAY", 37); keywords.set("BRIDGE", 38); keywords.set("TUNNEL", 39); keywords.set("SIZE", 40); keywords.set("m", 41); keywords.set("mm", 42); keywords.set("px", 44); keywords.set("NODE", 46); keywords.set("TEXT", 48); keywords.set("ICON", 50); keywords.set("WAY", 51); keywords.set("SHIELD", 52); keywords.set("AREA", 53); keywords.set("BORDERTEXT", 54); keywords.set("BORDERSYMBOL", 55); keywords.set("color", 56); keywords.set("dash", 57); keywords.set("gapColor", 58); keywords.set("displayWidth", 59); keywords.set("width", 60); keywords.set("displayOffset", 61); keywords.set("offset", 62); keywords.set("cap", 63); keywords.set("joinCap", 64); keywords.set("endCap", 65); keywords.set("priority", 66); keywords.set("zIndex", 67); keywords.set("pattern", 68); keywords.set("patternMinMag", 69); keywords.set("borderColor", 70); keywords.set("borderWidth", 71); keywords.set("borderDash", 72); keywords.set("label", 73); keywords.set("style", 74); keywords.set("size", 75); keywords.set("scaleMag", 76); keywords.set("autoSize", 77); keywords.set("position", 78); keywords.set("backgroundColor", 79); keywords.set("shieldSpace", 80); keywords.set("symbol", 81); keywords.set("symbolSpace", 82); keywords.set("name", 83); keywords.set("butt", 84); keywords.set("round", 85); keywords.set("square", 86); keywords.set("normal", 87); keywords.set("emphasize", 88); keywords.set("lighten", 89); keywords.set("darken", 92); tvalLength = 128; tval = new char[tvalLength]; // text of current token pos = -1; line = 1; col = 0; oldEols = 0; NextCh(); if (ch == 0xEF) { // check optional byte order mark for UTF-8 NextCh(); int ch1 = ch; NextCh(); int ch2 = ch; if (ch1 != 0xBB || ch2 != 0xBF) { std::cerr << "Illegal byte order mark at start of file" << std::endl; exit(1); } NextCh(); } pt = tokens = CreateToken(); // first token is a dummy }
void Scanner::Init() { EOL = '\n'; eofSym = 0; maxT = 38; noSym = 38; int i; for (i = 97; i <= 98; ++i) start.set(i, 1); for (i = 100; i <= 117; ++i) start.set(i, 1); for (i = 119; i <= 122; ++i) start.set(i, 1); for (i = 48; i <= 57; ++i) start.set(i, 5); start.set(39, 2); start.set(60, 8); start.set(62, 9); start.set(33, 6); start.set(61, 17); start.set(59, 10); start.set(40, 11); start.set(44, 12); start.set(41, 13); start.set(99, 18); start.set(118, 19); start.set(42, 16); start.set(Buffer::EoF, -1); keywords.set(L"create", 6); keywords.set(L"table", 7); keywords.set(L"null", 11); keywords.set(L"not", 12); keywords.set(L"default", 13); keywords.set(L"index", 14); keywords.set(L"int", 15); keywords.set(L"float", 16); keywords.set(L"long", 17); keywords.set(L"date", 18); keywords.set(L"time", 19); keywords.set(L"drop", 22); keywords.set(L"select", 23); keywords.set(L"distinct", 24); keywords.set(L"all", 25); keywords.set(L"from", 27); keywords.set(L"where", 28); keywords.set(L"update", 29); keywords.set(L"set", 30); keywords.set(L"insert", 32); keywords.set(L"into", 33); keywords.set(L"values", 34); keywords.set(L"delete", 35); keywords.set(L"and", 36); keywords.set(L"or", 37); tvalLength = 128; tval = new wchar_t[tvalLength]; // text of current token // HEAP_BLOCK_SIZE byte heap + pointer to next heap block heap = malloc(HEAP_BLOCK_SIZE + sizeof(void*)); firstHeap = heap; heapEnd = (void**) (((char*) heap) + HEAP_BLOCK_SIZE); *heapEnd = 0; heapTop = heap; if (sizeof(Token) > HEAP_BLOCK_SIZE) { wprintf(L"--- Too small HEAP_BLOCK_SIZE\n"); exit(1); } pos = -1; line = 1; col = 0; oldEols = 0; NextCh(); if (ch == 0xEF) { // check optional byte order mark for UTF-8 NextCh(); int ch1 = ch; NextCh(); int ch2 = ch; if (ch1 != 0xBB || ch2 != 0xBF) { wprintf(L"Illegal byte order mark at start of file"); exit(1); } Buffer *oldBuf = buffer; buffer = new UTF8Buffer(buffer); col = 0; delete oldBuf; oldBuf = NULL; NextCh(); } pt = tokens = CreateToken(); // first token is a dummy }
void Scanner::Init() { EOL = '\n'; eofSym = 0; maxT = 61; noSym = 61; int i; for (i = 97; i <= 122; ++i) start.set(i, 1); for (i = 48; i <= 57; ++i) start.set(i, 61); start.set(34, 4); start.set(39, 6); start.set(10, 9); start.set(46, 62); start.set(65, 60); start.set(67, 64); start.set(123, 70); start.set(125, 71); start.set(86, 106); start.set(70, 73); start.set(80, 82); start.set(44, 90); start.set(40, 91); start.set(41, 92); start.set(45, 94); start.set(60, 107); start.set(62, 108); start.set(33, 98); start.set(61, 100); start.set(89, 101); start.set(79, 102); start.set(43, 103); start.set(42, 104); start.set(47, 105); start.set(Buffer::EoF, -1); keywords.set(L"motor", 20); keywords.set(L"servo", 21); keywords.set(L"pantalla", 22); keywords.set(L"led", 23); keywords.set(L"boton", 24); keywords.set(L"sensor", 25); keywords.set(L"pin", 27); keywords.set(L"espacios", 28); keywords.set(L"entero", 29); keywords.set(L"decimal", 30); keywords.set(L"frase", 31); keywords.set(L"enteroA", 32); keywords.set(L"decimalA", 33); keywords.set(L"condicion", 38); keywords.set(L"ciclo", 39); keywords.set(L"llamar", 40); keywords.set(L"asignar", 41); keywords.set(L"esperar", 42); keywords.set(L"si", 43); keywords.set(L"sino", 44); keywords.set(L"mientras", 45); keywords.set(L"haz", 46); keywords.set(L"veces", 47); keywords.set(L"posicion", 48); tvalLength = 128; tval = new wchar_t[tvalLength]; // text of current token // COCO_HEAP_BLOCK_SIZE byte heap + pointer to next heap block heap = malloc(COCO_HEAP_BLOCK_SIZE + sizeof(void*)); firstHeap = heap; heapEnd = (void**) (((char*) heap) + COCO_HEAP_BLOCK_SIZE); *heapEnd = 0; heapTop = heap; if (sizeof(Token) > COCO_HEAP_BLOCK_SIZE) { wprintf(L"--- Too small COCO_HEAP_BLOCK_SIZE\n"); exit(1); } pos = -1; line = 1; col = 0; charPos = -1; oldEols = 0; NextCh(); if (ch == 0xEF) { // check optional byte order mark for UTF-8 NextCh(); int ch1 = ch; NextCh(); int ch2 = ch; if (ch1 != 0xBB || ch2 != 0xBF) { wprintf(L"Illegal byte order mark at start of file"); exit(1); } Buffer *oldBuf = buffer; buffer = new UTF8Buffer(buffer); col = 0; charPos = -1; delete oldBuf; oldBuf = NULL; NextCh(); } pt = tokens = CreateToken(); // first token is a dummy }
Token* Scanner::NextToken() { while (ch == ' ' || (ch >= 9 && ch <= 10) || ch == 13 ) NextCh(); if ((ch == L'/' && Comment0()) || (ch == L'/' && Comment1()) || (ch == L'/' && Comment2())) return NextToken(); int recKind = noSym; int recEnd = pos; t = CreateToken(); t->pos = pos; t->col = col; t->line = line; t->charPos = charPos; int state = start.state(ch); tlen = 0; AddCh(); switch (state) { case -1: { t->kind = eofSym; // NextCh already done break; } case 0: { case_0: if (recKind != noSym) { tlen = recEnd - t->pos; SetScannerBehindT(); } t->kind = recKind; break; } // NextCh already done case 1: case_1: recEnd = pos; recKind = 1; if ((ch >= L'0' && ch <= L'9') || (ch >= L'A' && ch <= L'Z') || ch == L'_' || (ch >= L'a' && ch <= L'z')) { AddCh(); goto case_1; } else { t->kind = 1; wchar_t *literal = coco_string_create(tval, 0, tlen); t->kind = keywords.get(literal, t->kind); coco_string_delete(literal); break; } case 2: case_2: recEnd = pos; recKind = 2; if ((ch >= L'0' && ch <= L'9')) { AddCh(); goto case_2; } else if (ch == L'F' || ch == L'L' || ch == L'f' || ch == L'l') { AddCh(); goto case_13; } else if (ch == L'E' || ch == L'e') { AddCh(); goto case_3; } else { t->kind = 2; break; } case 3: case_3: if ((ch >= L'0' && ch <= L'9')) { AddCh(); goto case_5; } else if (ch == L'+' || ch == L'-') { AddCh(); goto case_4; } else { goto case_0; } case 4: case_4: if ((ch >= L'0' && ch <= L'9')) { AddCh(); goto case_5; } else { goto case_0; } case 5: case_5: recEnd = pos; recKind = 2; if ((ch >= L'0' && ch <= L'9')) { AddCh(); goto case_5; } else if (ch == L'F' || ch == L'L' || ch == L'f' || ch == L'l') { AddCh(); goto case_13; } else { t->kind = 2; break; } case 6: case_6: recEnd = pos; recKind = 2; if ((ch >= L'0' && ch <= L'9')) { AddCh(); goto case_6; } else if (ch == L'F' || ch == L'L' || ch == L'f' || ch == L'l') { AddCh(); goto case_13; } else if (ch == L'E' || ch == L'e') { AddCh(); goto case_7; } else { t->kind = 2; break; } case 7: case_7: if ((ch >= L'0' && ch <= L'9')) { AddCh(); goto case_9; } else if (ch == L'+' || ch == L'-') { AddCh(); goto case_8; } else { goto case_0; } case 8: case_8: if ((ch >= L'0' && ch <= L'9')) { AddCh(); goto case_9; } else { goto case_0; } case 9: case_9: recEnd = pos; recKind = 2; if ((ch >= L'0' && ch <= L'9')) { AddCh(); goto case_9; } else if (ch == L'F' || ch == L'L' || ch == L'f' || ch == L'l') { AddCh(); goto case_13; } else { t->kind = 2; break; } case 10: case_10: if ((ch >= L'0' && ch <= L'9')) { AddCh(); goto case_12; } else if (ch == L'+' || ch == L'-') { AddCh(); goto case_11; } else { goto case_0; } case 11: case_11: if ((ch >= L'0' && ch <= L'9')) { AddCh(); goto case_12; } else { goto case_0; } case 12: case_12: recEnd = pos; recKind = 2; if ((ch >= L'0' && ch <= L'9')) { AddCh(); goto case_12; } else if (ch == L'F' || ch == L'L' || ch == L'f' || ch == L'l') { AddCh(); goto case_13; } else { t->kind = 2; break; } case 13: case_13: { t->kind = 2; break; } case 14: case_14: recEnd = pos; recKind = 3; if (ch == L'L' || ch == L'U' || ch == L'l' || ch == L'u') { AddCh(); goto case_14; } else { t->kind = 3; break; } case 15: case_15: if ((ch >= L'0' && ch <= L'9') || (ch >= L'A' && ch <= L'F') || (ch >= L'a' && ch <= L'f')) { AddCh(); goto case_16; } else { goto case_0; } case 16: case_16: recEnd = pos; recKind = 3; if ((ch >= L'0' && ch <= L'9') || (ch >= L'A' && ch <= L'F') || (ch >= L'a' && ch <= L'f')) { AddCh(); goto case_16; } else if (ch == L'L' || ch == L'U' || ch == L'l' || ch == L'u') { AddCh(); goto case_14; } else { t->kind = 3; break; } case 17: if (ch <= 9 || (ch >= 11 && ch <= 12) || (ch >= 14 && ch <= L'&') || (ch >= L'(' && ch <= 65535)) { AddCh(); goto case_18; } else { goto case_0; } case 18: case_18: if (ch <= 9 || (ch >= 11 && ch <= 12) || (ch >= 14 && ch <= L'&') || (ch >= L'(' && ch <= 65535)) { AddCh(); goto case_18; } else if (ch == 39) { AddCh(); goto case_19; } else { goto case_0; } case 19: case_19: { t->kind = 4; break; } case 20: case_20: if (ch <= 9 || (ch >= 11 && ch <= 12) || (ch >= 14 && ch <= L'!') || (ch >= L'#' && ch <= 65535)) { AddCh(); goto case_20; } else if (ch == L'"') { AddCh(); goto case_21; } else { goto case_0; } case 21: case_21: { t->kind = 5; break; } case 22: { t->kind = 25; break; } case 23: { t->kind = 26; break; } case 24: { t->kind = 27; break; } case 25: { t->kind = 28; break; } case 26: { t->kind = 29; break; } case 27: { t->kind = 30; break; } case 28: { t->kind = 31; break; } case 29: { t->kind = 32; break; } case 30: { t->kind = 33; break; } case 31: { t->kind = 34; break; } case 32: case_32: if (ch == L'.') { AddCh(); goto case_33; } else { goto case_0; } case 33: case_33: { t->kind = 35; break; } case 34: case_34: if (ch == L'e') { AddCh(); goto case_35; } else { goto case_0; } case 35: case_35: if (ch == L'f') { AddCh(); goto case_36; } else { goto case_0; } case 36: case_36: if (ch == L'i') { AddCh(); goto case_37; } else { goto case_0; } case 37: case_37: if (ch == L'n') { AddCh(); goto case_38; } else { goto case_0; } case 38: case_38: if (ch == L'e') { AddCh(); goto case_39; } else { goto case_0; } case 39: case_39: if (ch == 10 || ch == 13) { AddCh(); goto case_40; } else if (ch <= 9 || (ch >= 11 && ch <= 12) || (ch >= 14 && ch <= 65535)) { AddCh(); goto case_39; } else { goto case_0; } case 40: case_40: { t->kind = 110; break; } case 41: case_41: if (ch == L'n') { AddCh(); goto case_42; } else { goto case_0; } case 42: case_42: if (ch == L'd') { AddCh(); goto case_43; } else { goto case_0; } case 43: case_43: if (ch == L'e') { AddCh(); goto case_44; } else { goto case_0; } case 44: case_44: if (ch == L'f') { AddCh(); goto case_45; } else { goto case_0; } case 45: case_45: if (ch == 10 || ch == 13) { AddCh(); goto case_46; } else if (ch <= 9 || (ch >= 11 && ch <= 12) || (ch >= 14 && ch <= 65535)) { AddCh(); goto case_45; } else { goto case_0; } case 46: case_46: { t->kind = 111; break; } case 47: case_47: if (ch == 10 || ch == 13) { AddCh(); goto case_48; } else if (ch <= 9 || (ch >= 11 && ch <= 12) || (ch >= 14 && ch <= 65535)) { AddCh(); goto case_47; } else { goto case_0; } case 48: case_48: { t->kind = 112; break; } case 49: case_49: if (ch == L'f') { AddCh(); goto case_50; } else { goto case_0; } case 50: case_50: if (ch == 10 || ch == 13) { AddCh(); goto case_51; } else if (ch <= 9 || (ch >= 11 && ch <= 12) || (ch >= 14 && ch <= 65535)) { AddCh(); goto case_50; } else { goto case_0; } case 51: case_51: { t->kind = 113; break; } case 52: case_52: if (ch == L'e') { AddCh(); goto case_53; } else { goto case_0; } case 53: case_53: if (ch == 10 || ch == 13) { AddCh(); goto case_54; } else if (ch <= 9 || (ch >= 11 && ch <= 12) || (ch >= 14 && ch <= 65535)) { AddCh(); goto case_53; } else { goto case_0; } case 54: case_54: { t->kind = 114; break; } case 55: case_55: if (ch == L'd') { AddCh(); goto case_56; } else { goto case_0; } case 56: case_56: if (ch == L'i') { AddCh(); goto case_57; } else { goto case_0; } case 57: case_57: if (ch == L'f') { AddCh(); goto case_58; } else { goto case_0; } case 58: case_58: if (ch == 10 || ch == 13) { AddCh(); goto case_59; } else if (ch <= 9 || (ch >= 11 && ch <= 12) || (ch >= 14 && ch <= 65535)) { AddCh(); goto case_58; } else { goto case_0; } case 59: case_59: { t->kind = 115; break; } case 60: case_60: if (ch == L'c') { AddCh(); goto case_61; } else { goto case_0; } case 61: case_61: if (ch == L'l') { AddCh(); goto case_62; } else { goto case_0; } case 62: case_62: if (ch == L'u') { AddCh(); goto case_63; } else { goto case_0; } case 63: case_63: if (ch == L'd') { AddCh(); goto case_64; } else { goto case_0; } case 64: case_64: if (ch == L'e') { AddCh(); goto case_65; } else { goto case_0; } case 65: case_65: if (ch == 10 || ch == 13) { AddCh(); goto case_66; } else if (ch <= 9 || (ch >= 11 && ch <= 12) || (ch >= 14 && ch <= 65535)) { AddCh(); goto case_65; } else { goto case_0; } case 66: case_66: { t->kind = 116; break; } case 67: recEnd = pos; recKind = 3; if ((ch >= L'8' && ch <= L'9')) { AddCh(); goto case_71; } else if ((ch >= L'0' && ch <= L'7')) { AddCh(); goto case_72; } else if (ch == L'.') { AddCh(); goto case_6; } else if (ch == L'E' || ch == L'e') { AddCh(); goto case_10; } else if (ch == L'L' || ch == L'U' || ch == L'l' || ch == L'u') { AddCh(); goto case_14; } else if (ch == L'X' || ch == L'x') { AddCh(); goto case_15; } else { t->kind = 3; break; } case 68: case_68: recEnd = pos; recKind = 3; if ((ch >= L'0' && ch <= L'9')) { AddCh(); goto case_68; } else if (ch == L'.') { AddCh(); goto case_6; } else if (ch == L'E' || ch == L'e') { AddCh(); goto case_10; } else if (ch == L'L' || ch == L'U' || ch == L'l' || ch == L'u') { AddCh(); goto case_14; } else { t->kind = 3; break; } case 69: if ((ch >= L'0' && ch <= L'9')) { AddCh(); goto case_2; } else if (ch == L'.') { AddCh(); goto case_32; } else { goto case_0; } case 70: case_70: if (ch == 9 || (ch >= 11 && ch <= 12) || ch == L' ') { AddCh(); goto case_70; } else if (ch == L'd') { AddCh(); goto case_34; } else if (ch == L'u') { AddCh(); goto case_41; } else if (ch == L'i') { AddCh(); goto case_73; } else if (ch == L'e') { AddCh(); goto case_74; } else { goto case_0; } case 71: case_71: if ((ch >= L'0' && ch <= L'9')) { AddCh(); goto case_71; } else if (ch == L'.') { AddCh(); goto case_6; } else if (ch == L'E' || ch == L'e') { AddCh(); goto case_10; } else { goto case_0; } case 72: case_72: recEnd = pos; recKind = 3; if ((ch >= L'8' && ch <= L'9')) { AddCh(); goto case_71; } else if ((ch >= L'0' && ch <= L'7')) { AddCh(); goto case_72; } else if (ch == L'.') { AddCh(); goto case_6; } else if (ch == L'E' || ch == L'e') { AddCh(); goto case_10; } else if (ch == L'L' || ch == L'U' || ch == L'l' || ch == L'u') { AddCh(); goto case_14; } else { t->kind = 3; break; } case 73: case_73: if (ch == L'f') { AddCh(); goto case_47; } else if (ch == L'n') { AddCh(); goto case_60; } else { goto case_0; } case 74: case_74: if (ch == L'l') { AddCh(); goto case_75; } else if (ch == L'n') { AddCh(); goto case_55; } else { goto case_0; } case 75: case_75: if (ch == L'i') { AddCh(); goto case_49; } else if (ch == L's') { AddCh(); goto case_52; } else { goto case_0; } case 76: case_76: { t->kind = 93; break; } case 77: { t->kind = 96; break; } case 78: { t->kind = 99; break; } case 79: { t->kind = 100; break; } case 80: recEnd = pos; recKind = 1; if ((ch >= L'0' && ch <= L'9') || (ch >= L'A' && ch <= L'F') || (ch >= L'H' && ch <= L'Z') || ch == L'_' || (ch >= L'a' && ch <= L'z')) { AddCh(); goto case_1; } else if (ch == L'G') { AddCh(); goto case_81; } else { t->kind = 1; wchar_t *literal = coco_string_create(tval, 0, tlen); t->kind = keywords.get(literal, t->kind); coco_string_delete(literal); break; } case 81: case_81: recEnd = pos; recKind = 1; if ((ch >= L'0' && ch <= L'9') || ch == L'A' || (ch >= L'C' && ch <= L'Z') || ch == L'_' || (ch >= L'a' && ch <= L'z')) { AddCh(); goto case_1; } else if (ch == L'B') { AddCh(); goto case_82; } else { t->kind = 1; wchar_t *literal = coco_string_create(tval, 0, tlen); t->kind = keywords.get(literal, t->kind); coco_string_delete(literal); break; } case 82: case_82: recEnd = pos; recKind = 1; if ((ch >= L'0' && ch <= L'9') || (ch >= L'B' && ch <= L'Z') || ch == L'_' || (ch >= L'a' && ch <= L'z')) { AddCh(); goto case_1; } else if (ch == L'A') { AddCh(); goto case_83; } else { t->kind = 1; wchar_t *literal = coco_string_create(tval, 0, tlen); t->kind = keywords.get(literal, t->kind); coco_string_delete(literal); break; } case 83: case_83: recEnd = pos; recKind = 1; if ((ch >= L'0' && ch <= L'9') || (ch >= L'A' && ch <= L'Z') || ch == L'_' || (ch >= L'a' && ch <= L'z')) { AddCh(); goto case_1; } else if (ch == L'(') { AddCh(); goto case_76; } else { t->kind = 1; wchar_t *literal = coco_string_create(tval, 0, tlen); t->kind = keywords.get(literal, t->kind); coco_string_delete(literal); break; } } AppendVal(t); return t; }
Token* Scanner::NextToken() { while (ch == ' ' || ch == 13 ) NextCh(); if ((ch == L'/' && Comment0())) return NextToken(); int recKind = noSym; int recEnd = pos; t = CreateToken(); t->pos = pos; t->col = col; t->line = line; t->charPos = charPos; int state = start.state(ch); tlen = 0; AddCh(); switch (state) { case -1: { t->kind = eofSym; break; } // NextCh already done case 0: { case_0: if (recKind != noSym) { tlen = recEnd - t->pos; SetScannerBehindT(); } t->kind = recKind; break; } // NextCh already done case 1: case_1: recEnd = pos; recKind = 1; if ((ch >= L'0' && ch <= L'9') || (ch >= L'A' && ch <= L'Z') || (ch >= L'a' && ch <= L'z')) {AddCh(); goto case_1;} else {t->kind = 1; wchar_t *literal = coco_string_create(tval, 0, tlen); t->kind = keywords.get(literal, t->kind); coco_string_delete(literal); break;} case 2: case_2: if ((ch >= L'0' && ch <= L'9')) {AddCh(); goto case_3;} else {goto case_0;} case 3: case_3: recEnd = pos; recKind = 3; if ((ch >= L'0' && ch <= L'9')) {AddCh(); goto case_3;} else {t->kind = 3; break;} case 4: case_4: if (ch == L' ' || (ch >= L'0' && ch <= L'9') || (ch >= L'A' && ch <= L'Z') || (ch >= L'a' && ch <= L'z')) {AddCh(); goto case_4;} else if (ch == L'"') {AddCh(); goto case_5;} else {goto case_0;} case 5: case_5: {t->kind = 4; break;} case 6: if ((ch >= L'A' && ch <= L'Z') || (ch >= L'a' && ch <= L'z')) {AddCh(); goto case_7;} else {goto case_0;} case 7: case_7: if (ch == 39) {AddCh(); goto case_8;} else {goto case_0;} case 8: case_8: {t->kind = 5; break;} case 9: {t->kind = 6; break;} case 10: case_10: if (ch == L'e') {AddCh(); goto case_11;} else {goto case_0;} case 11: case_11: if (ch == L'l') {AddCh(); goto case_12;} else {goto case_0;} case 12: case_12: if (ch == L'o') {AddCh(); goto case_13;} else {goto case_0;} case 13: case_13: if (ch == L'c') {AddCh(); goto case_14;} else {goto case_0;} case 14: case_14: if (ch == L'i') {AddCh(); goto case_15;} else {goto case_0;} case 15: case_15: if (ch == L'd') {AddCh(); goto case_16;} else {goto case_0;} case 16: case_16: if (ch == L'a') {AddCh(); goto case_17;} else {goto case_0;} case 17: case_17: if (ch == L'd') {AddCh(); goto case_26;} else {goto case_0;} case 18: case_18: if (ch == L'i') {AddCh(); goto case_19;} else {goto case_0;} case 19: case_19: if (ch == L'r') {AddCh(); goto case_20;} else {goto case_0;} case 20: case_20: if (ch == L'e') {AddCh(); goto case_21;} else {goto case_0;} case 21: case_21: if (ch == L'c') {AddCh(); goto case_22;} else {goto case_0;} case 22: case_22: if (ch == L'c') {AddCh(); goto case_23;} else {goto case_0;} case 23: case_23: if (ch == L'i') {AddCh(); goto case_24;} else {goto case_0;} case 24: case_24: if (ch == L'o') {AddCh(); goto case_25;} else {goto case_0;} case 25: case_25: if (ch == L'n') {AddCh(); goto case_26;} else {goto case_0;} case 26: case_26: {t->kind = 7; break;} case 27: case_27: if (ch == L'e') {AddCh(); goto case_28;} else {goto case_0;} case 28: case_28: if (ch == L'c') {AddCh(); goto case_29;} else {goto case_0;} case 29: case_29: if (ch == L't') {AddCh(); goto case_30;} else {goto case_0;} case 30: case_30: if (ch == L'u') {AddCh(); goto case_31;} else {goto case_0;} case 31: case_31: if (ch == L'r') {AddCh(); goto case_32;} else {goto case_0;} case 32: case_32: if (ch == L'a') {AddCh(); goto case_33;} else {goto case_0;} case 33: case_33: {t->kind = 8; break;} case 34: case_34: if (ch == L'c') {AddCh(); goto case_35;} else {goto case_0;} case 35: case_35: if (ch == L'e') {AddCh(); goto case_36;} else {goto case_0;} case 36: case_36: if (ch == L'n') {AddCh(); goto case_37;} else {goto case_0;} case 37: case_37: if (ch == L'd') {AddCh(); goto case_38;} else {goto case_0;} case 38: case_38: if (ch == L'i') {AddCh(); goto case_39;} else {goto case_0;} case 39: case_39: if (ch == L'd') {AddCh(); goto case_40;} else {goto case_0;} case 40: case_40: if (ch == L'o') {AddCh(); goto case_41;} else {goto case_0;} case 41: case_41: {t->kind = 9; break;} case 42: case_42: if (ch == L'm') {AddCh(); goto case_43;} else {goto case_0;} case 43: case_43: if (ch == L'p') {AddCh(); goto case_44;} else {goto case_0;} case 44: case_44: if (ch == L'r') {AddCh(); goto case_45;} else {goto case_0;} case 45: case_45: if (ch == L'i') {AddCh(); goto case_46;} else {goto case_0;} case 46: case_46: if (ch == L'm') {AddCh(); goto case_47;} else {goto case_0;} case 47: case_47: if (ch == L'e') {AddCh(); goto case_48;} else {goto case_0;} case 48: case_48: {t->kind = 10; break;} case 49: case_49: if (ch == L'n') {AddCh(); goto case_50;} else {goto case_0;} case 50: case_50: if (ch == L'g') {AddCh(); goto case_51;} else {goto case_0;} case 51: case_51: if (ch == L'u') {AddCh(); goto case_52;} else {goto case_0;} case 52: case_52: if (ch == L'l') {AddCh(); goto case_53;} else {goto case_0;} case 53: case_53: if (ch == L'o') {AddCh(); goto case_54;} else {goto case_0;} case 54: case_54: {t->kind = 11; break;} case 55: case_55: if (ch == L't') {AddCh(); goto case_56;} else {goto case_0;} case 56: case_56: if (ch == L'a') {AddCh(); goto case_57;} else {goto case_0;} case 57: case_57: if (ch == L'd') {AddCh(); goto case_58;} else {goto case_0;} case 58: case_58: if (ch == L'o') {AddCh(); goto case_59;} else {goto case_0;} case 59: case_59: {t->kind = 12; break;} case 60: case_60: recEnd = pos; recKind = 13; if ((ch >= L'0' && ch <= L'9')) {AddCh(); goto case_60;} else {t->kind = 13; break;} case 61: case_61: recEnd = pos; recKind = 2; if ((ch >= L'0' && ch <= L'9')) {AddCh(); goto case_61;} else if (ch == L'.') {AddCh(); goto case_2;} else {t->kind = 2; break;} case 62: if (ch == L'v') {AddCh(); goto case_10;} else if (ch == L'd') {AddCh(); goto case_18;} else if (ch == L'l') {AddCh(); goto case_27;} else if (ch == L'e') {AddCh(); goto case_63;} else if (ch == L'i') {AddCh(); goto case_42;} else if (ch == L'a') {AddCh(); goto case_49;} else {goto case_0;} case 63: case_63: if (ch == L'n') {AddCh(); goto case_34;} else if (ch == L's') {AddCh(); goto case_55;} else {goto case_0;} case 64: if (ch == L'O') {AddCh(); goto case_65;} else {goto case_0;} case 65: case_65: if (ch == L'N') {AddCh(); goto case_66;} else {goto case_0;} case 66: case_66: if (ch == L'F') {AddCh(); goto case_67;} else {goto case_0;} case 67: case_67: if (ch == L'I') {AddCh(); goto case_68;} else {goto case_0;} case 68: case_68: if (ch == L'G') {AddCh(); goto case_69;} else {goto case_0;} case 69: case_69: {t->kind = 14; break;} case 70: {t->kind = 15; break;} case 71: {t->kind = 16; break;} case 72: case_72: {t->kind = 17; break;} case 73: if (ch == L'U') {AddCh(); goto case_74;} else {goto case_0;} case 74: case_74: if (ch == L'N') {AddCh(); goto case_75;} else {goto case_0;} case 75: case_75: if (ch == L'C') {AddCh(); goto case_76;} else {goto case_0;} case 76: case_76: if (ch == L'I') {AddCh(); goto case_77;} else {goto case_0;} case 77: case_77: if (ch == L'O') {AddCh(); goto case_78;} else {goto case_0;} case 78: case_78: if (ch == L'N') {AddCh(); goto case_79;} else {goto case_0;} case 79: case_79: if (ch == L'E') {AddCh(); goto case_80;} else {goto case_0;} case 80: case_80: if (ch == L'S') {AddCh(); goto case_81;} else {goto case_0;} case 81: case_81: {t->kind = 18; break;} case 82: if (ch == L'R') {AddCh(); goto case_83;} else {goto case_0;} case 83: case_83: if (ch == L'O') {AddCh(); goto case_84;} else {goto case_0;} case 84: case_84: if (ch == L'G') {AddCh(); goto case_85;} else {goto case_0;} case 85: case_85: if (ch == L'R') {AddCh(); goto case_86;} else {goto case_0;} case 86: case_86: if (ch == L'A') {AddCh(); goto case_87;} else {goto case_0;} case 87: case_87: if (ch == L'M') {AddCh(); goto case_88;} else {goto case_0;} case 88: case_88: if (ch == L'A') {AddCh(); goto case_89;} else {goto case_0;} case 89: case_89: {t->kind = 19; break;} case 90: {t->kind = 26; break;} case 91: {t->kind = 34; break;} case 92: {t->kind = 35; break;} case 93: case_93: {t->kind = 36; break;} case 94: {t->kind = 37; break;} case 95: case_95: {t->kind = 49; break;} case 96: case_96: {t->kind = 52; break;} case 97: case_97: {t->kind = 53; break;} case 98: if (ch == L'=') {AddCh(); goto case_99;} else {goto case_0;} case 99: case_99: {t->kind = 54; break;} case 100: {t->kind = 55; break;} case 101: {t->kind = 56; break;} case 102: {t->kind = 57; break;} case 103: {t->kind = 58; break;} case 104: {t->kind = 59; break;} case 105: {t->kind = 60; break;} case 106: if (ch == L'A') {AddCh(); goto case_109;} else {goto case_0;} case 107: recEnd = pos; recKind = 51; if (ch == L'-') {AddCh(); goto case_95;} else if (ch == L'=') {AddCh(); goto case_96;} else {t->kind = 51; break;} case 108: recEnd = pos; recKind = 50; if (ch == L'=') {AddCh(); goto case_97;} else {t->kind = 50; break;} case 109: case_109: if (ch == L'R') {AddCh(); goto case_110;} else {goto case_0;} case 110: case_110: if (ch == L'G') {AddCh(); goto case_72;} else if (ch == L'F') {AddCh(); goto case_93;} else {goto case_0;} } AppendVal(t); return t; }
void Scanner::Init() { EOL = '\n'; eofSym = 0; maxT = 109; noSym = 109; int i; for (i = 65; i <= 81; ++i) start.set(i, 1); for (i = 83; i <= 90; ++i) start.set(i, 1); for (i = 95; i <= 95; ++i) start.set(i, 1); for (i = 97; i <= 122; ++i) start.set(i, 1); start.set(48, 67); for (i = 49; i <= 57; ++i) start.set(i, 68); start.set(46, 69); start.set(39, 17); start.set(34, 20); start.set(44, 22); start.set(59, 23); start.set(58, 24); start.set(42, 25); start.set(40, 26); start.set(41, 27); start.set(91, 28); start.set(93, 29); start.set(123, 30); start.set(125, 31); start.set(35, 70); start.set(82, 80); start.set(61, 77); start.set(60, 78); start.set(62, 79); start.set(Buffer::EoF, -1); keywords.set(L"auto", 6); keywords.set(L"case", 7); keywords.set(L"cbuffer", 8); keywords.set(L"char", 9); keywords.set(L"const", 10); keywords.set(L"default", 11); keywords.set(L"double", 12); keywords.set(L"enum", 13); keywords.set(L"extern", 14); keywords.set(L"register", 15); keywords.set(L"short", 16); keywords.set(L"signed", 17); keywords.set(L"static", 18); keywords.set(L"struct", 19); keywords.set(L"typedef", 20); keywords.set(L"union", 21); keywords.set(L"unsigned", 22); keywords.set(L"void", 23); keywords.set(L"volatile", 24); keywords.set(L"HS", 36); keywords.set(L"DS", 37); keywords.set(L"TS", 38); keywords.set(L"VS", 39); keywords.set(L"GS", 40); keywords.set(L"PS", 41); keywords.set(L"RenderTarget", 42); keywords.set(L"SizeX", 43); keywords.set(L"ScaleX", 44); keywords.set(L"SizeY", 45); keywords.set(L"ScaleY", 46); keywords.set(L"Format", 47); keywords.set(L"Info", 48); keywords.set(L"MultiRenderTarget", 49); keywords.set(L"ClearColor", 50); keywords.set(L"SamplerState", 51); keywords.set(L"SamplerComparisonState", 52); keywords.set(L"Filter", 53); keywords.set(L"AddressU", 54); keywords.set(L"AddressV", 55); keywords.set(L"AddressW", 56); keywords.set(L"MipLODBias", 57); keywords.set(L"MaxAnisotropy", 58); keywords.set(L"ComparisonFunc", 59); keywords.set(L"BorderColor", 60); keywords.set(L"MinLOD", 61); keywords.set(L"MaxLOD", 62); keywords.set(L"DepthStencilState", 63); keywords.set(L"DepthEnable", 64); keywords.set(L"DepthWriteMask", 65); keywords.set(L"DepthFunc", 66); keywords.set(L"StencilEnable", 67); keywords.set(L"RasterizerState", 68); keywords.set(L"FillMode", 69); keywords.set(L"CullMode", 70); keywords.set(L"FrontCounterClockwise", 71); keywords.set(L"DepthBias", 72); keywords.set(L"DepthBiasClamp", 73); keywords.set(L"SlopeScaledDepthBias", 74); keywords.set(L"DepthClipEnable", 75); keywords.set(L"ScissorEnable", 76); keywords.set(L"MultisampleEnable", 77); keywords.set(L"AntialiasedLineEnable", 78); keywords.set(L"BlendState", 79); keywords.set(L"BlendEnable", 80); keywords.set(L"AlphaToCoverageEnable", 81); keywords.set(L"SrcBlend", 82); keywords.set(L"DestBlend", 83); keywords.set(L"BlendOp", 84); keywords.set(L"SrcBlendAlpha", 85); keywords.set(L"DestBlendAlpha", 86); keywords.set(L"BlendOpAlpha", 87); keywords.set(L"RenderTargetWriteMask", 88); keywords.set(L"StateBlock", 89); keywords.set(L"StencilRef", 90); keywords.set(L"BlendFactor", 91); keywords.set(L"SampleMask", 92); keywords.set(L"Shader", 94); keywords.set(L"Code", 95); keywords.set(L"VertexShader", 97); keywords.set(L"PixelShader", 98); keywords.set(L"Inputs", 101); keywords.set(L"Texture2D", 102); keywords.set(L"Vertex", 103); keywords.set(L"SemanticName", 104); keywords.set(L"SemanticIndex", 105); keywords.set(L"InputSlot", 106); keywords.set(L"Shared", 107); keywords.set(L"Variables", 108); tvalLength = 128; tval = new wchar_t[tvalLength]; // text of current token // HEAP_BLOCK_SIZE byte heap + pointer to next heap block heap = mxAlloc(HEAP_BLOCK_SIZE + sizeof(void*)); firstHeap = heap; heapEnd = (void**) (((char*) heap) + HEAP_BLOCK_SIZE); *heapEnd = 0; heapTop = heap; if (sizeof(Token) > HEAP_BLOCK_SIZE) { wprintf(L"--- Too small HEAP_BLOCK_SIZE\n"); MX_DEBUG_BREAK; exit(1); } pos = -1; line = 1; col = 0; charPos = -1; oldEols = 0; NextCh(); if (ch == 0xEF) { // check optional byte order mark for UTF-8 NextCh(); int ch1 = ch; NextCh(); int ch2 = ch; if (ch1 != 0xBB || ch2 != 0xBF) { wprintf(L"Illegal byte order mark at start of file"); MX_DEBUG_BREAK; exit(1); } Buffer *oldBuf = buffer; buffer = new UTF8Buffer(buffer); col = 0; charPos = -1; delete oldBuf; oldBuf = NULL; NextCh(); } pt = tokens = CreateToken(); // first token is a dummy }
Token* Scanner::NextToken() { while (ch == ' ' || (ch >= 9 && ch <= 10) || ch == 13 ) NextCh(); if ((ch == L'/' && Comment0()) || (ch == L'/' && Comment1())) return NextToken(); int recKind = noSym; int recEnd = pos; t = CreateToken(); t->pos = pos; t->col = col; t->line = line; t->charPos = charPos; int state = start.state(ch); tlen = 0; AddCh(); switch (state) { case -1: { t->kind = eofSym; break; } // NextCh already done case 0: { case_0: if (recKind != noSym) { tlen = recEnd - t->pos; SetScannerBehindT(); } t->kind = recKind; break; } // NextCh already done case 1: case_1: recEnd = pos; recKind = 1; if ((ch >= L'0' && ch <= L'9') || (ch >= L'A' && ch <= L'Z') || ch == L'_' || (ch >= L'a' && ch <= L'z')) {AddCh(); goto case_1;} else {t->kind = 1; wchar_t *literal = coco_string_create(tval, 0, tlen); t->kind = keywords.get(literal, t->kind); coco_string_delete(literal); break;} case 2: case_2: recEnd = pos; recKind = 2; if ((ch >= L'0' && ch <= L'9')) {AddCh(); goto case_2;} else {t->kind = 2; break;} case 3: case_3: {t->kind = 3; break;} case 4: case_4: {t->kind = 4; break;} case 5: if (ch <= 9 || (ch >= 11 && ch <= 12) || (ch >= 14 && ch <= L'&') || (ch >= L'(' && ch <= L'[') || (ch >= L']' && ch <= 65535)) {AddCh(); goto case_6;} else if (ch == 92) {AddCh(); goto case_7;} else {goto case_0;} case 6: case_6: if (ch == 39) {AddCh(); goto case_9;} else {goto case_0;} case 7: case_7: if ((ch >= L' ' && ch <= L'~')) {AddCh(); goto case_8;} else {goto case_0;} case 8: case_8: if ((ch >= L'0' && ch <= L'9') || (ch >= L'a' && ch <= L'f')) {AddCh(); goto case_8;} else if (ch == 39) {AddCh(); goto case_9;} else {goto case_0;} case 9: case_9: {t->kind = 5; break;} case 10: case_10: recEnd = pos; recKind = 42; if ((ch >= L'0' && ch <= L'9') || (ch >= L'A' && ch <= L'Z') || ch == L'_' || (ch >= L'a' && ch <= L'z')) {AddCh(); goto case_10;} else {t->kind = 42; break;} case 11: case_11: recEnd = pos; recKind = 43; if ((ch >= L'-' && ch <= L'.') || (ch >= L'0' && ch <= L':') || (ch >= L'A' && ch <= L'Z') || ch == L'_' || (ch >= L'a' && ch <= L'z')) {AddCh(); goto case_11;} else {t->kind = 43; break;} case 12: case_12: if (ch <= 9 || (ch >= 11 && ch <= 12) || (ch >= 14 && ch <= L'!') || (ch >= L'#' && ch <= L'[') || (ch >= L']' && ch <= 65535)) {AddCh(); goto case_12;} else if (ch == 10 || ch == 13) {AddCh(); goto case_4;} else if (ch == L'"') {AddCh(); goto case_3;} else if (ch == 92) {AddCh(); goto case_14;} else {goto case_0;} case 13: recEnd = pos; recKind = 42; if ((ch >= L'0' && ch <= L'9')) {AddCh(); goto case_10;} else if ((ch >= L'A' && ch <= L'Z') || ch == L'_' || (ch >= L'a' && ch <= L'z')) {AddCh(); goto case_15;} else {t->kind = 42; break;} case 14: case_14: if ((ch >= L' ' && ch <= L'~')) {AddCh(); goto case_12;} else {goto case_0;} case 15: case_15: recEnd = pos; recKind = 42; if ((ch >= L'0' && ch <= L'9')) {AddCh(); goto case_10;} else if ((ch >= L'A' && ch <= L'Z') || ch == L'_' || (ch >= L'a' && ch <= L'z')) {AddCh(); goto case_15;} else if (ch == L'=') {AddCh(); goto case_11;} else {t->kind = 42; break;} case 16: {t->kind = 17; break;} case 17: {t->kind = 20; break;} case 18: {t->kind = 21; break;} case 19: case_19: {t->kind = 22; break;} case 20: {t->kind = 25; break;} case 21: case_21: {t->kind = 26; break;} case 22: case_22: {t->kind = 27; break;} case 23: {t->kind = 28; break;} case 24: {t->kind = 31; break;} case 25: {t->kind = 32; break;} case 26: {t->kind = 33; break;} case 27: {t->kind = 34; break;} case 28: {t->kind = 35; break;} case 29: case_29: {t->kind = 39; break;} case 30: case_30: {t->kind = 40; break;} case 31: recEnd = pos; recKind = 18; if (ch == L'.') {AddCh(); goto case_19;} else if (ch == L'>') {AddCh(); goto case_22;} else if (ch == L')') {AddCh(); goto case_30;} else {t->kind = 18; break;} case 32: recEnd = pos; recKind = 24; if (ch == L'.') {AddCh(); goto case_21;} else {t->kind = 24; break;} case 33: recEnd = pos; recKind = 30; if (ch == L'.') {AddCh(); goto case_29;} else {t->kind = 30; break;} } AppendVal(t); return t; }
void Scanner::Init() { EOL = '\n'; eofSym = 0; maxT = 41; noSym = 41; int i; for (i = 65; i <= 90; ++i) start.set(i, 1); for (i = 95; i <= 95; ++i) start.set(i, 1); for (i = 97; i <= 122; ++i) start.set(i, 1); for (i = 48; i <= 57; ++i) start.set(i, 2); start.set(34, 12); start.set(39, 5); start.set(36, 13); start.set(61, 16); start.set(46, 31); start.set(43, 17); start.set(45, 18); start.set(60, 32); start.set(62, 20); start.set(124, 23); start.set(40, 33); start.set(41, 24); start.set(91, 25); start.set(93, 26); start.set(123, 27); start.set(125, 28); start.set(Buffer::EoF, -1); keywords.set(L"COMPILER", 6); keywords.set(L"IGNORECASE", 7); keywords.set(L"CHARACTERS", 8); keywords.set(L"TOKENS", 9); keywords.set(L"PRAGMAS", 10); keywords.set(L"COMMENTS", 11); keywords.set(L"FROM", 12); keywords.set(L"TO", 13); keywords.set(L"NESTED", 14); keywords.set(L"IGNORE", 15); keywords.set(L"PRODUCTIONS", 16); keywords.set(L"END", 19); keywords.set(L"ANY", 23); keywords.set(L"WEAK", 29); keywords.set(L"SYNC", 36); keywords.set(L"IF", 37); keywords.set(L"CONTEXT", 38); tvalLength = 128; tval = new wchar_t[tvalLength]; // text of current token // COCO_HEAP_BLOCK_SIZE byte heap + pointer to next heap block heap = malloc(COCO_HEAP_BLOCK_SIZE + sizeof(void*)); firstHeap = heap; heapEnd = (void**) (((char*) heap) + COCO_HEAP_BLOCK_SIZE); *heapEnd = 0; heapTop = heap; if (sizeof(Token) > COCO_HEAP_BLOCK_SIZE) { wprintf(L"--- Too small COCO_HEAP_BLOCK_SIZE\n"); exit(1); } pos = -1; line = 1; col = 0; charPos = -1; oldEols = 0; NextCh(); if (ch == 0xEF) { // check optional byte order mark for UTF-8 NextCh(); int ch1 = ch; NextCh(); int ch2 = ch; if (ch1 != 0xBB || ch2 != 0xBF) { wprintf(L"Illegal byte order mark at start of file"); exit(1); } Buffer *oldBuf = buffer; buffer = new UTF8Buffer(buffer); col = 0; charPos = -1; delete oldBuf; oldBuf = NULL; NextCh(); } pt = tokens = CreateToken(); // first token is a dummy }