static void parse_dc_deviceid(char *line, struct membuffer *str, void *_dc) { struct divecomputer *dc = _dc; dc->deviceid = get_hex(line); }
int main() { /* __xdata uint8_t *xp; __sfr *sp; */ init_serial_1(); printf("Started\n"); while(1) { do { char c; skip_white(); c = getchar(); if (c == 'r') { __data uint8_t *p; uint16_t l; p = (__data uint8_t*)get_hex(); if (skip_space()) break; l = get_hex(); while(l > 0) { printf(" %02x", *p); p++; l--; } } else if (c == 'i') { uint8_t p; uint8_t v = 0; if (skip_space()) break; p = get_hex(); switch(p) { case 0: v = P0; break; case 1: v = P1; break; case 2: v = P2; break; case 3: v = P3; break; case 4: v = P4; break; case 5: v = P5; break; case 6: v = P6; break; case 7: v = P7; break; case 8: v = P8; break; } printf(" %02x", v); } else if (c == 'o') { uint8_t p; uint8_t v = 0; if (skip_space()) break; p = get_hex(); if (skip_space()) break; v = get_hex(); switch(p) { case 0: P0 = v; break; case 1: P1 = v; break; case 2: P2 = v; break; case 3: P3 = v; break; case 4: P4 = v; break; case 5: P5 = v; break; case 6: P6 = v; break; } } else if (c == 'R') { __xdata uint8_t *p; uint16_t l; p = (__xdata uint8_t*)get_hex(); if (skip_space()) break; l = get_hex(); while(l > 0) { printf(" %02x", *p); p++; l--; } } else if (c == 'W') { __xdata uint8_t *p; uint8_t v; p = (__xdata uint8_t*)get_hex(); while(1) { if (skip_space()) break; v = get_hex(); *p++ = v; } } else if (c == 'c') { /* Call a subroutine */ uint16_t a; a = get_hex(); ((GenericCall)a)(); } } while(0); skip_to_eol(); putchar('\n'); putchar('>'); } }
static void parse_dive_divesiteid(char *line, struct membuffer *str, void *_dive) { struct dive *dive = _dive; dive->dive_site_uuid = get_hex(line); }
javacall_result bt_push_parse_url(const char *url, bt_port_t *port, bt_params_t *params) { int i; if (test_prefix(&url, "btl2cap://")) { port->protocol = BT_L2CAP; } else if (test_prefix(&url, "btspp://")) { port->protocol = BT_SPP; } else if (test_prefix(&url, "btgoep://")) { port->protocol = BT_GOEP; } else { return JAVACALL_FAIL; } if (!test_prefix(&url, "localhost:")) { return JAVACALL_FAIL; } for (i = 0; i < 16; i++) { int hex1, hex2; hex1 = get_hex(*url++); if (hex1 < 0) { return JAVACALL_FAIL; } hex2 = get_hex(*url++); if (hex2 < 0) { return JAVACALL_FAIL; } port->uuid[i] = hex1 << 4 | hex2; } if (params == NULL) { /* params parsing is not needed */ return JAVACALL_OK; } params->authenticate = JAVACALL_FALSE; params->authorize = JAVACALL_FALSE; params->encrypt = JAVACALL_FALSE; params->rmtu = DEFAULT_MTU; params->tmtu = -1; params->authenticate = 0; params->authorize = 0; params->encrypt = 0; params->master = 0; url = strchr(url, ';'); while (url != NULL) { if (test_prefix(&url, ";authenticate=")) { params->authenticate = read_bool(&url); } else if (test_prefix(&url, ";authorize=")) { params->authorize = read_bool(&url); } else if (test_prefix(&url, ";encrypt=")) { params->encrypt = read_bool(&url); } else if (test_prefix(&url, ";master=")) { params->master = read_bool(&url); } else if (test_prefix(&url, ";receiveMTU=")) { params->rmtu = read_short(&url); } else if (test_prefix(&url, ";transmitMTU=")) { params->tmtu = read_short(&url); } url = strchr(url + 1, ';'); } if (params->authorize || params->encrypt) { params->authenticate = JAVACALL_TRUE; } return JAVACALL_OK; }
STATIC void parse_string_literal(mp_lexer_t *lex, bool is_raw) { // get first quoting character char quote_char = '\''; if (is_char(lex, '\"')) { quote_char = '\"'; } next_char(lex); // work out if it's a single or triple quoted literal size_t num_quotes; if (is_char_and(lex, quote_char, quote_char)) { // triple quotes next_char(lex); next_char(lex); num_quotes = 3; } else { // single quotes num_quotes = 1; } size_t n_closing = 0; while (!is_end(lex) && (num_quotes > 1 || !is_char(lex, '\n')) && n_closing < num_quotes) { if (is_char(lex, quote_char)) { n_closing += 1; vstr_add_char(&lex->vstr, CUR_CHAR(lex)); } else { n_closing = 0; if (is_char(lex, '\\')) { next_char(lex); unichar c = CUR_CHAR(lex); if (is_raw) { // raw strings allow escaping of quotes, but the backslash is also emitted vstr_add_char(&lex->vstr, '\\'); } else { switch (c) { // note: "c" can never be MP_LEXER_EOF because next_char // always inserts a newline at the end of the input stream case '\n': c = MP_LEXER_EOF; break; // backslash escape the newline, just ignore it case '\\': break; case '\'': break; case '"': break; case 'a': c = 0x07; break; case 'b': c = 0x08; break; case 't': c = 0x09; break; case 'n': c = 0x0a; break; case 'v': c = 0x0b; break; case 'f': c = 0x0c; break; case 'r': c = 0x0d; break; case 'u': case 'U': if (lex->tok_kind == MP_TOKEN_BYTES) { // b'\u1234' == b'\\u1234' vstr_add_char(&lex->vstr, '\\'); break; } // Otherwise fall through. case 'x': { mp_uint_t num = 0; if (!get_hex(lex, (c == 'x' ? 2 : c == 'u' ? 4 : 8), &num)) { // not enough hex chars for escape sequence lex->tok_kind = MP_TOKEN_INVALID; } c = num; break; } case 'N': // Supporting '\N{LATIN SMALL LETTER A}' == 'a' would require keeping the // entire Unicode name table in the core. As of Unicode 6.3.0, that's nearly // 3MB of text; even gzip-compressed and with minimal structure, it'll take // roughly half a meg of storage. This form of Unicode escape may be added // later on, but it's definitely not a priority right now. -- CJA 20140607 mp_not_implemented("unicode name escapes"); break; default: if (c >= '0' && c <= '7') { // Octal sequence, 1-3 chars size_t digits = 3; mp_uint_t num = c - '0'; while (is_following_odigit(lex) && --digits != 0) { next_char(lex); num = num * 8 + (CUR_CHAR(lex) - '0'); } c = num; } else { // unrecognised escape character; CPython lets this through verbatim as '\' and then the character vstr_add_char(&lex->vstr, '\\'); } break; } } if (c != MP_LEXER_EOF) { if (MICROPY_PY_BUILTINS_STR_UNICODE_DYNAMIC) { if (c < 0x110000 && lex->tok_kind == MP_TOKEN_STRING) { vstr_add_char(&lex->vstr, c); } else if (c < 0x100 && lex->tok_kind == MP_TOKEN_BYTES) { vstr_add_byte(&lex->vstr, c); } else { // unicode character out of range // this raises a generic SyntaxError; could provide more info lex->tok_kind = MP_TOKEN_INVALID; } } else { // without unicode everything is just added as an 8-bit byte if (c < 0x100) { vstr_add_byte(&lex->vstr, c); } else { // 8-bit character out of range // this raises a generic SyntaxError; could provide more info lex->tok_kind = MP_TOKEN_INVALID; } } } } else { // Add the "character" as a byte so that we remain 8-bit clean. // This way, strings are parsed correctly whether or not they contain utf-8 chars. vstr_add_byte(&lex->vstr, CUR_CHAR(lex)); } } next_char(lex); } // check we got the required end quotes if (n_closing < num_quotes) { lex->tok_kind = MP_TOKEN_LONELY_STRING_OPEN; } // cut off the end quotes from the token text vstr_cut_tail_bytes(&lex->vstr, n_closing); }
Type Lexer::get_token(char &c, std::string &str) { str = ""; if(is_whitespace(c)) { return get_whitespace(c, str); } if(c == '0') { str += c; c = fgetc(input_file); if(c == 'x') { str += c; c = fgetc(input_file); return get_hex(c, str); } while(is_digit(c)) { str += c; c = fgetc(input_file); } if(is_hex(c)) { return get_hex(c, str); } else { return DEC; } } if(is_digit(c)) { while(is_digit(c)) { str += c; c = fgetc(input_file); } if(is_hex(c)) { return get_hex(c, str); } else { return DEC; } } if(is_alpha(c) || c == '.' || c == '_') { uint64_t hash = 0; if(is_hex(c)) { while(is_hex(c)) { hash <<= 8; hash += c; str += c; c = fgetc(input_file); } if(is_alpha(c) || is_digit(c) || c == '.' || c =='_') { while(is_alpha(c) || is_digit(c) || c == '.' || c =='_') { hash <<= 8; hash += c; str += c; c = fgetc(input_file); } return get_type_by_hash(hash); } else if(instruction_set.find(hash) != instruction_set.end()) return HEX_OR_INSTRUCTION; else return HEX; } else { while(is_alpha(c) || is_digit(c) || c == '.' || c =='_') { hash <<= 8; hash += c; str += c; c = fgetc(input_file); } return get_type_by_hash(hash); } } str += c; if(c == '/') { c = fgetc(input_file); str += c; if(c == '/') { c = fgetc(input_file); return DOUBLE_SLASH; } else return BAD_TOKEN; } char s = c; c = fgetc(input_file); switch(s) { case '!': return EXCLAMATION; case '\n': return NEWLINE; case '#': return HASH; case ':': return COLON; case ';': return SEMICOLON; case '[': return LEFT_SQ_BRACKET; case ']': return RIGHT_SQ_BRACKET; case ',': return COMMA; case '@': return AT; case '+': return PLUS; case '-': return MINUS; case '<': return LESS_THAN; case '>': return MORE_THAN; default: return BAD_TOKEN; } }
void AWorldSpawn::gen_hex(int x, int y, int z) { Hex* hex = get_hex(x, y, z); if (hex) hex->type = get_type(x, y, z); }
void AWorldSpawn::gen_chunk() { if (hex_asset) { for (int z = 0; z < SIZEZ; ++z) { for (int y = 0; y < SIZEY; ++y) { for (int x = 0; x < SIZEX; ++x) { Hex* hex = new Hex(); hex->x = x; hex->y = y; hex->z = z; hex_list.push_back(hex); gen_hex(x, y, z); } } } for (int z = -1; z < SIZEZ; ++z) { for (int y = -1; y < SIZEY; ++y) { for (int x = -1; x < SIZEX; ++x) { bool corner0 = (get_hex(x + 1, y, z + 1) != NULL && get_hex(x + 1, y, z + 1)->type != BLOCK_TYPE_AIR); bool corner1 = (get_hex(x + 1, y + 1, z + 1) != NULL && get_hex(x + 1, y + 1, z + 1)->type != BLOCK_TYPE_AIR); bool corner2 = (get_hex(x, y + 1, z + 1) != NULL && get_hex(x, y + 1, z + 1)->type != BLOCK_TYPE_AIR); bool corner3 = (get_hex(x, y, z + 1) != NULL && get_hex(x, y, z + 1)->type != BLOCK_TYPE_AIR); bool corner4 = (get_hex(x + 1, y, z) != NULL && get_hex(x + 1, y, z)->type != BLOCK_TYPE_AIR); bool corner5 = (get_hex(x + 1, y + 1, z) != NULL && get_hex(x + 1, y + 1, z)->type != BLOCK_TYPE_AIR); bool corner6 = (get_hex(x, y + 1, z) != NULL && get_hex(x, y + 1, z)->type != BLOCK_TYPE_AIR); bool corner7 = (get_hex(x, y, z) != NULL && get_hex(x, y, z)->type != BLOCK_TYPE_AIR); //UE_LOG(LogTemp, Warning, TEXT("(%d, %d, %d): %d, %d, %d, %d, %d, %d, %d, %d"), x, y, z, corner0, corner1, corner2, corner3, corner4, corner5, corner6, corner7); FVector vertlist[12]; //bool isolevel = corner0 && corner1 && corner2 && corner3 && corner4 && corner5 && corner6 && corner7; //isolevel = false; float isolevel = 1; int cubeindex = 0; if (corner0) cubeindex |= 1; if (corner1) cubeindex |= 2; if (corner2) cubeindex |= 4; if (corner3) cubeindex |= 8; if (corner4) cubeindex |= 16; if (corner5) cubeindex |= 32; if (corner6) cubeindex |= 64; if (corner7) cubeindex |= 128; //UE_LOG(LogTemp, Warning, TEXT("%d, %d"), cubeindex, isolevel); /* Cube is entirely in/out of the surface */ if (edgeTable[cubeindex] == 0) continue; /* Find the vertices where the surface intersects the cube */ if (edgeTable[cubeindex] & 1) vertlist[0] = vertex_interp(corner_points[0], corner_points[1], corner0, corner1); if (edgeTable[cubeindex] & 2) vertlist[1] = vertex_interp(corner_points[1], corner_points[2], corner1, corner2); if (edgeTable[cubeindex] & 4) vertlist[2] = vertex_interp(corner_points[2], corner_points[3], corner2, corner3); if (edgeTable[cubeindex] & 8) vertlist[3] = vertex_interp(corner_points[3], corner_points[0], corner3, corner0); if (edgeTable[cubeindex] & 16) vertlist[4] = vertex_interp(corner_points[4], corner_points[5], corner4, corner5); if (edgeTable[cubeindex] & 32) vertlist[5] = vertex_interp(corner_points[5], corner_points[6], corner5, corner6); if (edgeTable[cubeindex] & 64) vertlist[6] = vertex_interp(corner_points[6], corner_points[7], corner6, corner7); if (edgeTable[cubeindex] & 128) vertlist[7] = vertex_interp(corner_points[7], corner_points[4], corner7, corner4); if (edgeTable[cubeindex] & 256) vertlist[8] = vertex_interp(corner_points[0], corner_points[4], corner0, corner4); if (edgeTable[cubeindex] & 512) vertlist[9] = vertex_interp(corner_points[1], corner_points[5], corner1, corner5); if (edgeTable[cubeindex] & 1024) vertlist[10] = vertex_interp(corner_points[2], corner_points[6], corner2, corner6); if (edgeTable[cubeindex] & 2048) vertlist[11] = vertex_interp(corner_points[3], corner_points[7], corner3, corner7); /* Create the triangle */ int ntriang = 0; FGeneratedMeshTriangle tri; for (int i = 0; triTable[cubeindex][i] != -1; i += 3) { tri.set_vertex(vertlist[triTable[cubeindex][i]] * 100.0f, 2); tri.set_vertex(vertlist[triTable[cubeindex][i + 1]] * 100.0f, 1); tri.set_vertex(vertlist[triTable[cubeindex][i + 2]] * 100.0f, 0); tri.Vertex0.X += x * 100.0f; tri.Vertex1.X += x * 100.0f; tri.Vertex2.X += x * 100.0f; tri.Vertex0.Y += y * 100.0f; tri.Vertex1.Y += y * 100.0f; tri.Vertex2.Y += y * 100.0f; tri.Vertex0.Z += z * 100.0f; tri.Vertex1.Z += z * 100.0f; tri.Vertex2.Z += z * 100.0f; chunk_triangles.Add(tri); //UE_LOG(LogTemp, Warning, TEXT("--tri--")); //UE_LOG(LogTemp, Warning, TEXT("%f, %f, %f"), tri.Vertex0.X, tri.Vertex0.Y, tri.Vertex0.Z); //UE_LOG(LogTemp, Warning, TEXT("%f, %f, %f"), tri.Vertex1.X, tri.Vertex1.Y, tri.Vertex1.Z); //UE_LOG(LogTemp, Warning, TEXT("%f, %f, %f"), tri.Vertex2.X, tri.Vertex2.Y, tri.Vertex2.Z); ntriang++; } } } } custom_mesh = NewObject<UGeneratedMeshComponent>(this); custom_mesh->RegisterComponent(); custom_mesh->SetGeneratedMeshTriangles(chunk_triangles); custom_mesh->SetMaterial(0, water_mat_asset); UE_LOG(LogTemp, Warning, TEXT("spawned")); }else { UE_LOG(LogTemp, Warning, TEXT("Could not find hexagon asset")); } }
/* returns the length of the reply string */ int parse_command (socket_t* socket, char* line) { unsigned int start_addr; unsigned int address; unsigned int range; int len, error = 0; /* All commands are just a single character. Just ignore anything else */ switch (line[0]) { /* Disconnect */ case 'e': case 'x': case 'q': socket->tcp_disconnect = 1; return 0; case 'r': /* Read mem */ { if (len = get_hex (&line[2], &start_addr)) { if (len = get_hex (&line[3+len], &range)) { for (address=start_addr; address<start_addr+range; address+=4) { put_line (socket->telnet_txbuf, "0x%08x 0x%08x\r\n", address, *(unsigned int *)address); } } else { put_line (socket->telnet_txbuf, "0x%08x 0x%08x\r\n", start_addr, *(unsigned int *)start_addr); } } else error=1; break; } case 'h': {/* Help */ put_line (socket->telnet_txbuf, "You need help alright\r\n"); break; } case 's': {/* Status */ put_line (socket->telnet_txbuf, "Socket ID %d\r\n", socket->id); put_line (socket->telnet_txbuf, "Packets received %d\r\n", socket->packets_received); put_line (socket->telnet_txbuf, "Packets transmitted %d\r\n", socket->packets_sent); put_line (socket->telnet_txbuf, "Packets resent %d\r\n", socket->packets_resent); put_line (socket->telnet_txbuf, "TCP checksum errors %d\r\n", tcp_checksum_errors_g); put_line (socket->telnet_txbuf, "Counterparty IP %d.%d.%d.%d\r\n", socket->rx_packet->src_ip[0], socket->rx_packet->src_ip[1], socket->rx_packet->src_ip[2], socket->rx_packet->src_ip[3]); put_line (socket->telnet_txbuf, "Counterparty Port %d\r\n", socket->rx_packet->tcp_src_port); put_line (socket->telnet_txbuf, "Malloc pointer 0x%08x\r\n", *(unsigned int *)(ADR_MALLOC_POINTER)); put_line (socket->telnet_txbuf, "Malloc count %d\r\n", *(unsigned int *)(ADR_MALLOC_COUNT)); put_line (socket->telnet_txbuf, "Uptime %d seconds\r\n", current_time_g->seconds); break; } default: { error=1; break; } } if (error) put_line (socket->telnet_txbuf, "You're not making any sense\r\n", line[0], line[1], line[2]); put_line (socket->telnet_txbuf, "> "); return 0; }
STATIC void mp_lexer_next_token_into(mp_lexer_t *lex, mp_token_t *tok, bool first_token) { // skip white space and comments bool had_physical_newline = false; while (!is_end(lex)) { if (is_physical_newline(lex)) { had_physical_newline = true; next_char(lex); } else if (is_whitespace(lex)) { next_char(lex); } else if (is_char(lex, '#')) { next_char(lex); while (!is_end(lex) && !is_physical_newline(lex)) { next_char(lex); } // had_physical_newline will be set on next loop } else if (is_char(lex, '\\')) { // backslash (outside string literals) must appear just before a physical newline next_char(lex); if (!is_physical_newline(lex)) { // SyntaxError: unexpected character after line continuation character tok->src_line = lex->line; tok->src_column = lex->column; tok->kind = MP_TOKEN_BAD_LINE_CONTINUATION; vstr_reset(&lex->vstr); tok->str = vstr_str(&lex->vstr); tok->len = 0; return; } else { next_char(lex); } } else { break; } } // set token source information tok->src_line = lex->line; tok->src_column = lex->column; // start new token text vstr_reset(&lex->vstr); if (first_token && lex->line == 1 && lex->column != 1) { // check that the first token is in the first column // if first token is not on first line, we get a physical newline and // this check is done as part of normal indent/dedent checking below // (done to get equivalence with CPython) tok->kind = MP_TOKEN_INDENT; } else if (lex->emit_dent < 0) { tok->kind = MP_TOKEN_DEDENT; lex->emit_dent += 1; } else if (lex->emit_dent > 0) { tok->kind = MP_TOKEN_INDENT; lex->emit_dent -= 1; } else if (had_physical_newline && lex->nested_bracket_level == 0) { tok->kind = MP_TOKEN_NEWLINE; uint num_spaces = lex->column - 1; lex->emit_dent = 0; if (num_spaces == indent_top(lex)) { } else if (num_spaces > indent_top(lex)) { indent_push(lex, num_spaces); lex->emit_dent += 1; } else { while (num_spaces < indent_top(lex)) { indent_pop(lex); lex->emit_dent -= 1; } if (num_spaces != indent_top(lex)) { tok->kind = MP_TOKEN_DEDENT_MISMATCH; } } } else if (is_end(lex)) { if (indent_top(lex) > 0) { tok->kind = MP_TOKEN_NEWLINE; lex->emit_dent = 0; while (indent_top(lex) > 0) { indent_pop(lex); lex->emit_dent -= 1; } } else { tok->kind = MP_TOKEN_END; } } else if (is_char_or(lex, '\'', '\"') || (is_char_or3(lex, 'r', 'u', 'b') && is_char_following_or(lex, '\'', '\"')) || ((is_char_and(lex, 'r', 'b') || is_char_and(lex, 'b', 'r')) && is_char_following_following_or(lex, '\'', '\"'))) { // a string or bytes literal // parse type codes bool is_raw = false; bool is_bytes = false; if (is_char(lex, 'u')) { next_char(lex); } else if (is_char(lex, 'b')) { is_bytes = true; next_char(lex); if (is_char(lex, 'r')) { is_raw = true; next_char(lex); } } else if (is_char(lex, 'r')) { is_raw = true; next_char(lex); if (is_char(lex, 'b')) { is_bytes = true; next_char(lex); } } // set token kind if (is_bytes) { tok->kind = MP_TOKEN_BYTES; } else { tok->kind = MP_TOKEN_STRING; } // get first quoting character char quote_char = '\''; if (is_char(lex, '\"')) { quote_char = '\"'; } next_char(lex); // work out if it's a single or triple quoted literal int num_quotes; if (is_char_and(lex, quote_char, quote_char)) { // triple quotes next_char(lex); next_char(lex); num_quotes = 3; } else { // single quotes num_quotes = 1; } // parse the literal int n_closing = 0; while (!is_end(lex) && (num_quotes > 1 || !is_char(lex, '\n')) && n_closing < num_quotes) { if (is_char(lex, quote_char)) { n_closing += 1; vstr_add_char(&lex->vstr, CUR_CHAR(lex)); } else { n_closing = 0; if (is_char(lex, '\\')) { next_char(lex); unichar c = CUR_CHAR(lex); if (is_raw) { // raw strings allow escaping of quotes, but the backslash is also emitted vstr_add_char(&lex->vstr, '\\'); } else { switch (c) { case MP_LEXER_CHAR_EOF: break; // TODO a proper error message? case '\n': c = MP_LEXER_CHAR_EOF; break; // TODO check this works correctly (we are supposed to ignore it case '\\': break; case '\'': break; case '"': break; case 'a': c = 0x07; break; case 'b': c = 0x08; break; case 't': c = 0x09; break; case 'n': c = 0x0a; break; case 'v': c = 0x0b; break; case 'f': c = 0x0c; break; case 'r': c = 0x0d; break; case 'x': { uint num = 0; if (!get_hex(lex, 2, &num)) { // TODO error message assert(0); } c = num; break; } case 'N': break; // TODO \N{name} only in strings case 'u': break; // TODO \uxxxx only in strings case 'U': break; // TODO \Uxxxxxxxx only in strings default: if (c >= '0' && c <= '7') { // Octal sequence, 1-3 chars int digits = 3; int num = c - '0'; while (is_following_odigit(lex) && --digits != 0) { next_char(lex); num = num * 8 + (CUR_CHAR(lex) - '0'); } c = num; } else { // unrecognised escape character; CPython lets this through verbatim as '\' and then the character vstr_add_char(&lex->vstr, '\\'); } break; } } if (c != MP_LEXER_CHAR_EOF) { vstr_add_char(&lex->vstr, c); } } else { vstr_add_char(&lex->vstr, CUR_CHAR(lex)); } } next_char(lex); } // check we got the required end quotes if (n_closing < num_quotes) { tok->kind = MP_TOKEN_LONELY_STRING_OPEN; } // cut off the end quotes from the token text vstr_cut_tail_bytes(&lex->vstr, n_closing); } else if (is_head_of_identifier(lex)) { tok->kind = MP_TOKEN_NAME; // get first char vstr_add_char(&lex->vstr, CUR_CHAR(lex)); next_char(lex); // get tail chars while (!is_end(lex) && is_tail_of_identifier(lex)) { vstr_add_char(&lex->vstr, CUR_CHAR(lex)); next_char(lex); } } else if (is_digit(lex) || (is_char(lex, '.') && is_following_digit(lex))) { tok->kind = MP_TOKEN_NUMBER; // get first char vstr_add_char(&lex->vstr, CUR_CHAR(lex)); next_char(lex); // get tail chars while (!is_end(lex)) { if (is_char_or(lex, 'e', 'E')) { vstr_add_char(&lex->vstr, 'e'); next_char(lex); if (is_char(lex, '+') || is_char(lex, '-')) { vstr_add_char(&lex->vstr, CUR_CHAR(lex)); next_char(lex); } } else if (is_letter(lex) || is_digit(lex) || is_char_or(lex, '_', '.')) { vstr_add_char(&lex->vstr, CUR_CHAR(lex)); next_char(lex); } else { break; } } } else if (is_char(lex, '.')) { // special handling for . and ... operators, because .. is not a valid operator // get first char vstr_add_char(&lex->vstr, '.'); next_char(lex); if (is_char_and(lex, '.', '.')) { vstr_add_char(&lex->vstr, '.'); vstr_add_char(&lex->vstr, '.'); next_char(lex); next_char(lex); tok->kind = MP_TOKEN_ELLIPSIS; } else { tok->kind = MP_TOKEN_DEL_PERIOD; } } else { // search for encoded delimiter or operator const char *t = tok_enc; uint tok_enc_index = 0; for (; *t != 0 && !is_char(lex, *t); t += 1) { if (*t == 'e' || *t == 'c') { t += 1; } else if (*t == 'E') { tok_enc_index -= 1; t += 1; } tok_enc_index += 1; } next_char(lex); if (*t == 0) { // didn't match any delimiter or operator characters tok->kind = MP_TOKEN_INVALID; } else { // matched a delimiter or operator character // get the maximum characters for a valid token t += 1; uint t_index = tok_enc_index; for (;;) { for (; *t == 'e'; t += 1) { t += 1; t_index += 1; if (is_char(lex, *t)) { next_char(lex); tok_enc_index = t_index; break; } } if (*t == 'E') { t += 1; if (is_char(lex, *t)) { next_char(lex); tok_enc_index = t_index; } else { tok->kind = MP_TOKEN_INVALID; goto tok_enc_no_match; } break; } if (*t == 'c') { t += 1; t_index += 1; if (is_char(lex, *t)) { next_char(lex); tok_enc_index = t_index; t += 1; } else { break; } } else { break; } } // set token kind tok->kind = tok_enc_kind[tok_enc_index]; tok_enc_no_match: // compute bracket level for implicit line joining if (tok->kind == MP_TOKEN_DEL_PAREN_OPEN || tok->kind == MP_TOKEN_DEL_BRACKET_OPEN || tok->kind == MP_TOKEN_DEL_BRACE_OPEN) { lex->nested_bracket_level += 1; } else if (tok->kind == MP_TOKEN_DEL_PAREN_CLOSE || tok->kind == MP_TOKEN_DEL_BRACKET_CLOSE || tok->kind == MP_TOKEN_DEL_BRACE_CLOSE) { lex->nested_bracket_level -= 1; } } } // point token text to vstr buffer tok->str = vstr_str(&lex->vstr); tok->len = vstr_len(&lex->vstr); // check for keywords if (tok->kind == MP_TOKEN_NAME) { // We check for __debug__ here and convert it to its value. This is so // the parser gives a syntax error on, eg, x.__debug__. Otherwise, we // need to check for this special token in many places in the compiler. // TODO improve speed of these string comparisons //for (int i = 0; tok_kw[i] != NULL; i++) { for (int i = 0; i < ARRAY_SIZE(tok_kw); i++) { if (str_strn_equal(tok_kw[i], tok->str, tok->len)) { if (i == ARRAY_SIZE(tok_kw) - 1) { tok->kind = mp_debug_value; } else { tok->kind = MP_TOKEN_KW_FALSE + i; } break; } } } }