/** * Reads all characters from the input stream until one of the tokens is encountered. The terminating * token is consumed by this operation. */ int scan_until_with_raw_args(scanner_p scanner, slice_t* slice, int tokens[]){ int character; while (true) { character = read_next_char(scanner); /* printf("size: %zd, pos: %zd, consumed: %zd, filled: %zd\n", scanner->buffer_size, scanner->buffer_pos, scanner->buffer_consumed, scanner->buffer_filled); */ // Look for each terminator for(size_t token_idx = 0; tokens[token_idx] != -2; token_idx++){ if (character == tokens[token_idx]){ // We found it, copy the unconsumed buffer into a new memory area and return it. // The buffer_pos has already been incremented by read_next_char. Therefore start // to copy one char back. if (slice != NULL){ size_t terminator_pos = (character != EOF) ? scanner->buffer_pos - 1 : scanner->buffer_pos; size_t content_length = terminator_pos - scanner->buffer_consumed; slice->ptr = malloc(content_length + 1); memcpy(slice->ptr, scanner->buffer_ptr + scanner->buffer_consumed, content_length); slice->ptr[content_length] = '\0'; slice->length = content_length; } scanner->buffer_consumed = scanner->buffer_pos; return character; } } } }
keyword_t get_keyword () { keyword_t key; int i; i = 0; while (isalpha (current_char) && i < MAX_KEYWORD) { key.val[i++] = current_char; read_next_char (); } if (i == MAX_KEYWORD) { i = 0; } key.val[i] = '\0'; if (! key.val ) { key.code = INVALID; return key; } for (i = 0 ; i < number_of_keywords; i++) { if ( ! strcmp(key.val,keywords[i].val) ) { key.code = keywords[i].code; return key; } } key.code = INVALID; return key; }
void mm_text_io_c::detect_eol_style() { if (m_eol_style_detected) return; m_eol_style_detected = true; bool found_cr_or_nl = false; save_pos(); while (1) { char utf8char[9]; size_t len = read_next_char(utf8char); if (0 == len) break; if ((1 == len) && ('\r' == utf8char[0])) { found_cr_or_nl = true; m_uses_carriage_returns = true; } else if ((1 == len) && ('\n' == utf8char[0])) { found_cr_or_nl = true; m_uses_newlines = true; } else if (found_cr_or_nl) break; } restore_pos(); }
int read_and_parse_shell () { keyword_t key; prompt(); fgets(read_buffer,BUFFER_LENGTH,stdin); read_next_char (); key = get_keyword (); return key.code; }
boolean MiniWebServer::get_line(char* buffer, int size) { int i = 0; char ch; buffer[0] = 0; for (; i < size - 1; i++) { if (!read_next_char(client_, (uint8_t*)&ch)) { continue; } if (ch == '\n') { break; } buffer[i] = ch; } buffer[i] = 0; return i < size - 1; }
fint Scanner::get_char() { fint c; if (chars) { c = chars->c; chars = chars->prev; } else { c = read_next_char(); if (c == '\n' || c == '\r') { line ++; column = 1; } else { column ++; } } if (c != EOF) { is_buffer_filled() ? sourceBuf->advance() : sourceBuf->nextPut(char(c)); } return c; }
/** * Consumes one character and returns it. Only useful right now to consume individual characters. * No error handling yet. * * The character is only consumed if it matches one of the tokens. */ int scan_one_of_with_raw_args(scanner_p scanner, int tokens[]){ int c = read_next_char(scanner); for(size_t token_idx = 0; tokens[token_idx] != -2; token_idx++){ if (c == tokens[token_idx]){ scanner->buffer_consumed++; return c; } } /* printf("failed to consume one of the tokens:"); for(size_t token_idx = 0; tokens[token_idx] != -2; token_idx++) printf(" %c", tokens[token_idx]); printf("\n"); */ return c; }
/** * Same as scan_while_with_raw_args() but uses functions instead of characters for the check. Build in a * way that the ctype.h functions like isdigit() can be used. */ int scan_while_func_with_raw_args(scanner_p scanner, slice_t* slice, scanner_check_func_t funcs[]){ int character; bool passed; while (true) { character = read_next_char(scanner); passed = false; // Look for each terminator for(size_t func_idx = 0; funcs[func_idx] != NULL; func_idx++){ if ( funcs[func_idx](character) ){ passed = true; break; } } if (!passed){ // We found the first character that is not included in the list of tokens. Copy the // unconsumed buffer into a new memory area and return it. // The buffer_pos has already been incremented by read_next_char. Therefore start // to copy one char back. Except we're at an EOF, in this case use all of the buffer // since the EOF itself is not in the buffer. if (slice != NULL){ size_t terminator_pos = (character != EOF) ? scanner->buffer_pos - 1 : scanner->buffer_pos; size_t content_length = terminator_pos - scanner->buffer_consumed; slice->ptr = malloc(content_length + 1); memcpy(slice->ptr, scanner->buffer_ptr + scanner->buffer_consumed, content_length); slice->ptr[content_length] = '\0'; slice->length = content_length; } // Go back one char in the buffer (except we got an EOF). We only want o peek at // the terminator not consume it. if (character != EOF) rewind_one_char(scanner); scanner->buffer_consumed = scanner->buffer_pos; return character; } } }
/** * Same as scan_until_with_raw_args() but uses functions instead of characters for the check. Build in a * way that the ctype.h functions like isdigit() can be used. */ int scan_until_func_with_raw_args(scanner_p scanner, slice_t* slice, scanner_check_func_t funcs[]){ int character; while (true) { character = read_next_char(scanner); // Look for each terminator for(size_t func_idx = 0; funcs[func_idx] != NULL; func_idx++){ if ( funcs[func_idx](character) ){ // We found it, copy the unconsumed buffer into a new memory area and return it. // The buffer_pos has already been incremented by read_next_char. Therefore start // to copy one char back. if (slice != NULL){ size_t terminator_pos = (character != EOF) ? scanner->buffer_pos - 1 : scanner->buffer_pos; size_t content_length = terminator_pos - scanner->buffer_consumed; slice->ptr = malloc(content_length + 1); memcpy(slice->ptr, scanner->buffer_ptr + scanner->buffer_consumed, content_length); slice->ptr[content_length] = '\0'; slice->length = content_length; } scanner->buffer_consumed = scanner->buffer_pos; return character; } } } }
void huffman_decoder::decode_and_write(std::ostream &outs) { code_char_t c = 0; while (read_next_char(c)) { outs.put(c); } }
// Process headers. boolean MiniWebServer::process_headers() { if (headers_) { // First clear the header values from the previous HTTP request. for (int i = 0; headers_[i].header; i++) { if (headers_[i].value) { free(headers_[i].value); // Ensure the pointer is cleared once the memory is freed. headers_[i].value = NULL; } } } enum State { ERROR, START_LINE, HEADER_NAME, HEADER_VALUE, HEADER_VALUE_SKIP_INITIAL_SPACES, HEADER_IGNORE_VALUE, END_HEADERS, }; State state = START_LINE; char ch; int pos; const char* header; while (1) { if (should_stop_processing()) { return false; } if (!read_next_char(client_, (uint8_t*)&ch)) { continue; } #if DEBUG Serial.print(ch); #endif switch (state) { case START_LINE: if (ch == '\r') { break; } else if (ch == '\n') { state = END_HEADERS; } else if (isalnum(ch) || ch == '-') { pos = 0; buffer[pos++] = ch; state = HEADER_NAME; } else { state = ERROR; } break; case HEADER_NAME: if (pos + 1 >= sizeof(buffer)) { state = ERROR; break; } if (ch == ':') { buffer[pos] = 0; header = buffer; if (is_requested_header(&header)) { state = HEADER_VALUE_SKIP_INITIAL_SPACES; } else { state = HEADER_IGNORE_VALUE; } pos = 0; } else if (isalnum(ch) || ch == '-') { buffer[pos++] = ch; } else { state = ERROR; break; } break; case HEADER_VALUE_SKIP_INITIAL_SPACES: if (pos + 1 >= sizeof(buffer)) { state = ERROR; break; } if (ch != ' ') { buffer[pos++] = ch; state = HEADER_VALUE; } break; case HEADER_VALUE: if (pos + 1 >= sizeof(buffer)) { state = ERROR; break; } if (ch == '\n') { buffer[pos] = 0; if (!assign_header_value(header, buffer)) { state = ERROR; break; } state = START_LINE; } else { if (ch != '\r') { buffer[pos++] = ch; } } break; case HEADER_IGNORE_VALUE: if (ch == '\n') { state = START_LINE; } break; default: break; } if (state == END_HEADERS) { break; } if (state == ERROR) { return false; } } return true; }
void InteractiveScanner::discardInput() { fint c; do { c = read_next_char(); } while (c != EOF && c != '\n'); }
void get_next_token(int is_declaration) { char temp_string[500]; int counter; token.type = TOKEN_TYPE_IGNORED; token.value = NULL; if (reading_head.current == EOF) { /* return a END_OF_FILE token if the file was completely read */ token.type = TOKEN_TYPE_END_OF_FILE; } else { /* extract a token from file */ while (reading_head.current != EOF && token.type == TOKEN_TYPE_IGNORED) { /* chars being discarted */ /* the transducer automata will execute a transition to consume the current char */ token.type = transducer_consume_input(reading_head.current, reading_head.next); read_next_char(); } /* * the previous char will not be dicarted because it created a transition to a * valid state with transducer automata */ temp_string[0] = reading_head.previous; counter = 1; token.line = reading_head.line; token.column = reading_head.column-1; while (reading_head.current != EOF && token.type == TOKEN_TYPE_INCOMPLETE) { /* building token */ /* the transducer automata will execute a transition to consume the current char */ token.type = transducer_consume_input(reading_head.current, reading_head.next); temp_string[counter] = reading_head.current; counter += 1; read_next_char(); } /* allocate memory to token value */ token.value = (char*) malloc(counter*sizeof(char)); while (counter > 0) { /* fill each char of the token value */ token.value[counter-1] = temp_string[counter-1]; counter -= 1; } if (token.type == TOKEN_TYPE_IGNORED) { free(token.value); token.value = NULL; } else if (token.type == TOKEN_TYPE_INCOMPLETE) { token.type = TOKEN_TYPE_INVALID; } else if (token.type == TOKEN_TYPE_IDENTIFIER && (find_by_key(&table_reserved_words, token.value) >= 0)) { token.type = TOKEN_TYPE_RESERVED_WORD; } //Only identifiers being declared will be added to the symbol table. if (token.type == TOKEN_TYPE_IDENTIFIER) { if (is_declaration == 1 ) { token.index = update_semantic_tables(); } } else { token.index = update_semantic_tables(); } } }