Ejemplo n.º 1
0
/**
 * Reads all characters from the input stream until one of the tokens is encountered. The terminating
 * token is consumed by this operation.
 */
int scan_until_with_raw_args(scanner_p scanner, slice_t* slice, int tokens[]){
	int character;
	while (true) {
		character = read_next_char(scanner);
		/*
		printf("size: %zd, pos: %zd, consumed: %zd, filled: %zd\n",
			scanner->buffer_size, scanner->buffer_pos, scanner->buffer_consumed, scanner->buffer_filled);
		*/
		// Look for each terminator
		for(size_t token_idx = 0; tokens[token_idx] != -2; token_idx++){
			if (character == tokens[token_idx]){
				// We found it, copy the unconsumed buffer into a new memory area and return it.
				// The buffer_pos has already been incremented by read_next_char. Therefore start
				// to copy one char back.
				if (slice != NULL){
					size_t terminator_pos = (character != EOF) ? scanner->buffer_pos - 1 : scanner->buffer_pos;
					size_t content_length = terminator_pos - scanner->buffer_consumed;
					slice->ptr = malloc(content_length + 1);
					memcpy(slice->ptr, scanner->buffer_ptr + scanner->buffer_consumed, content_length);
					slice->ptr[content_length] = '\0';
					slice->length = content_length;
				}
				scanner->buffer_consumed = scanner->buffer_pos;
				return character;
			}
		}
	}
}
Ejemplo n.º 2
0
keyword_t get_keyword ()
{
    keyword_t key;
    int i;
    i = 0;
    while (isalpha (current_char) && i < MAX_KEYWORD) {
        key.val[i++] = current_char;
        read_next_char ();
    }

    if (i == MAX_KEYWORD) {
        i = 0;
    }

    key.val[i] = '\0';

    if (! key.val ) {
        key.code = INVALID;
        return key;
    }

    for (i = 0 ; i < number_of_keywords; i++) {
        if ( ! strcmp(key.val,keywords[i].val) ) {
            key.code = keywords[i].code;
            return key;
        }
    }

    key.code = INVALID;
    return key;
}
Ejemplo n.º 3
0
void
mm_text_io_c::detect_eol_style() {
  if (m_eol_style_detected)
    return;

  m_eol_style_detected = true;
  bool found_cr_or_nl  = false;

  save_pos();

  while (1) {
    char utf8char[9];
    size_t len = read_next_char(utf8char);
    if (0 == len)
      break;

    if ((1 == len) && ('\r' == utf8char[0])) {
      found_cr_or_nl          = true;
      m_uses_carriage_returns = true;

    } else if ((1 == len) && ('\n' == utf8char[0])) {
      found_cr_or_nl  = true;
      m_uses_newlines = true;

    } else if (found_cr_or_nl)
      break;
  }

  restore_pos();
}
Ejemplo n.º 4
0
int read_and_parse_shell ()
{
    keyword_t key;
    prompt();
    fgets(read_buffer,BUFFER_LENGTH,stdin);

    read_next_char ();
    key = get_keyword ();
    return key.code; 
}
Ejemplo n.º 5
0
boolean MiniWebServer::get_line(char* buffer, int size) {
    int i = 0;
    char ch;

    buffer[0] = 0;
    for (; i < size - 1; i++) {
        if (!read_next_char(client_, (uint8_t*)&ch)) {
            continue;
        }
        if (ch == '\n') {
            break;
        }
        buffer[i] = ch;
    }
    buffer[i] = 0;
    return i < size - 1;
}
Ejemplo n.º 6
0
fint Scanner::get_char() {
  fint c;
  if (chars) {
    c = chars->c;
    chars = chars->prev;
  } else {
    c = read_next_char();
    if (c == '\n' || c == '\r') {
      line ++;
      column = 1;
    } else {
      column ++;
    }
  }
  if (c != EOF) {
    is_buffer_filled() ? sourceBuf->advance() : sourceBuf->nextPut(char(c));
  }
  return c;
}
Ejemplo n.º 7
0
/**
 * Consumes one character and returns it. Only useful right now to consume individual characters.
 * No error handling yet.
 * 
 * The character is only consumed if it matches one of the tokens.
 */
int scan_one_of_with_raw_args(scanner_p scanner, int tokens[]){
	int c = read_next_char(scanner);
	
	for(size_t token_idx = 0; tokens[token_idx] != -2; token_idx++){
		if (c == tokens[token_idx]){
			scanner->buffer_consumed++;
			return c;
		}
	}
	
	/*
	printf("failed to consume one of the tokens:");
	for(size_t token_idx = 0; tokens[token_idx] != -2; token_idx++)
		printf(" %c", tokens[token_idx]);
	printf("\n");
	*/
	
	return c;
}
Ejemplo n.º 8
0
/**
 * Same as scan_while_with_raw_args() but uses functions instead of characters for the check. Build in a
 * way that the ctype.h functions like isdigit() can be used.
 */
int scan_while_func_with_raw_args(scanner_p scanner, slice_t* slice, scanner_check_func_t funcs[]){
	int character;
	bool passed;
	while (true) {
		character = read_next_char(scanner);
		passed = false;
		
		// Look for each terminator
		for(size_t func_idx = 0; funcs[func_idx] != NULL; func_idx++){
			if ( funcs[func_idx](character) ){
				passed = true;
				break;
			}
		}
		
		if (!passed){
			// We found the first character that is not included in the list of tokens. Copy the
			// unconsumed buffer into a new memory area and return it.
			// The buffer_pos has already been incremented by read_next_char. Therefore start
			// to copy one char back. Except we're at an EOF, in this case use all of the buffer
			// since the EOF itself is not in the buffer.
			if (slice != NULL){
				size_t terminator_pos = (character != EOF) ? scanner->buffer_pos - 1 : scanner->buffer_pos;
				size_t content_length = terminator_pos - scanner->buffer_consumed;
				slice->ptr = malloc(content_length + 1);
				memcpy(slice->ptr, scanner->buffer_ptr + scanner->buffer_consumed, content_length);
				slice->ptr[content_length] = '\0';
				slice->length = content_length;
			}
			// Go back one char in the buffer (except we got an EOF). We only want o peek at
			// the terminator not consume it.
			if (character != EOF)
				rewind_one_char(scanner);
			scanner->buffer_consumed = scanner->buffer_pos;
			return character;
		}
	}
}
Ejemplo n.º 9
0
/**
 * Same as scan_until_with_raw_args() but uses functions instead of characters for the check. Build in a
 * way that the ctype.h functions like isdigit() can be used.
 */
int scan_until_func_with_raw_args(scanner_p scanner, slice_t* slice, scanner_check_func_t funcs[]){
	int character;
	while (true) {
		character = read_next_char(scanner);
		// Look for each terminator
		for(size_t func_idx = 0; funcs[func_idx] != NULL; func_idx++){
			if ( funcs[func_idx](character) ){
				// We found it, copy the unconsumed buffer into a new memory area and return it.
				// The buffer_pos has already been incremented by read_next_char. Therefore start
				// to copy one char back.
				if (slice != NULL){
					size_t terminator_pos = (character != EOF) ? scanner->buffer_pos - 1 : scanner->buffer_pos;
					size_t content_length = terminator_pos - scanner->buffer_consumed;
					slice->ptr = malloc(content_length + 1);
					memcpy(slice->ptr, scanner->buffer_ptr + scanner->buffer_consumed, content_length);
					slice->ptr[content_length] = '\0';
					slice->length = content_length;
				}
				scanner->buffer_consumed = scanner->buffer_pos;
				return character;
			}
		}
	}
}
Ejemplo n.º 10
0
void huffman_decoder::decode_and_write(std::ostream &outs) {
    code_char_t c = 0;
    while (read_next_char(c)) {
        outs.put(c);
    }
}
Ejemplo n.º 11
0
// Process headers.
boolean MiniWebServer::process_headers() {
    if (headers_) {
        // First clear the header values from the previous HTTP request.
        for (int i = 0; headers_[i].header; i++) {
            if (headers_[i].value) {
                free(headers_[i].value);
                // Ensure the pointer is cleared once the memory is freed.
                headers_[i].value = NULL;
            }
        }
    }

    enum State {
        ERROR,
        START_LINE,
        HEADER_NAME,
        HEADER_VALUE,
        HEADER_VALUE_SKIP_INITIAL_SPACES,
        HEADER_IGNORE_VALUE,
        END_HEADERS,
    };
    State state = START_LINE;

    char ch;
    int pos;
    const char* header;
    while (1) {
        if (should_stop_processing()) {
            return false;
        }
        if (!read_next_char(client_, (uint8_t*)&ch)) {
            continue;
        }
#if DEBUG
        Serial.print(ch);
#endif
        switch (state) {
        case START_LINE:
            if (ch == '\r') {
                break;
            } else if (ch == '\n') {
                state = END_HEADERS;
            } else if (isalnum(ch) || ch == '-') {
                pos = 0;
                buffer[pos++] = ch;
                state = HEADER_NAME;
            } else {
                state = ERROR;
            }
            break;

        case HEADER_NAME:
            if (pos + 1 >= sizeof(buffer)) {
                state = ERROR;
                break;
            }
            if (ch == ':') {
                buffer[pos] = 0;
                header = buffer;
                if (is_requested_header(&header)) {
                    state = HEADER_VALUE_SKIP_INITIAL_SPACES;
                } else {
                    state = HEADER_IGNORE_VALUE;
                }
                pos = 0;
            } else if (isalnum(ch) || ch == '-') {
                buffer[pos++] = ch;
            } else {
                state = ERROR;
                break;
            }
            break;

        case HEADER_VALUE_SKIP_INITIAL_SPACES:
            if (pos + 1 >= sizeof(buffer)) {
                state = ERROR;
                break;
            }
            if (ch != ' ') {
                buffer[pos++] = ch;
                state = HEADER_VALUE;
            }
            break;

        case HEADER_VALUE:
            if (pos + 1 >= sizeof(buffer)) {
                state = ERROR;
                break;
            }
            if (ch == '\n') {
                buffer[pos] = 0;
                if (!assign_header_value(header, buffer)) {
                    state = ERROR;
                    break;
                }
                state = START_LINE;
            } else {
                if (ch != '\r') {
                    buffer[pos++] = ch;
                }
            }
            break;

        case HEADER_IGNORE_VALUE:
            if (ch == '\n') {
                state = START_LINE;
            }
            break;

        default:
            break;
        }

        if (state == END_HEADERS) {
            break;
        }
        if (state == ERROR) {
            return false;
        }
    }
    return true;
}
Ejemplo n.º 12
0
void InteractiveScanner::discardInput() {
  fint c;
  do {
    c = read_next_char();
  } while (c != EOF  &&  c != '\n');
}
Ejemplo n.º 13
0
void get_next_token(int is_declaration) {
	
	char temp_string[500];
	int counter;
	
	token.type = TOKEN_TYPE_IGNORED;
	token.value = NULL;
	
	if (reading_head.current == EOF) {
		/* return a END_OF_FILE token if the file was completely read */
		
		token.type = TOKEN_TYPE_END_OF_FILE;
		
	} else {
		/* extract a token from file */
		
		while (reading_head.current != EOF && token.type == TOKEN_TYPE_IGNORED) {
			/* chars being discarted */
			
			/* the transducer automata will execute a transition to consume the current char */
			token.type = transducer_consume_input(reading_head.current, reading_head.next);
			
			read_next_char();
		}
		
		
		/* 
		 * the previous char will not be dicarted because it created a transition to a
		 * valid state with transducer automata
		 */
		temp_string[0] = reading_head.previous;
		counter = 1;
		
		token.line = reading_head.line;
		token.column = reading_head.column-1;
		
		while (reading_head.current != EOF && token.type == TOKEN_TYPE_INCOMPLETE) {
			/* building token */
			
			/* the transducer automata will execute a transition to consume the current char */
			token.type = transducer_consume_input(reading_head.current, reading_head.next);
			
			temp_string[counter] = reading_head.current;
			counter += 1;
			
			read_next_char();
		}

		
		/* allocate memory to token value */
		token.value = (char*) malloc(counter*sizeof(char));
		while (counter > 0) {
			/* fill each char of the token value */
			token.value[counter-1] = temp_string[counter-1];
			counter -= 1;
		}
		
		
		if (token.type == TOKEN_TYPE_IGNORED) {
			free(token.value);
			token.value = NULL;
		} else if (token.type == TOKEN_TYPE_INCOMPLETE) {
			token.type = TOKEN_TYPE_INVALID;
		} else if (token.type == TOKEN_TYPE_IDENTIFIER && (find_by_key(&table_reserved_words, token.value) >= 0)) {
			token.type = TOKEN_TYPE_RESERVED_WORD;
		}
		
		//Only identifiers being declared will be added to the symbol table.
		if (token.type == TOKEN_TYPE_IDENTIFIER) {
			if (is_declaration == 1 ) {
				token.index = update_semantic_tables();
			}
		}
		else {
			token.index = update_semantic_tables();
		}

		
		
	}
	
}