Exemplos de lexer_load_chunk em C++ (Cpp)

Exemplo n.º 1

Arquivo: lexer.c Projeto: hobbit19/cctools-1

struct lexer_book *lexer_init_book(int type, void *data, int line_number, int column_number)
{
    struct lexer_book *bk = malloc(sizeof(struct lexer_book));

    bk->line_number = line_number;
    bk->column_number = column_number;
    bk->column_numbers = list_create(0);

    bk->stream = NULL;
    bk->buffer = NULL;
    bk->eof = 0;

    bk->lexeme = calloc(BUFFER_CHUNK_SIZE, sizeof(char));
    bk->lexeme_size = 0;
    bk->lexeme_max = BUFFER_CHUNK_SIZE;

    bk->token_queue = list_create(0);

    bk->buffer = calloc(2 * BUFFER_CHUNK_SIZE, sizeof(char));
    if(!bk->buffer)
        fatal("Could not allocate memory for input buffer.\n");

    bk->lexeme_end = (bk->buffer + 2 * BUFFER_CHUNK_SIZE - 2);

    if(type == STREAM) {
        bk->stream = (FILE *) data;

        bk->chunk_last_loaded = 2;	// Bootstrap load_chunk to load chunk 1.
        lexer_load_chunk(bk);
    } else {
        lexer_load_string(bk, (char *) data);
    }

    return bk;
}

Exemplo n.º 2

Arquivo: lexer.c Projeto: brenden/cctools

char lexer_next_char(struct lexer_book *bk)
{
	if( *bk->lexeme_end == CHAR_EOF )
	{
		return CHAR_EOF;
	}

	/* If at the end of chunk, load the next chunk. */
	if( ((bk->lexeme_end + 1) == (bk->buffer +   BUFFER_CHUNK_SIZE - 1)) ||
	    ((bk->lexeme_end + 1) == (bk->buffer + 2*BUFFER_CHUNK_SIZE - 1)) )
	{
		if( bk->lexeme_max == BUFFER_CHUNK_SIZE - 1 )
			lexer_report_error(bk, "Input buffer is full. Runaway token?"); //BUG: This is really a recoverable error, increase the buffer size.
		/* Wrap around the file chunks */
		else if( bk->lexeme_end == bk->buffer + 2 * BUFFER_CHUNK_SIZE - 2)
			bk->lexeme_end = bk->buffer;
		/* Position at the beginning of next chunk */
		else
			bk->lexeme_end+=2;

		lexer_load_chunk(bk);
	}
	else
		bk->lexeme_end++;

	char c = *bk->lexeme_end;

	if( c == '\n' )
	{
		bk->line_number++;
		list_push_head(bk->column_numbers, (uint64_t *) bk->column_number);
		bk->column_number = 1;
	}
	else
	{
		bk->column_number++;
	}

	if( c == CHAR_EOF )
	{
		bk->eof = 1;
	}

	return c;
}

Exemplo n.º 3

Arquivo: lexer.c Projeto: NeilB879/cctools

struct lexer *lexer_create(int type, void *data, int line_number, int column_number)
{
	struct lexer *lx = malloc(sizeof(struct lexer));

	lx->line_number = line_number;
	lx->column_number = column_number;
	lx->column_numbers = list_create(0);

	lx->stream = NULL;
	lx->buffer = NULL;
	lx->eof = 0;

	lx->depth = 0;

	lx->lexeme = calloc(BUFFER_CHUNK_SIZE, sizeof(char));
	lx->lexeme_size = 0;
	lx->lexeme_max = BUFFER_CHUNK_SIZE;

	lx->token_queue = list_create(0);

	lx->buffer = calloc(2 * BUFFER_CHUNK_SIZE, sizeof(char));
	if(!lx->buffer)
		fatal("Could not allocate memory for input buffer.\n");

	lx->lexeme_end = (lx->buffer + 2 * BUFFER_CHUNK_SIZE - 2);

	if(type == STREAM) {
		lx->stream = (FILE *) data;

		lx->chunk_last_loaded = 2;	// Bootstrap load_chunk to load chunk 1.
		lexer_load_chunk(lx);
	} else {
		lexer_load_string(lx, (char *) data);
	}

	return lx;
}