示例#1
0
int
cha_parse_bos(cha_lat_t *lat)
{
    static int path0 = -1;

    lat->offset = lat->cursor = 0;
    lat->anno = lat->last_anno = -1;
    lat->head_path = 1;

    cha_block_clear(Cha_mrph_block);

    free_chars();
    free_path();

    Cha_path[0].start = Cha_path[0].end = 0;
    Cha_path[0].path = &path0;
    Cha_path[0].cost = 0;
    Cha_path[0].mrph_p = 0;
    Cha_path[0].state = 0;

    Cha_path_num = 1;
    register_bos_eos();

    return 0;
}
示例#2
0
/* Produces a string consisting of the characters available now in all decdoed
 * buffers. */
static MVMString * get_all_in_buffer(MVMThreadContext *tc, MVMDecodeStream *ds) {
    MVMString *result = (MVMString *)MVM_repr_alloc_init(tc, tc->instance->VMString);
    result->body.storage_type = MVM_STRING_GRAPHEME_32;

    /* If there's no codepoint buffer, then return the empty string. */
    if (!ds->chars_head) {
        result->body.storage.blob_32 = NULL;
        result->body.num_graphs      = 0;
    }

    /* If there's exactly one resulting codepoint buffer and we swallowed none
     * of it, just use it. */
    else if (ds->chars_head == ds->chars_tail && ds->chars_head_pos == 0) {
        /* Set up result string. */
        result->body.storage.blob_32 = ds->chars_head->chars;
        result->body.num_graphs      = ds->chars_head->length;

        /* Don't free the buffer's memory itself, just the holder, as we
         * stole that for the buffer into the string above. */
        free_chars(tc, ds, ds->chars_head);
        ds->chars_head = ds->chars_tail = NULL;
    }

    /* Otherwise, need to assemble all the things. */
    else {
        /* Calculate length. */
        MVMint32 length = 0, pos = 0;
        MVMDecodeStreamChars *cur_chars = ds->chars_head;
        while (cur_chars) {
            if (cur_chars == ds->chars_head)
                length += cur_chars->length - ds->chars_head_pos;
            else
                length += cur_chars->length;
            cur_chars = cur_chars->next;
        }

        /* Allocate a result buffer of the right size. */
        result->body.storage.blob_32 = MVM_malloc(length * sizeof(MVMGrapheme32));
        result->body.num_graphs      = length;

        /* Copy all the things into the target, freeing as we go. */
        cur_chars = ds->chars_head;
        while (cur_chars) {
            MVMDecodeStreamChars *next_chars = cur_chars->next;
            if (cur_chars == ds->chars_head) {
                MVMint32 to_copy = ds->chars_head->length - ds->chars_head_pos;
                memcpy(result->body.storage.blob_32 + pos, cur_chars->chars + ds->chars_head_pos,
                    to_copy * sizeof(MVMGrapheme32));
                pos += to_copy;
            }
            else {
                memcpy(result->body.storage.blob_32 + pos, cur_chars->chars,
                    cur_chars->length * sizeof(MVMGrapheme32));
                pos += cur_chars->length;
            }
            MVM_free(cur_chars->chars);
            free_chars(tc, ds, cur_chars);
            cur_chars = next_chars;
        }
        ds->chars_head = ds->chars_tail = NULL;
    }

    return result;
}
示例#3
0
static MVMString * take_chars(MVMThreadContext *tc, MVMDecodeStream *ds, MVMint32 chars, MVMint32 exclude) {
    MVMString *result;
    MVMint32   found = 0;
    MVMint32   result_found = 0;

    MVMint32   result_chars = chars - exclude;
    if (result_chars < 0)
        MVM_exception_throw_adhoc(tc, "DecodeStream take_chars: chars - exclude < 0 should never happen");

    result                       = (MVMString *)MVM_repr_alloc_init(tc, tc->instance->VMString);
    result->body.storage_type    = MVM_STRING_GRAPHEME_32;
    result->body.num_graphs      = result_chars;

    /* In the best case, the head char buffer has exactly what we need. This
     * will typically happen when it a steady state of decoding lines. */
    if (ds->chars_head->length == chars && ds->chars_head_pos == 0) {
        MVMDecodeStreamChars *cur_chars = ds->chars_head;
        result->body.storage.blob_32 = cur_chars->chars;
        ds->chars_head = cur_chars->next;
        if (ds->chars_head == NULL)
            ds->chars_tail = NULL;
        free_chars(tc, ds, cur_chars);
    }

    /* Otherwise, need to take and copy. */
    else {
        result->body.storage.blob_32 = MVM_malloc(result_chars * sizeof(MVMGrapheme32));
        while (found < chars) {
            MVMDecodeStreamChars *cur_chars = ds->chars_head;
            MVMint32 available = cur_chars->length - ds->chars_head_pos;
            if (available <= chars - found) {
                /* We need all that's left in this buffer and likely
                 * more. */
                MVMDecodeStreamChars *next_chars = cur_chars->next;
                if (available <= result_chars - result_found) {
                    memcpy(result->body.storage.blob_32 + result_found,
                        cur_chars->chars + ds->chars_head_pos,
                        available * sizeof(MVMGrapheme32));
                    result_found += available;
                }
                else {
                    MVMint32 to_copy = result_chars - result_found;
                    memcpy(result->body.storage.blob_32 + result_found,
                        cur_chars->chars + ds->chars_head_pos,
                        to_copy * sizeof(MVMGrapheme32));
                    result_found += to_copy;
                }
                found += available;
                MVM_free(cur_chars->chars);
                free_chars(tc, ds, cur_chars);
                ds->chars_head = next_chars;
                ds->chars_head_pos = 0;
                if (ds->chars_head == NULL)
                    ds->chars_tail = NULL;
            }
            else {
                /* There's enough in this buffer to satisfy us, and we'll leave
                 * some behind. */
                MVMint32 take = chars - found;
                MVMint32 to_copy = result_chars - result_found;
                memcpy(result->body.storage.blob_32 + result_found,
                    cur_chars->chars + ds->chars_head_pos,
                    to_copy * sizeof(MVMGrapheme32));
                result_found += to_copy;
                found += take;
                ds->chars_head_pos += take;
            }
        }
    }
    return result;
}
示例#4
0
文件: buffer.c 项目: dmt4/ne
int insert_stream(buffer * const b, line_desc * ld, int64_t line, int64_t pos, const char * const stream, const int64_t stream_len) {
	if (!b || !ld || !stream || stream_len < 1 || pos > ld->line_len) return ERROR;

	assert_line_desc(ld, b->encoding);
	assert_buffer(b);

	block_signals();

	if (b->opt.do_undo && !(b->undoing || b->redoing)) {
		const int error = add_undo_step(b, line, pos, -stream_len);
		if (error) {
			release_signals();
			return error;
		}
	}

	const char *s = stream;
	while(s - stream < stream_len) {
		int64_t const len = strnlen_ne(s, stream_len - (s - stream));
		if (len) {

			/* First case; there is no character allocated on this line. We
			have to freshly allocate the line. */

			if (!ld->line) {
				if (ld->line = alloc_chars(b, len)) {
					memcpy(ld->line, s, len);
					ld->line_len = len;
				}
				else {
					release_signals();
					return OUT_OF_MEMORY;
				}
			}


			/* Second case. There are not enough characters around ld->line. Note
			that the value of the check_first_before parameter depends on
			the position at which the insertion will be done, and it is chosen
			in such a way to minimize the number of characters to move. */

			else {
				const int64_t result = alloc_chars_around(b, ld, len, pos < ld->line_len / 2);
				if (result < 0) {
					char * const p = alloc_chars(b, ld->line_len + len);
					if (p) {
						memcpy(p, ld->line, pos);
						memcpy(&p[pos], s, len);
						memcpy(&p[pos + len], ld->line + pos, ld->line_len - pos);
						free_chars(b, ld->line, ld->line_len);
						ld->line = p;
						ld->line_len += len;
					}
					else {
						release_signals();
						return OUT_OF_MEMORY;
					}
				}
				else { /* Third case. There are enough free characters around ld->line. */
					if (len - result) memmove(ld->line - (len - result), ld->line, pos);
					if (result) memmove(ld->line + pos + result, ld->line + pos, ld->line_len - pos);
					memcpy(ld->line - (len - result) + pos, s, len);
	
					ld->line -= (len - result);
					ld->line_len += len;
				}
			}
			b->is_modified = 1;

			/* We just inserted len chars at (line,pos); adjust bookmarks and mark accordingly. */
			if (b->marking && b->block_start_line == line && b->block_start_pos > pos) b->block_start_pos += len;

			for (int i = 0, mask = b->bookmark_mask; mask; i++, mask >>= 1) 
				if ((mask & 1) && b->bookmark[i].line == line && b->bookmark[i].pos > pos) b->bookmark[i].pos += len;
		}

		/* If the string we have inserted has a NULL at the end, we create a new
			line under the current one and set ld to point to it. */

		if (len + (s - stream) < stream_len) {
			line_desc *new_ld;

			if (new_ld = alloc_line_desc(b)) {

				add(&new_ld->ld_node, &ld->ld_node);
				b->num_lines++;

				if (pos + len < ld->line_len) {
					new_ld->line_len = ld->line_len - pos - len;
					new_ld->line = &ld->line[pos + len];
					ld->line_len = pos + len;
					if (pos + len == 0) ld->line = NULL;
				}

				b->is_modified = 1;
				ld = new_ld;

				/* We just inserted a line break at (line,pos);
				   adjust the buffer bookmarks and mark accordingly. */
				if (b->marking) {
					if (b->block_start_line == line && b->block_start_pos > pos) {
						b->block_start_pos -= pos;
						b->block_start_line++;
					}
					else if (b->block_start_line > line) b->block_start_line++;
				}
				for (int i = 0, mask=b->bookmark_mask; mask; i++, mask >>= 1) {
					if (mask & 1) {
						if (b->bookmark[i].line == line && b->bookmark[i].pos > pos) {
							b->bookmark[i].pos -= pos;
							b->bookmark[i].line++;
						}
						else if (b->bookmark[i].line > line) b->bookmark[i].line++;
					}
				}
				pos = 0;
				line++;
			}
			else {