voikko_token_type Tokenizer::nextToken(voikko_options_t * options, const wchar_t * text, size_t textlen, size_t * tokenlen) {
if (textlen == 0) {
*tokenlen = 0;
return TOKEN_NONE;
}
switch (get_char_type(text[0])) {
case CHAR_LETTER:
case CHAR_DIGIT:
*tokenlen = word_length(text, textlen, options);
return TOKEN_WORD;
case CHAR_WHITESPACE:
for (size_t i = 1; i < textlen; i++) {
if (get_char_type(text[i]) != CHAR_WHITESPACE) {
*tokenlen = i;
return TOKEN_WHITESPACE;
}
}
*tokenlen = textlen;
return TOKEN_WHITESPACE;
case CHAR_PUNCTUATION:
if (wcschr(L"-\u2010\u2011", text[0])) {
if (textlen == 1) {
*tokenlen = 1;
return TOKEN_PUNCTUATION;
}
size_t wlen = word_length(text + 1, textlen - 1, options);
if (wlen == 0) {
*tokenlen = 1;
return TOKEN_PUNCTUATION;
}
*tokenlen = wlen + 1;
return TOKEN_WORD;
}
else if (textlen >= 3 && text[0] == L'.' && text[1] == L'.'
&& text[2] == L'.') {
*tokenlen = 3;
return TOKEN_PUNCTUATION;
}
*tokenlen = 1;
return TOKEN_PUNCTUATION;
case CHAR_UNKNOWN:
*tokenlen = 1;
return TOKEN_UNKNOWN;
}
return TOKEN_NONE; // unreachable
}
int os_call_shell(char_u *cmd, ShellOpts opts, char_u *extra_shell_arg)
{
uv_stdio_container_t proc_stdio[3];
uv_process_options_t proc_opts;
uv_process_t proc;
uv_pipe_t proc_stdin, proc_stdout;
uv_write_t write_req;
int expected_exits = 1;
ProcessData pdata = {
.reading = false,
.exited = 0,
.old_mode = cur_tmode,
.old_state = State,
.shell_stdin = (uv_stream_t *)&proc_stdin,
.wbuffer = NULL,
};
out_flush();
if (opts & kShellOptCooked) {
// set to normal mode
settmode(TMODE_COOK);
}
// While the child is running, ignore terminating signals
signal_reject_deadly();
// Create argv for `uv_spawn`
// TODO(tarruda): we can use a static buffer for small argument vectors. 1024
// bytes should be enough for most of the commands and if more is necessary
// we can allocate a another buffer
proc_opts.args = shell_build_argv(cmd, extra_shell_arg);
proc_opts.file = proc_opts.args[0];
proc_opts.exit_cb = exit_cb;
// Initialize libuv structures
proc_opts.stdio = proc_stdio;
proc_opts.stdio_count = 3;
// Hide window on Windows :)
proc_opts.flags = UV_PROCESS_WINDOWS_HIDE;
proc_opts.cwd = NULL;
proc_opts.env = NULL;
// The default is to inherit all standard file descriptors(this will change
// when the UI is moved to an external process)
proc_stdio[0].flags = UV_INHERIT_FD;
proc_stdio[0].data.fd = 0;
proc_stdio[1].flags = UV_INHERIT_FD;
proc_stdio[1].data.fd = 1;
proc_stdio[2].flags = UV_INHERIT_FD;
proc_stdio[2].data.fd = 2;
if (opts & (kShellOptHideMess | kShellOptExpand)) {
// Ignore the shell stdio(redirects to /dev/null on unixes)
proc_stdio[0].flags = UV_IGNORE;
proc_stdio[1].flags = UV_IGNORE;
proc_stdio[2].flags = UV_IGNORE;
} else {
State = EXTERNCMD;
if (opts & kShellOptWrite) {
// Write from the current buffer into the process stdin
uv_pipe_init(uv_default_loop(), &proc_stdin, 0);
write_req.data = &pdata;
proc_stdio[0].flags = UV_CREATE_PIPE | UV_READABLE_PIPE;
proc_stdio[0].data.stream = (uv_stream_t *)&proc_stdin;
}
if (opts & kShellOptRead) {
// Read from the process stdout into the current buffer
uv_pipe_init(uv_default_loop(), &proc_stdout, 0);
proc_stdout.data = &pdata;
proc_stdio[1].flags = UV_CREATE_PIPE | UV_WRITABLE_PIPE;
proc_stdio[1].data.stream = (uv_stream_t *)&proc_stdout;
ga_init(&pdata.ga, 1, BUFFER_LENGTH);
}
}
if (uv_spawn(uv_default_loop(), &proc, &proc_opts)) {
// Failed, probably due to `sh` not being executable
if (!emsg_silent) {
MSG_PUTS(_("\nCannot execute shell "));
msg_outtrans(p_sh);
msg_putchar('\n');
}
return proc_cleanup_exit(&pdata, &proc_opts, opts);
}
// Assign the flag address after `proc` is initialized by `uv_spawn`
proc.data = &pdata;
if (opts & kShellOptWrite) {
// Queue everything for writing to the shell stdin
write_selection(&write_req);
expected_exits++;
}
if (opts & kShellOptRead) {
// Start the read stream for the shell stdout
uv_read_start((uv_stream_t *)&proc_stdout, alloc_cb, read_cb);
expected_exits++;
}
// Keep running the loop until all three handles are completely closed
while (pdata.exited < expected_exits) {
uv_run(uv_default_loop(), UV_RUN_ONCE);
if (got_int) {
// Forward SIGINT to the shell
// TODO(tarruda): for now this is only needed if the terminal is in raw
// mode, but when the UI is externalized we'll also need it, so leave it
// here
uv_process_kill(&proc, SIGINT);
got_int = false;
}
}
if (opts & kShellOptRead) {
if (pdata.ga.ga_len > 0) {
// If there's an unfinished line in the growable array, append it now.
append_ga_line(&pdata.ga);
// remember that the NL was missing
curbuf->b_no_eol_lnum = curwin->w_cursor.lnum;
} else {
curbuf->b_no_eol_lnum = 0;
}
ga_clear(&pdata.ga);
}
if (opts & kShellOptWrite) {
free(pdata.wbuffer);
}
return proc_cleanup_exit(&pdata, &proc_opts, opts);
}
static int tokenize(char_u *str, char **argv)
{
int argc = 0, len;
char_u *p = str;
while (*p != NUL) {
len = word_length(p);
if (argv != NULL) {
// Fill the slot
argv[argc] = xmalloc(len + 1);
memcpy(argv[argc], p, len);
argv[argc][len] = NUL;
}
argc++;
p += len;
p = skipwhite(p);
}
return argc;
}
