voikko_token_type Tokenizer::nextToken(voikko_options_t * options, const wchar_t * text, size_t textlen, size_t * tokenlen) { if (textlen == 0) { *tokenlen = 0; return TOKEN_NONE; } switch (get_char_type(text[0])) { case CHAR_LETTER: case CHAR_DIGIT: *tokenlen = word_length(text, textlen, options); return TOKEN_WORD; case CHAR_WHITESPACE: for (size_t i = 1; i < textlen; i++) { if (get_char_type(text[i]) != CHAR_WHITESPACE) { *tokenlen = i; return TOKEN_WHITESPACE; } } *tokenlen = textlen; return TOKEN_WHITESPACE; case CHAR_PUNCTUATION: if (wcschr(L"-\u2010\u2011", text[0])) { if (textlen == 1) { *tokenlen = 1; return TOKEN_PUNCTUATION; } size_t wlen = word_length(text + 1, textlen - 1, options); if (wlen == 0) { *tokenlen = 1; return TOKEN_PUNCTUATION; } *tokenlen = wlen + 1; return TOKEN_WORD; } else if (textlen >= 3 && text[0] == L'.' && text[1] == L'.' && text[2] == L'.') { *tokenlen = 3; return TOKEN_PUNCTUATION; } *tokenlen = 1; return TOKEN_PUNCTUATION; case CHAR_UNKNOWN: *tokenlen = 1; return TOKEN_UNKNOWN; } return TOKEN_NONE; // unreachable }
int os_call_shell(char_u *cmd, ShellOpts opts, char_u *extra_shell_arg) { uv_stdio_container_t proc_stdio[3]; uv_process_options_t proc_opts; uv_process_t proc; uv_pipe_t proc_stdin, proc_stdout; uv_write_t write_req; int expected_exits = 1; ProcessData pdata = { .reading = false, .exited = 0, .old_mode = cur_tmode, .old_state = State, .shell_stdin = (uv_stream_t *)&proc_stdin, .wbuffer = NULL, }; out_flush(); if (opts & kShellOptCooked) { // set to normal mode settmode(TMODE_COOK); } // While the child is running, ignore terminating signals signal_reject_deadly(); // Create argv for `uv_spawn` // TODO(tarruda): we can use a static buffer for small argument vectors. 1024 // bytes should be enough for most of the commands and if more is necessary // we can allocate a another buffer proc_opts.args = shell_build_argv(cmd, extra_shell_arg); proc_opts.file = proc_opts.args[0]; proc_opts.exit_cb = exit_cb; // Initialize libuv structures proc_opts.stdio = proc_stdio; proc_opts.stdio_count = 3; // Hide window on Windows :) proc_opts.flags = UV_PROCESS_WINDOWS_HIDE; proc_opts.cwd = NULL; proc_opts.env = NULL; // The default is to inherit all standard file descriptors(this will change // when the UI is moved to an external process) proc_stdio[0].flags = UV_INHERIT_FD; proc_stdio[0].data.fd = 0; proc_stdio[1].flags = UV_INHERIT_FD; proc_stdio[1].data.fd = 1; proc_stdio[2].flags = UV_INHERIT_FD; proc_stdio[2].data.fd = 2; if (opts & (kShellOptHideMess | kShellOptExpand)) { // Ignore the shell stdio(redirects to /dev/null on unixes) proc_stdio[0].flags = UV_IGNORE; proc_stdio[1].flags = UV_IGNORE; proc_stdio[2].flags = UV_IGNORE; } else { State = EXTERNCMD; if (opts & kShellOptWrite) { // Write from the current buffer into the process stdin uv_pipe_init(uv_default_loop(), &proc_stdin, 0); write_req.data = &pdata; proc_stdio[0].flags = UV_CREATE_PIPE | UV_READABLE_PIPE; proc_stdio[0].data.stream = (uv_stream_t *)&proc_stdin; } if (opts & kShellOptRead) { // Read from the process stdout into the current buffer uv_pipe_init(uv_default_loop(), &proc_stdout, 0); proc_stdout.data = &pdata; proc_stdio[1].flags = UV_CREATE_PIPE | UV_WRITABLE_PIPE; proc_stdio[1].data.stream = (uv_stream_t *)&proc_stdout; ga_init(&pdata.ga, 1, BUFFER_LENGTH); } } if (uv_spawn(uv_default_loop(), &proc, &proc_opts)) { // Failed, probably due to `sh` not being executable if (!emsg_silent) { MSG_PUTS(_("\nCannot execute shell ")); msg_outtrans(p_sh); msg_putchar('\n'); } return proc_cleanup_exit(&pdata, &proc_opts, opts); } // Assign the flag address after `proc` is initialized by `uv_spawn` proc.data = &pdata; if (opts & kShellOptWrite) { // Queue everything for writing to the shell stdin write_selection(&write_req); expected_exits++; } if (opts & kShellOptRead) { // Start the read stream for the shell stdout uv_read_start((uv_stream_t *)&proc_stdout, alloc_cb, read_cb); expected_exits++; } // Keep running the loop until all three handles are completely closed while (pdata.exited < expected_exits) { uv_run(uv_default_loop(), UV_RUN_ONCE); if (got_int) { // Forward SIGINT to the shell // TODO(tarruda): for now this is only needed if the terminal is in raw // mode, but when the UI is externalized we'll also need it, so leave it // here uv_process_kill(&proc, SIGINT); got_int = false; } } if (opts & kShellOptRead) { if (pdata.ga.ga_len > 0) { // If there's an unfinished line in the growable array, append it now. append_ga_line(&pdata.ga); // remember that the NL was missing curbuf->b_no_eol_lnum = curwin->w_cursor.lnum; } else { curbuf->b_no_eol_lnum = 0; } ga_clear(&pdata.ga); } if (opts & kShellOptWrite) { free(pdata.wbuffer); } return proc_cleanup_exit(&pdata, &proc_opts, opts); } static int tokenize(char_u *str, char **argv) { int argc = 0, len; char_u *p = str; while (*p != NUL) { len = word_length(p); if (argv != NULL) { // Fill the slot argv[argc] = xmalloc(len + 1); memcpy(argv[argc], p, len); argv[argc][len] = NUL; } argc++; p += len; p = skipwhite(p); } return argc; }