To: vim_dev@googlegroups.com Subject: Patch 7.3.240 Fcc: outbox From: Bram Moolenaar Mime-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ------------ Note: I haven't verified this works or even compiles. Please send me a patch if you see a problem and can fix it. Patch 7.3.240 Problem: External commands can't use pipes on MS-Windows. Solution: Implement pipes and use them when 'shelltemp' isn't set. (Vincent Berthoux) Files: src/eval.c, src/ex_cmds.c, src/misc2.c, src/os_unix.c, src/os_win32.c, src/proto/misc2.pro, src/ui.c *** ../vim-7.3.239/src/eval.c 2011-06-19 02:55:32.000000000 +0200 --- src/eval.c 2011-07-07 15:44:56.000000000 +0200 *************** *** 11931,11937 **** #ifdef FEAT_SEARCHPATH "file_in_path", #endif ! #if defined(UNIX) && !defined(USE_SYSTEM) "filterpipe", #endif #ifdef FEAT_FIND_ID --- 11931,11937 ---- #ifdef FEAT_SEARCHPATH "file_in_path", #endif ! #if (defined(UNIX) && !defined(USE_SYSTEM)) || defined(WIN3264) "filterpipe", #endif #ifdef FEAT_FIND_ID *** ../vim-7.3.239/src/ex_cmds.c 2011-06-12 22:03:15.000000000 +0200 --- src/ex_cmds.c 2011-07-07 15:44:56.000000000 +0200 *************** *** 1107,1113 **** if (do_out) shell_flags |= SHELL_DOOUT; ! #if !defined(USE_SYSTEM) && defined(UNIX) if (!do_in && do_out && !p_stmp) { /* Use a pipe to fetch stdout of the command, do not use a temp file. */ --- 1107,1113 ---- if (do_out) shell_flags |= SHELL_DOOUT; ! #if (!defined(USE_SYSTEM) && defined(UNIX)) || defined(WIN3264) if (!do_in && do_out && !p_stmp) { /* Use a pipe to fetch stdout of the command, do not use a temp file. */ *** ../vim-7.3.239/src/misc2.c 2011-07-07 15:08:53.000000000 +0200 --- src/misc2.c 2011-07-07 15:55:42.000000000 +0200 *************** *** 2146,2151 **** --- 2146,2170 ---- } } + #if (defined(UNIX) && !defined(USE_SYSTEM)) || defined(WIN3264) + /* + * Append the text in "gap" below the cursor line and clear "gap". + */ + void + append_ga_line(gap) + garray_T *gap; + { + /* Remove trailing CR. */ + if (gap->ga_len > 0 + && !curbuf->b_p_bin + && ((char_u *)gap->ga_data)[gap->ga_len - 1] == CAR) + --gap->ga_len; + ga_append(gap, NUL); + ml_append(curwin->w_cursor.lnum++, gap->ga_data, 0, FALSE); + gap->ga_len = 0; + } + #endif + /************************************************************************ * functions that use lookup tables for various things, generally to do with * special key codes. *** ../vim-7.3.239/src/os_unix.c 2011-04-11 16:56:29.000000000 +0200 --- src/os_unix.c 2011-07-07 15:54:58.000000000 +0200 *************** *** 3660,3686 **** /* Nothing to do. */ } - #ifndef USE_SYSTEM - static void append_ga_line __ARGS((garray_T *gap)); - - /* - * Append the text in "gap" below the cursor line and clear "gap". - */ - static void - append_ga_line(gap) - garray_T *gap; - { - /* Remove trailing CR. */ - if (gap->ga_len > 0 - && !curbuf->b_p_bin - && ((char_u *)gap->ga_data)[gap->ga_len - 1] == CAR) - --gap->ga_len; - ga_append(gap, NUL); - ml_append(curwin->w_cursor.lnum++, gap->ga_data, 0, FALSE); - gap->ga_len = 0; - } - #endif - int mch_call_shell(cmd, options) char_u *cmd; --- 3660,3665 ---- *** ../vim-7.3.239/src/os_win32.c 2011-05-25 17:06:16.000000000 +0200 --- src/os_win32.c 2011-07-07 16:08:30.000000000 +0200 *************** *** 417,422 **** --- 417,427 ---- static PGNSECINFO pGetNamedSecurityInfo; #endif + typedef BOOL (WINAPI *PSETHANDLEINFORMATION)(HANDLE, DWORD, DWORD); + + static BOOL allowPiping = FALSE; + static PSETHANDLEINFORMATION pSetHandleInformation; + /* * Set g_PlatformId to VER_PLATFORM_WIN32_NT (NT) or * VER_PLATFORM_WIN32_WINDOWS (Win95). *************** *** 467,472 **** --- 472,489 ---- } } #endif + /* + * If we are on windows NT, try to load the pipe functions, only + * available from Win2K. + */ + if (g_PlatformId == VER_PLATFORM_WIN32_NT) + { + HANDLE kernel32 = GetModuleHandle("kernel32"); + pSetHandleInformation = (PSETHANDLEINFORMATION)GetProcAddress( + kernel32, "SetHandleInformation"); + + allowPiping = pSetHandleInformation != NULL; + } done = TRUE; } } *************** *** 1635,1641 **** } #if ((defined(__MINGW32__) || defined (__CYGWIN32__)) && \ ! __MSVCRT_VERSION__ >= 0x800) || (defined(_MSC_VER) && _MSC_VER >= 1400) /* * Bad parameter handler. * --- 1652,1658 ---- } #if ((defined(__MINGW32__) || defined (__CYGWIN32__)) && \ ! __MSVCRT_VERSION__ >= 0x800) || (defined(_MSC_VER) && _MSC_VER >= 1400) /* * Bad parameter handler. * *************** *** 3210,3216 **** * 4. Prompt the user to press a key to close the console window */ static int ! mch_system(char *cmd, int options) { STARTUPINFO si; PROCESS_INFORMATION pi; --- 3227,3233 ---- * 4. Prompt the user to press a key to close the console window */ static int ! mch_system_classic(char *cmd, int options) { STARTUPINFO si; PROCESS_INFORMATION pi; *************** *** 3315,3320 **** --- 3332,3829 ---- return ret; } + + /* + * Thread launched by the gui to send the current buffer data to the + * process. This way avoid to hang up vim totally if the children + * process take a long time to process the lines. + */ + static DWORD WINAPI + sub_process_writer(LPVOID param) + { + HANDLE g_hChildStd_IN_Wr = param; + linenr_T lnum = curbuf->b_op_start.lnum; + DWORD len = 0; + DWORD l; + char_u *lp = ml_get(lnum); + char_u *s; + int written = 0; + + for (;;) + { + l = (DWORD)STRLEN(lp + written); + if (l == 0) + len = 0; + else if (lp[written] == NL) + { + /* NL -> NUL translation */ + WriteFile(g_hChildStd_IN_Wr, "", 1, &len, NULL); + } + else + { + s = vim_strchr(lp + written, NL); + WriteFile(g_hChildStd_IN_Wr, (char *)lp + written, + s == NULL ? l : (DWORD)(s - (lp + written)), + &len, NULL); + } + if (len == (int)l) + { + /* Finished a line, add a NL, unless this line should not have + * one. */ + if (lnum != curbuf->b_op_end.lnum + || !curbuf->b_p_bin + || (lnum != curbuf->b_no_eol_lnum + && (lnum != curbuf->b_ml.ml_line_count + || curbuf->b_p_eol))) + { + WriteFile(g_hChildStd_IN_Wr, "\n", 1, &ignored, NULL); + } + + ++lnum; + if (lnum > curbuf->b_op_end.lnum) + break; + + lp = ml_get(lnum); + written = 0; + } + else if (len > 0) + written += len; + } + + /* finished all the lines, close pipe */ + CloseHandle(g_hChildStd_IN_Wr); + ExitThread(0); + } + + + # define BUFLEN 100 /* length for buffer, stolen from unix version */ + + /* + * This function read from the children's stdout and write the + * data on screen or in the buffer accordingly. + */ + static void + dump_pipe(int options, + HANDLE g_hChildStd_OUT_Rd, + garray_T *ga, + char_u buffer[], + DWORD *buffer_off) + { + DWORD availableBytes = 0; + DWORD i; + int c; + char_u *p; + int ret; + DWORD len; + DWORD toRead; + int repeatCount; + + /* we query the pipe to see if there is any data to read + * to avoid to perform a blocking read */ + ret = PeekNamedPipe(g_hChildStd_OUT_Rd, /* pipe to query */ + NULL, /* optional buffer */ + 0, /* buffe size */ + NULL, /* number of read bytes */ + &availableBytes, /* available bytes total */ + NULL); /* byteLeft */ + + repeatCount = 0; + /* We got real data in the pipe, read it */ + while (ret != 0 && availableBytes > 0 && availableBytes > 0) + { + repeatCount++; + toRead = + # ifdef FEAT_MBYTE + (DWORD)(BUFLEN - *buffer_off); + # else + (DWORD)BUFLEN; + # endif + toRead = availableBytes < toRead ? availableBytes : toRead; + ReadFile(g_hChildStd_OUT_Rd, buffer + # ifdef FEAT_MBYTE + + *buffer_off, toRead + # else + , toRead + # endif + , &len, NULL); + + /* If we haven't read anything, there is a problem */ + if (len == 0) + break; + + availableBytes -= len; + + if (options & SHELL_READ) + { + /* Do NUL -> NL translation, append NL separated + * lines to the current buffer. */ + for (i = 0; i < len; ++i) + { + if (buffer[i] == NL) + append_ga_line(ga); + else if (buffer[i] == NUL) + ga_append(ga, NL); + else + ga_append(ga, buffer[i]); + } + } + # ifdef FEAT_MBYTE + else if (has_mbyte) + { + int l; + + len += *buffer_off; + buffer[len] = NUL; + + /* Check if the last character in buffer[] is + * incomplete, keep these bytes for the next + * round. */ + for (p = buffer; p < buffer + len; p += l) + { + l = mb_cptr2len(p); + if (l == 0) + l = 1; /* NUL byte? */ + else if (MB_BYTE2LEN(*p) != l) + break; + } + if (p == buffer) /* no complete character */ + { + /* avoid getting stuck at an illegal byte */ + if (len >= 12) + ++p; + else + { + *buffer_off = len; + return; + } + } + c = *p; + *p = NUL; + msg_puts(buffer); + if (p < buffer + len) + { + *p = c; + *buffer_off = (DWORD)((buffer + len) - p); + mch_memmove(buffer, p, *buffer_off); + return; + } + *buffer_off = 0; + } + # endif /* FEAT_MBYTE */ + else + { + buffer[len] = NUL; + msg_puts(buffer); + } + + windgoto(msg_row, msg_col); + cursor_on(); + out_flush(); + } + } + + /* + * Version of system to use for windows NT > 5.0 (Win2K), use pipe + * for communication and doesn't open any new window. + */ + static int + mch_system_piped(char *cmd, int options) + { + STARTUPINFO si; + PROCESS_INFORMATION pi; + DWORD ret = 0; + + HANDLE g_hChildStd_IN_Rd = NULL; + HANDLE g_hChildStd_IN_Wr = NULL; + HANDLE g_hChildStd_OUT_Rd = NULL; + HANDLE g_hChildStd_OUT_Wr = NULL; + + char_u buffer[BUFLEN + 1]; /* reading buffer + size */ + DWORD len; + + /* buffer used to receive keys */ + char_u ta_buf[BUFLEN + 1]; /* TypeAHead */ + int ta_len = 0; /* valid bytes in ta_buf[] */ + + DWORD i; + int c; + int noread_cnt = 0; + garray_T ga; + int delay = 1; + # ifdef FEAT_MBYTE + DWORD buffer_off = 0; /* valid bytes in buffer[] */ + # endif + + SECURITY_ATTRIBUTES saAttr; + + /* Set the bInheritHandle flag so pipe handles are inherited. */ + saAttr.nLength = sizeof(SECURITY_ATTRIBUTES); + saAttr.bInheritHandle = TRUE; + saAttr.lpSecurityDescriptor = NULL; + + if ( ! CreatePipe(&g_hChildStd_OUT_Rd, &g_hChildStd_OUT_Wr, &saAttr, 0) + /* Ensure the read handle to the pipe for STDOUT is not inherited. */ + || ! pSetHandleInformation(g_hChildStd_OUT_Rd, HANDLE_FLAG_INHERIT, 0) + /* Create a pipe for the child process's STDIN. */ + || ! CreatePipe(&g_hChildStd_IN_Rd, &g_hChildStd_IN_Wr, &saAttr, 0) + /* Ensure the write handle to the pipe for STDIN is not inherited. */ + || ! pSetHandleInformation(g_hChildStd_IN_Wr, HANDLE_FLAG_INHERIT, 0) ) + { + CloseHandle(g_hChildStd_IN_Rd); + CloseHandle(g_hChildStd_IN_Wr); + CloseHandle(g_hChildStd_OUT_Rd); + CloseHandle(g_hChildStd_OUT_Wr); + MSG_PUTS(_("\nCannot create pipes\n")); + } + + si.cb = sizeof(si); + si.lpReserved = NULL; + si.lpDesktop = NULL; + si.lpTitle = NULL; + si.dwFlags = STARTF_USESHOWWINDOW | STARTF_USESTDHANDLES; + + /* set-up our file redirection */ + si.hStdError = g_hChildStd_OUT_Wr; + si.hStdOutput = g_hChildStd_OUT_Wr; + si.hStdInput = g_hChildStd_IN_Rd; + si.wShowWindow = SW_HIDE; + si.cbReserved2 = 0; + si.lpReserved2 = NULL; + + if (options & SHELL_READ) + ga_init2(&ga, 1, BUFLEN); + + /* Now, run the command */ + CreateProcess(NULL, /* Executable name */ + cmd, /* Command to execute */ + NULL, /* Process security attributes */ + NULL, /* Thread security attributes */ + + // this command can be litigeous, handle inheritence was + // deactivated for pending temp file, but, if we deactivate + // it, the pipes don't work for some reason. + TRUE, /* Inherit handles, first deactivated, + * but needed */ + CREATE_DEFAULT_ERROR_MODE, /* Creation flags */ + NULL, /* Environment */ + NULL, /* Current directory */ + &si, /* Startup information */ + &pi); /* Process information */ + + + /* Close our unused side of the pipes */ + CloseHandle(g_hChildStd_IN_Rd); + CloseHandle(g_hChildStd_OUT_Wr); + + if (options & SHELL_WRITE) + { + HANDLE thread = + CreateThread(NULL, /* security attributes */ + 0, /* default stack size */ + sub_process_writer, /* function to be executed */ + g_hChildStd_IN_Wr, /* parameter */ + 0, /* creation flag, start immediately */ + NULL); /* we don't care about thread id */ + CloseHandle(thread); + g_hChildStd_IN_Wr = NULL; + } + + /* Keep updating the window while waiting for the shell to finish. */ + for (;;) + { + MSG msg; + + if (PeekMessage(&msg, (HWND)NULL, 0, 0, PM_REMOVE)) + { + TranslateMessage(&msg); + DispatchMessage(&msg); + } + + /* write pipe information in the window */ + if ((options & (SHELL_READ|SHELL_WRITE)) + # ifdef FEAT_GUI + || gui.in_use + # endif + ) + { + len = 0; + if (!(options & SHELL_EXPAND) + && ((options & + (SHELL_READ|SHELL_WRITE|SHELL_COOKED)) + != (SHELL_READ|SHELL_WRITE|SHELL_COOKED) + # ifdef FEAT_GUI + || gui.in_use + # endif + ) + && (ta_len > 0 || noread_cnt > 4)) + { + if (ta_len == 0) + { + /* Get extra characters when we don't have any. Reset the + * counter and timer. */ + noread_cnt = 0; + # if defined(HAVE_GETTIMEOFDAY) && defined(HAVE_SYS_TIME_H) + gettimeofday(&start_tv, NULL); + # endif + len = ui_inchar(ta_buf, BUFLEN, 10L, 0); + } + if (ta_len > 0 || len > 0) + { + /* + * For pipes: Check for CTRL-C: send interrupt signal to + * child. Check for CTRL-D: EOF, close pipe to child. + */ + if (len == 1 && cmd != NULL) + { + if (ta_buf[ta_len] == Ctrl_C) + { + /* Learn what exit code is expected, for + * now put 9 as SIGKILL */ + TerminateProcess(pi.hProcess, 9); + } + if (ta_buf[ta_len] == Ctrl_D) + { + CloseHandle(g_hChildStd_IN_Wr); + g_hChildStd_IN_Wr = NULL; + } + } + + /* replace K_BS by and K_DEL by */ + for (i = ta_len; i < ta_len + len; ++i) + { + if (ta_buf[i] == CSI && len - i > 2) + { + c = TERMCAP2KEY(ta_buf[i + 1], ta_buf[i + 2]); + if (c == K_DEL || c == K_KDEL || c == K_BS) + { + mch_memmove(ta_buf + i + 1, ta_buf + i + 3, + (size_t)(len - i - 2)); + if (c == K_DEL || c == K_KDEL) + ta_buf[i] = DEL; + else + ta_buf[i] = Ctrl_H; + len -= 2; + } + } + else if (ta_buf[i] == '\r') + ta_buf[i] = '\n'; + # ifdef FEAT_MBYTE + if (has_mbyte) + i += (*mb_ptr2len_len)(ta_buf + i, + ta_len + len - i) - 1; + # endif + } + + /* + * For pipes: echo the typed characters. For a pty this + * does not seem to work. + */ + for (i = ta_len; i < ta_len + len; ++i) + { + if (ta_buf[i] == '\n' || ta_buf[i] == '\b') + msg_putchar(ta_buf[i]); + # ifdef FEAT_MBYTE + else if (has_mbyte) + { + int l = (*mb_ptr2len)(ta_buf + i); + + msg_outtrans_len(ta_buf + i, l); + i += l - 1; + } + # endif + else + msg_outtrans_len(ta_buf + i, 1); + } + windgoto(msg_row, msg_col); + out_flush(); + + ta_len += len; + + /* + * Write the characters to the child, unless EOF has been + * typed for pipes. Write one character at a time, to + * avoid losing too much typeahead. When writing buffer + * lines, drop the typed characters (only check for + * CTRL-C). + */ + if (options & SHELL_WRITE) + ta_len = 0; + else if (g_hChildStd_IN_Wr != NULL) + { + WriteFile(g_hChildStd_IN_Wr, (char*)ta_buf, + 1, &len, NULL); + // if we are typing in, we want to keep things reactive + delay = 1; + if (len > 0) + { + ta_len -= len; + mch_memmove(ta_buf, ta_buf + len, ta_len); + } + } + } + } + } + + if (ta_len) + ui_inchar_undo(ta_buf, ta_len); + + if (WaitForSingleObject(pi.hProcess, delay) != WAIT_TIMEOUT) + { + dump_pipe(options, g_hChildStd_OUT_Rd, + &ga, buffer, &buffer_off); + break; + } + + ++noread_cnt; + dump_pipe(options, g_hChildStd_OUT_Rd, + &ga, buffer, &buffer_off); + + /* We start waiting for a very short time and then increase it, so + * that we respond quickly when the process is quick, and don't + * consume too much overhead when it's slow. */ + if (delay < 50) + delay += 10; + } + + /* Close the pipe */ + CloseHandle(g_hChildStd_OUT_Rd); + if (g_hChildStd_IN_Wr != NULL) + CloseHandle(g_hChildStd_IN_Wr); + + WaitForSingleObject(pi.hProcess, INFINITE); + + /* Get the command exit code */ + GetExitCodeProcess(pi.hProcess, &ret); + + if (options & SHELL_READ) + { + if (ga.ga_len > 0) + { + append_ga_line(&ga); + /* remember that the NL was missing */ + curbuf->b_no_eol_lnum = curwin->w_cursor.lnum; + } + else + curbuf->b_no_eol_lnum = 0; + ga_clear(&ga); + } + + /* Close the handles to the subprocess, so that it goes away */ + CloseHandle(pi.hThread); + CloseHandle(pi.hProcess); + + return ret; + } + + static int + mch_system(char *cmd, int options) + { + /* if we can pipe and the shelltemp option is off */ + if (allowPiping && !p_stmp) + return mch_system_piped(cmd, options); + else + return mch_system_classic(cmd, options); + } #else # define mch_system(c, o) system(c) *************** *** 3388,3394 **** char_u *newcmd; long_u cmdlen = ( #ifdef FEAT_GUI_W32 ! STRLEN(vimrun_path) + #endif STRLEN(p_sh) + STRLEN(p_shcf) + STRLEN(cmd) + 10); --- 3897,3903 ---- char_u *newcmd; long_u cmdlen = ( #ifdef FEAT_GUI_W32 ! (allowPiping && !p_stmp ? 0 : STRLEN(vimrun_path)) + #endif STRLEN(p_sh) + STRLEN(p_shcf) + STRLEN(cmd) + 10); *************** *** 3497,3503 **** MB_ICONWARNING); need_vimrun_warning = FALSE; } ! if (!s_dont_use_vimrun) /* Use vimrun to execute the command. It opens a console * window, which can be closed without killing Vim. */ vim_snprintf((char *)newcmd, cmdlen, "%s%s%s %s %s", --- 4006,4012 ---- MB_ICONWARNING); need_vimrun_warning = FALSE; } ! if (!s_dont_use_vimrun && (!allowPiping || p_stmp)) /* Use vimrun to execute the command. It opens a console * window, which can be closed without killing Vim. */ vim_snprintf((char *)newcmd, cmdlen, "%s%s%s %s %s", *************** *** 3521,3527 **** /* Print the return value, unless "vimrun" was used. */ if (x != 0 && !(options & SHELL_SILENT) && !emsg_silent #if defined(FEAT_GUI_W32) ! && ((options & SHELL_DOOUT) || s_dont_use_vimrun) #endif ) { --- 4030,4037 ---- /* Print the return value, unless "vimrun" was used. */ if (x != 0 && !(options & SHELL_SILENT) && !emsg_silent #if defined(FEAT_GUI_W32) ! && ((options & SHELL_DOOUT) || s_dont_use_vimrun ! || (allowPiping && !p_stmp)) #endif ) { *** ../vim-7.3.239/src/proto/misc2.pro 2011-07-07 15:08:53.000000000 +0200 --- src/proto/misc2.pro 2011-07-07 15:56:16.000000000 +0200 *************** *** 58,63 **** --- 58,64 ---- char_u *ga_concat_strings __ARGS((garray_T *gap)); void ga_concat __ARGS((garray_T *gap, char_u *s)); void ga_append __ARGS((garray_T *gap, int c)); + void append_ga_line __ARGS((garray_T *gap)); int name_to_mod_mask __ARGS((int c)); int simplify_key __ARGS((int key, int *modifiers)); int handle_x_keys __ARGS((int key)); *** ../vim-7.3.239/src/ui.c 2011-06-19 01:14:23.000000000 +0200 --- src/ui.c 2011-07-07 15:44:56.000000000 +0200 *************** *** 58,64 **** #endif } ! #if defined(UNIX) || defined(VMS) || defined(PROTO) /* * When executing an external program, there may be some typed characters that * are not consumed by it. Give them back to ui_inchar() and they are stored --- 58,64 ---- #endif } ! #if defined(UNIX) || defined(VMS) || defined(PROTO) || defined(WIN3264) /* * When executing an external program, there may be some typed characters that * are not consumed by it. Give them back to ui_inchar() and they are stored *** ../vim-7.3.239/src/version.c 2011-07-07 15:08:53.000000000 +0200 --- src/version.c 2011-07-07 16:14:20.000000000 +0200 *************** *** 711,712 **** --- 711,714 ---- { /* Add new patch number below this line */ + /**/ + 240, /**/ -- hundred-and-one symptoms of being an internet addict: 257. Your "hundred-and-one" lists include well over 101 items, since you automatically interpret all numbers in hexadecimal notation. (hex 101 = decimal 257) /// Bram Moolenaar -- Bram@Moolenaar.net -- http://www.Moolenaar.net \\\ /// sponsor Vim, vote for features -- http://www.Vim.org/sponsor/ \\\ \\\ an exciting new programming language -- http://www.Zimbu.org /// \\\ help me help AIDS victims -- http://ICCF-Holland.org ///