To: vim_dev@googlegroups.com
Subject: Patch 8.1.2123
Fcc: outbox
From: Bram Moolenaar <Bram@moolenaar.net>
Mime-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
------------

Patch 8.1.2123
Problem:    Parsing CSI sequence is messy.
Solution:   Generalize parsing a CSI sequence.
Files:	    src/term.c


*** ../vim-8.1.2122/src/term.c	2019-09-23 21:16:51.387544361 +0200
--- src/term.c	2019-10-08 20:14:04.127049676 +0200
***************
*** 4435,4506 ****
  # endif
  	   )
  	{
! 	    /* Check for some responses from the terminal starting with
! 	     * "<Esc>[" or CSI:
  	     *
! 	     * - Xterm version string: <Esc>[>{x};{vers};{y}c
! 	     *   Libvterm returns {x} == 0, {vers} == 100, {y} == 0.
  	     *   Also eat other possible responses to t_RV, rxvt returns
! 	     *   "<Esc>[?1;2c". Also accept CSI instead of <Esc>[.
! 	     *   mrxvt has been reported to have "+" in the version. Assume
! 	     *   the escape sequence ends with a letter or one of "{|}~".
  	     *
! 	     * - Cursor position report: <Esc>[{row};{col}R
  	     *   The final byte must be 'R'. It is used for checking the
  	     *   ambiguous-width character state.
  	     *
! 	     * - window position reply: <Esc>[3;{x};{y}t
  	     */
! 	    char_u *argp = tp[0] == ESC ? tp + 2 : tp + 1;
! 
! 	    if ((*T_CRV != NUL || *T_U7 != NUL || did_request_winpos)
! 			&& ((tp[0] == ESC && len >= 3 && tp[1] == '[')
  			    || (tp[0] == CSI && len >= 2))
! 			&& (VIM_ISDIGIT(*argp) || *argp == '>' || *argp == '?'))
  	    {
! 		int col = 0;
! 		int semicols = 0;
! 		int row_char = NUL;
! 
! 		extra = 0;
! 		for (i = 2 + (tp[0] != CSI); i < len
! 				&& !(tp[i] >= '{' && tp[i] <= '~')
! 				&& !ASCII_ISALPHA(tp[i]); ++i)
! 		    if (tp[i] == ';' && ++semicols == 1)
  		    {
! 			extra = i + 1;
! 			row_char = tp[i - 1];
  		    }
! 		if (i == len)
! 		{
! 		    LOG_TR(("Not enough characters for CRV"));
! 		    return -1;
  		}
! 		if (extra > 0)
! 		    col = atoi((char *)tp + extra);
! 
! 		/* Eat it when it has 2 arguments and ends in 'R'. Also when
! 		 * u7_status is not "sent", it may be from a previous Vim that
! 		 * just exited.  But not for <S-F3>, it sends something
! 		 * similar, check for row and column to make sense. */
! 		if (semicols == 1 && tp[i] == 'R')
  		{
! 		    if (row_char == '2' && col >= 2)
  		    {
  			char *aw = NULL;
  
  			LOG_TR(("Received U7 status: %s", tp));
  			u7_status.tr_progress = STATUS_GOT;
  			did_cursorhold = TRUE;
! 			if (col == 2)
  			    aw = "single";
! 			else if (col == 3)
  			    aw = "double";
  			if (aw != NULL && STRCMP(aw, p_ambw) != 0)
  			{
! 			    /* Setting the option causes a screen redraw. Do
! 			     * that right away if possible, keeping any
! 			     * messages. */
  			    set_option_value((char_u *)"ambw", 0L,
  					     (char_u *)aw, 0);
  # ifdef DEBUG_TERMRESPONSE
--- 4435,4539 ----
  # endif
  	   )
  	{
! 	    char_u *argp = tp[0] == ESC ? tp + 2 : tp + 1;
! 
! 	    /*
! 	     * Check for responses from the terminal starting with {lead}:
! 	     * "<Esc>[" or CSI followed by [0-9>?]
  	     *
! 	     * - Xterm version string: {lead}>{x};{vers};{y}c
  	     *   Also eat other possible responses to t_RV, rxvt returns
! 	     *   "{lead}?1;2c".
  	     *
! 	     * - Cursor position report: {lead}{row};{col}R
  	     *   The final byte must be 'R'. It is used for checking the
  	     *   ambiguous-width character state.
  	     *
! 	     * - window position reply: {lead}3;{x};{y}t
! 	     *
! 	     * - key with modifiers when modifyOtherKeys is enabled:
! 	     *	    {lead}27;{modifier};{key}~
! 	     *	    {lead}{key};{modifier}u
  	     */
! 	    if (((tp[0] == ESC && len >= 3 && tp[1] == '[')
  			    || (tp[0] == CSI && len >= 2))
! 		    && (VIM_ISDIGIT(*argp) || *argp == '>' || *argp == '?'))
  	    {
! 		int	first = -1;  // optional char right after {lead}
! 		int	trail;	     // char that ends CSI sequence
! 		int	arg[3] = {-1, -1, -1};	// argument numbers
! 		int	argc;			// number of arguments
! 		char_u	*ap = argp;
! 		int	csi_len;
! 
! 		// Check for non-digit after CSI.
! 		if (!VIM_ISDIGIT(*ap))
! 		    first = *ap++;
! 
! 		// Find up to three argument numbers.
! 		for (argc = 0; argc < 3; )
! 		{
! 		    if (ap >= tp + len)
  		    {
! not_enough:
! 			LOG_TR(("Not enough characters for CSI sequence"));
! 			return -1;
  		    }
! 		    if (*ap == ';')
! 			arg[argc++] = -1;  // omitted number
! 		    else if (VIM_ISDIGIT(*ap))
! 		    {
! 			arg[argc] = 0;
! 			for (;;)
! 			{
! 			    if (ap >= tp + len)
! 				goto not_enough;
! 			    if (!VIM_ISDIGIT(*ap))
! 				break;
! 			    arg[argc] = arg[argc] * 10 + (*ap - '0');
! 			    ++ap;
! 			}
! 			++argc;
! 		    }
! 		    if (*ap == ';')
! 			++ap;
! 		    else
! 			break;
  		}
! 		// mrxvt has been reported to have "+" in the version. Assume
! 		// the escape sequence ends with a letter or one of "{|}~".
! 		while (ap < tp + len
! 			&& !(*ap >= '{' && *ap <= '~')
! 			&& !ASCII_ISALPHA(*ap))
! 		    ++ap;
! 		if (ap >= tp + len)
! 		    goto not_enough;
! 		trail = *ap;
! 		csi_len = (int)(ap - tp) + 1;
! 
! 		// Cursor position report: Eat it when there are 2 arguments
! 		// and it ends in 'R'. Also when u7_status is not "sent", it
! 		// may be from a previous Vim that just exited.  But not for
! 		// <S-F3>, it sends something similar, check for row and column
! 		// to make sense.
! 		if (first == -1 && argc == 2 && trail == 'R')
  		{
! 		    if (arg[0] == 2 && arg[1] >= 2)
  		    {
  			char *aw = NULL;
  
  			LOG_TR(("Received U7 status: %s", tp));
  			u7_status.tr_progress = STATUS_GOT;
  			did_cursorhold = TRUE;
! 			if (arg[1] == 2)
  			    aw = "single";
! 			else if (arg[1] == 3)
  			    aw = "double";
  			if (aw != NULL && STRCMP(aw, p_ambw) != 0)
  			{
! 			    // Setting the option causes a screen redraw. Do
! 			    // that right away if possible, keeping any
! 			    // messages.
  			    set_option_value((char_u *)"ambw", 0L,
  					     (char_u *)aw, 0);
  # ifdef DEBUG_TERMRESPONSE
***************
*** 4516,4549 ****
  		    }
  		    key_name[0] = (int)KS_EXTRA;
  		    key_name[1] = (int)KE_IGNORE;
! 		    slen = i + 1;
  # ifdef FEAT_EVAL
  		    set_vim_var_string(VV_TERMU7RESP, tp, slen);
  # endif
  		}
! 		/* eat it when at least one digit and ending in 'c' */
! 		else if (*T_CRV != NUL && i > 2 + (tp[0] != CSI)
! 							       && tp[i] == 'c')
  		{
! 		    int version = col;
  
  		    LOG_TR(("Received CRV response: %s", tp));
  		    crv_status.tr_progress = STATUS_GOT;
  		    did_cursorhold = TRUE;
  
! 		    /* If this code starts with CSI, you can bet that the
! 		     * terminal uses 8-bit codes. */
  		    if (tp[0] == CSI)
  			switch_to_8bit();
  
! 		    /* rxvt sends its version number: "20703" is 2.7.3.
! 		     * Screen sends 40500.
! 		     * Ignore it for when the user has set 'term' to xterm,
! 		     * even though it's an rxvt. */
  		    if (version > 20000)
  			version = 0;
  
! 		    if (tp[1 + (tp[0] != CSI)] == '>' && semicols == 2)
  		    {
  			int need_flush = FALSE;
  			int is_iterm2 = FALSE;
--- 4549,4583 ----
  		    }
  		    key_name[0] = (int)KS_EXTRA;
  		    key_name[1] = (int)KE_IGNORE;
! 		    slen = csi_len;
  # ifdef FEAT_EVAL
  		    set_vim_var_string(VV_TERMU7RESP, tp, slen);
  # endif
  		}
! 
! 		// Version string: Eat it when there is at least one digit and
! 		// it ends in 'c'
! 		else if (*T_CRV != NUL && ap > argp + 1 && trail == 'c')
  		{
! 		    int version = arg[1];
  
  		    LOG_TR(("Received CRV response: %s", tp));
  		    crv_status.tr_progress = STATUS_GOT;
  		    did_cursorhold = TRUE;
  
! 		    // If this code starts with CSI, you can bet that the
! 		    // terminal uses 8-bit codes.
  		    if (tp[0] == CSI)
  			switch_to_8bit();
  
! 		    // rxvt sends its version number: "20703" is 2.7.3.
! 		    // Screen sends 40500.
! 		    // Ignore it for when the user has set 'term' to xterm,
! 		    // even though it's an rxvt.
  		    if (version > 20000)
  			version = 0;
  
! 		    if (first == '>' && argc == 3)
  		    {
  			int need_flush = FALSE;
  			int is_iterm2 = FALSE;
***************
*** 4551,4560 ****
  
  			// mintty 2.9.5 sends 77;20905;0c.
  			// (77 is ASCII 'M' for mintty.)
! 			if (STRNCMP(tp + extra - 3, "77;", 3) == 0)
  			    is_mintty = TRUE;
  
! 			/* if xterm version >= 141 try to get termcap codes */
  			if (version >= 141)
  			{
  			    LOG_TR(("Enable checking for XT codes"));
--- 4585,4594 ----
  
  			// mintty 2.9.5 sends 77;20905;0c.
  			// (77 is ASCII 'M' for mintty.)
! 			if (arg[0] == 77)
  			    is_mintty = TRUE;
  
! 			// if xterm version >= 141 try to get termcap codes
  			if (version >= 141)
  			{
  			    LOG_TR(("Enable checking for XT codes"));
***************
*** 4563,4575 ****
  			    req_codes_from_term();
  			}
  
! 			/* libvterm sends 0;100;0 */
! 			if (version == 100
! 				&& STRNCMP(tp + extra - 2, "0;100;0c", 8) == 0)
  			{
! 			    /* If run from Vim $COLORS is set to the number of
! 			     * colors the terminal supports.  Otherwise assume
! 			     * 256, libvterm supports even more. */
  			    if (mch_getenv((char_u *)"COLORS") == NULL)
  				may_adjust_color_count(256);
  			    /* Libvterm can handle SGR mouse reporting. */
--- 4597,4608 ----
  			    req_codes_from_term();
  			}
  
! 			// libvterm sends 0;100;0
! 			if (version == 100 && arg[0] == 0 && arg[2] == 0)
  			{
! 			    // If run from Vim $COLORS is set to the number of
! 			    // colors the terminal supports.  Otherwise assume
! 			    // 256, libvterm supports even more.
  			    if (mch_getenv((char_u *)"COLORS") == NULL)
  				may_adjust_color_count(256);
  			    /* Libvterm can handle SGR mouse reporting. */
***************
*** 4581,4636 ****
  			if (version == 95)
  			{
  			    // Mac Terminal.app sends 1;95;0
! 			    if (STRNCMP(tp + extra - 2, "1;95;0c", 7) == 0)
  			    {
  				is_not_xterm = TRUE;
  				is_mac_terminal = TRUE;
  			    }
  			    // iTerm2 sends 0;95;0
! 			    if (STRNCMP(tp + extra - 2, "0;95;0c", 7) == 0)
  				is_iterm2 = TRUE;
  			    // old iTerm2 sends 0;95;
! 			    else if (STRNCMP(tp + extra - 2, "0;95;c", 6) == 0)
  				is_not_xterm = TRUE;
  			}
  
! 			/* Only set 'ttymouse' automatically if it was not set
! 			 * by the user already. */
  			if (!option_was_set((char_u *)"ttym"))
  			{
! 			    /* Xterm version 277 supports SGR.  Also support
! 			     * Terminal.app, iTerm2 and mintty. */
  			    if (version >= 277 || is_iterm2 || is_mac_terminal
  				    || is_mintty)
  				set_option_value((char_u *)"ttym", 0L,
  							  (char_u *)"sgr", 0);
! 			    /* if xterm version >= 95 use mouse dragging */
  			    else if (version >= 95)
  				set_option_value((char_u *)"ttym", 0L,
  						       (char_u *)"xterm2", 0);
  			}
  
! 			/* Detect terminals that set $TERM to something like
! 			 * "xterm-256colors"  but are not fully xterm
! 			 * compatible. */
! 
! 			/* Gnome terminal sends 1;3801;0, 1;4402;0 or 1;2501;0.
! 			 * xfce4-terminal sends 1;2802;0.
! 			 * screen sends 83;40500;0
! 			 * Assuming any version number over 2500 is not an
! 			 * xterm (without the limit for rxvt and screen). */
! 			if (col >= 2500)
  			    is_not_xterm = TRUE;
  
! 			/* PuTTY sends 0;136;0
! 			 * vandyke SecureCRT sends 1;136;0 */
! 			if (version == 136
! 				&& STRNCMP(tp + extra - 1, ";136;0c", 7) == 0)
  			    is_not_xterm = TRUE;
  
! 			/* Konsole sends 0;115;0 */
! 			if (version == 115
! 				&& STRNCMP(tp + extra - 2, "0;115;0c", 8) == 0)
  			    is_not_xterm = TRUE;
  
  			// Xterm first responded to this request at patch level
--- 4614,4667 ----
  			if (version == 95)
  			{
  			    // Mac Terminal.app sends 1;95;0
! 			    if (arg[0] == 1 && arg[2] == 0)
  			    {
  				is_not_xterm = TRUE;
  				is_mac_terminal = TRUE;
  			    }
  			    // iTerm2 sends 0;95;0
! 			    else if (arg[0] == 0 && arg[2] == 0)
  				is_iterm2 = TRUE;
  			    // old iTerm2 sends 0;95;
! 			    else if (arg[0] == 0 && arg[2] == -1)
  				is_not_xterm = TRUE;
  			}
  
! 			// Only set 'ttymouse' automatically if it was not set
! 			// by the user already.
  			if (!option_was_set((char_u *)"ttym"))
  			{
! 			    // Xterm version 277 supports SGR.  Also support
! 			    // Terminal.app, iTerm2 and mintty.
  			    if (version >= 277 || is_iterm2 || is_mac_terminal
  				    || is_mintty)
  				set_option_value((char_u *)"ttym", 0L,
  							  (char_u *)"sgr", 0);
! 			    // if xterm version >= 95 use mouse dragging
  			    else if (version >= 95)
  				set_option_value((char_u *)"ttym", 0L,
  						       (char_u *)"xterm2", 0);
  			}
  
! 			// Detect terminals that set $TERM to something like
! 			// "xterm-256colors"  but are not fully xterm
! 			// compatible.
! 
! 			// Gnome terminal sends 1;3801;0, 1;4402;0 or 1;2501;0.
! 			// xfce4-terminal sends 1;2802;0.
! 			// screen sends 83;40500;0
! 			// Assuming any version number over 2500 is not an
! 			// xterm (without the limit for rxvt and screen).
! 			if (arg[1] >= 2500)
  			    is_not_xterm = TRUE;
  
! 			// PuTTY sends 0;136;0
! 			// vandyke SecureCRT sends 1;136;0
! 			else if (version == 136 && arg[2] == 0)
  			    is_not_xterm = TRUE;
  
! 			// Konsole sends 0;115;0
! 			else if (version == 115 && arg[0] == 0 && arg[2] == 0)
  			    is_not_xterm = TRUE;
  
  			// Xterm first responded to this request at patch level
***************
*** 4638,4648 ****
  			if (version < 95)
  			    is_not_xterm = TRUE;
  
! 			/* Only request the cursor style if t_SH and t_RS are
! 			 * set. Only supported properly by xterm since version
! 			 * 279 (otherwise it returns 0x18).
! 			 * Not for Terminal.app, it can't handle t_RS, it
! 			 * echoes the characters to the screen. */
  			if (rcs_status.tr_progress == STATUS_GET
  				&& version >= 279
  				&& !is_not_xterm
--- 4669,4679 ----
  			if (version < 95)
  			    is_not_xterm = TRUE;
  
! 			// Only request the cursor style if t_SH and t_RS are
! 			// set. Only supported properly by xterm since version
! 			// 279 (otherwise it returns 0x18).
! 			// Not for Terminal.app, it can't handle t_RS, it
! 			// echoes the characters to the screen.
  			if (rcs_status.tr_progress == STATUS_GET
  				&& version >= 279
  				&& !is_not_xterm
***************
*** 4655,4663 ****
  			    need_flush = TRUE;
  			}
  
! 			/* Only request the cursor blink mode if t_RC set. Not
! 			 * for Gnome terminal, it can't handle t_RC, it
! 			 * echoes the characters to the screen. */
  			if (rbm_status.tr_progress == STATUS_GET
  				&& !is_not_xterm
  				&& *T_CRC != NUL)
--- 4686,4694 ----
  			    need_flush = TRUE;
  			}
  
! 			// Only request the cursor blink mode if t_RC set. Not
! 			// for Gnome terminal, it can't handle t_RC, it
! 			// echoes the characters to the screen.
  			if (rbm_status.tr_progress == STATUS_GET
  				&& !is_not_xterm
  				&& *T_CRC != NUL)
***************
*** 4671,4677 ****
  			if (need_flush)
  			    out_flush();
  		    }
! 		    slen = i + 1;
  # ifdef FEAT_EVAL
  		    set_vim_var_string(VV_TERMRESPONSE, tp, slen);
  # endif
--- 4702,4708 ----
  			if (need_flush)
  			    out_flush();
  		    }
! 		    slen = csi_len;
  # ifdef FEAT_EVAL
  		    set_vim_var_string(VV_TERMRESPONSE, tp, slen);
  # endif
***************
*** 4681,4749 ****
  		    key_name[1] = (int)KE_IGNORE;
  		}
  
! 		/* Check blinking cursor from xterm:
! 		 * {lead}?12;1$y       set
! 		 * {lead}?12;2$y       not set
! 		 *
! 		 * {lead} can be <Esc>[ or CSI
! 		 */
  		else if (rbm_status.tr_progress == STATUS_SENT
! 			&& tp[(j = 1 + (tp[0] == ESC))] == '?'
! 			&& i == j + 6
! 			&& tp[j + 1] == '1'
! 			&& tp[j + 2] == '2'
! 			&& tp[j + 3] == ';'
! 			&& tp[i - 1] == '$'
! 			&& tp[i] == 'y')
  		{
! 		    initial_cursor_blink = (tp[j + 4] == '1');
  		    rbm_status.tr_progress = STATUS_GOT;
  		    LOG_TR(("Received cursor blinking mode response: %s", tp));
  		    key_name[0] = (int)KS_EXTRA;
  		    key_name[1] = (int)KE_IGNORE;
! 		    slen = i + 1;
  # ifdef FEAT_EVAL
  		    set_vim_var_string(VV_TERMBLINKRESP, tp, slen);
  # endif
  		}
  
! 		/*
! 		 * Check for a window position response from the terminal:
! 		 *       {lead}3;{x};{y}t
! 		 */
! 		else if (did_request_winpos
! 			    && ((len >= 4 && tp[0] == ESC && tp[1] == '[')
! 				|| (len >= 3 && tp[0] == CSI))
! 			    && tp[(j = 1 + (tp[0] == ESC))] == '3'
! 			    && tp[j + 1] == ';')
  		{
! 		    j += 2;
! 		    for (i = j; i < len && vim_isdigit(tp[i]); ++i)
! 			;
! 		    if (i < len && tp[i] == ';')
! 		    {
! 			winpos_x = atoi((char *)tp + j);
! 			j = i + 1;
! 			for (i = j; i < len && vim_isdigit(tp[i]); ++i)
! 			    ;
! 			if (i < len && tp[i] == 't')
! 			{
! 			    winpos_y = atoi((char *)tp + j);
! 			    /* got finished code: consume it */
! 			    key_name[0] = (int)KS_EXTRA;
! 			    key_name[1] = (int)KE_IGNORE;
! 			    slen = i + 1;
  
! 			    if (--did_request_winpos <= 0)
! 				winpos_status.tr_progress = STATUS_GOT;
! 			}
! 		    }
! 		    if (i == len)
! 		    {
! 			LOG_TR(("not enough characters for winpos"));
! 			return -1;
! 		    }
  		}
  	    }
  
  	    /* Check for fore/background color response from the terminal:
--- 4712,4762 ----
  		    key_name[1] = (int)KE_IGNORE;
  		}
  
! 		// Check blinking cursor from xterm:
! 		// {lead}?12;1$y       set
! 		// {lead}?12;2$y       not set
! 		//
! 		// {lead} can be <Esc>[ or CSI
  		else if (rbm_status.tr_progress == STATUS_SENT
! 			&& first == '?'
! 			&& ap == argp + 6
! 			&& arg[0] == 12
! 			&& ap[-1] == '$'
! 			&& trail == 'y')
  		{
! 		    initial_cursor_blink = (arg[1] == '1');
  		    rbm_status.tr_progress = STATUS_GOT;
  		    LOG_TR(("Received cursor blinking mode response: %s", tp));
  		    key_name[0] = (int)KS_EXTRA;
  		    key_name[1] = (int)KE_IGNORE;
! 		    slen = csi_len;
  # ifdef FEAT_EVAL
  		    set_vim_var_string(VV_TERMBLINKRESP, tp, slen);
  # endif
  		}
  
! 		// Check for a window position response from the terminal:
! 		//       {lead}3;{x};{y}t
! 		else if (did_request_winpos && argc == 3 && arg[0] == 3
! 							       && trail == 't')
  		{
! 		    winpos_x = arg[1];
! 		    winpos_y = arg[2];
! 		    // got finished code: consume it
! 		    key_name[0] = (int)KS_EXTRA;
! 		    key_name[1] = (int)KE_IGNORE;
! 		    slen = csi_len;
  
! 		    if (--did_request_winpos <= 0)
! 			winpos_status.tr_progress = STATUS_GOT;
  		}
+ 
+ 		// TODO: key with modifier:
+ 		//	{lead}27;{modifier};{key}~
+ 		//	{lead}{key};{modifier}u
+ 
+ 		// else: Unknown CSI sequence.  We could drop it, but then the
+ 		// user can't create a map for it.
  	    }
  
  	    /* Check for fore/background color response from the terminal:
*** ../vim-8.1.2122/src/version.c	2019-10-07 22:38:54.816029419 +0200
--- src/version.c	2019-10-08 20:15:16.726739267 +0200
***************
*** 755,756 ****
--- 755,758 ----
  {   /* Add new patch number below this line */
+ /**/
+     2123,
  /**/

-- 
An indication you must be a manager:
You believe you never have any problems in your life, just
"issues" and "improvement opportunities".

 /// Bram Moolenaar -- Bram@Moolenaar.net -- http://www.Moolenaar.net   \\\
///        sponsor Vim, vote for features -- http://www.Vim.org/sponsor/ \\\
\\\  an exciting new programming language -- http://www.Zimbu.org        ///
 \\\            help me help AIDS victims -- http://ICCF-Holland.org    ///