To: vim_dev@googlegroups.com Subject: Patch 8.1.0633 Fcc: outbox From: Bram Moolenaar Mime-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ------------ Patch 8.1.0633 Problem: Crash when out of memory while opening a terminal window. Solution: Handle out-of-memory more gracefully. Files: src/terminal.c, src/libvterm/src/vterm.c, src/libvterm/src/state.c, src/libvterm/src/termscreen.c *** ../vim-8.1.0632/src/terminal.c 2018-12-21 20:55:18.892739645 +0100 --- src/terminal.c 2018-12-24 21:22:28.150773840 +0100 *************** *** 3430,3435 **** --- 3430,3436 ---- { int index = 0; VTermState *state = vterm_obtain_state(vterm); + for (; index < 16; index++) { VTermColor color; *************** *** 3703,3710 **** /* * Create a new vterm and initialize it. */ ! static void create_vterm(term_T *term, int rows, int cols) { VTerm *vterm; --- 3704,3712 ---- /* * Create a new vterm and initialize it. + * Return FAIL when out of memory. */ ! static int create_vterm(term_T *term, int rows, int cols) { VTerm *vterm; *************** *** 3714,3720 **** --- 3716,3733 ---- vterm = vterm_new_with_allocator(rows, cols, &vterm_allocator, NULL); term->tl_vterm = vterm; + if (vterm == NULL) + return FAIL; + + // Allocate screen and state here, so we can bail out if that fails. + state = vterm_obtain_state(vterm); screen = vterm_obtain_screen(vterm); + if (state == NULL || screen == NULL) + { + vterm_free(vterm); + return FAIL; + } + vterm_screen_set_callbacks(screen, &screen_callbacks, term); /* TODO: depends on 'encoding'. */ vterm_set_utf8(vterm, 1); *************** *** 3722,3728 **** init_default_colors(term); vterm_state_set_default_colors( ! vterm_obtain_state(vterm), &term->tl_default_color.fg, &term->tl_default_color.bg); --- 3735,3741 ---- init_default_colors(term); vterm_state_set_default_colors( ! state, &term->tl_default_color.fg, &term->tl_default_color.bg); *************** *** 3746,3754 **** #else value.boolean = 0; #endif - state = vterm_obtain_state(vterm); vterm_state_set_termprop(state, VTERM_PROP_CURSORBLINK, &value); vterm_state_set_unrecognised_fallbacks(state, &parser_fallbacks, term); } /* --- 3759,3768 ---- #else value.boolean = 0; #endif vterm_state_set_termprop(state, VTERM_PROP_CURSORBLINK, &value); vterm_state_set_unrecognised_fallbacks(state, &parser_fallbacks, term); + + return OK; } /* *************** *** 5629,5635 **** vim_free(cwd_wchar); vim_free(env_wchar); ! create_vterm(term, term->tl_rows, term->tl_cols); #if defined(FEAT_GUI) || defined(FEAT_TERMGUICOLORS) if (opt->jo_set2 & JO2_ANSI_COLORS) --- 5643,5650 ---- vim_free(cwd_wchar); vim_free(env_wchar); ! if (create_vterm(term, term->tl_rows, term->tl_cols) == FAIL) ! goto failed; #if defined(FEAT_GUI) || defined(FEAT_TERMGUICOLORS) if (opt->jo_set2 & JO2_ANSI_COLORS) *************** *** 5710,5716 **** char in_name[80], out_name[80]; channel_T *channel = NULL; ! create_vterm(term, term->tl_rows, term->tl_cols); vim_snprintf(in_name, sizeof(in_name), "\\\\.\\pipe\\vim-%d-in-%d", GetCurrentProcessId(), --- 5725,5732 ---- char in_name[80], out_name[80]; channel_T *channel = NULL; ! if (create_vterm(term, term->tl_rows, term->tl_cols) == FAIL) ! return FAIL; vim_snprintf(in_name, sizeof(in_name), "\\\\.\\pipe\\vim-%d-in-%d", GetCurrentProcessId(), *************** *** 5822,5828 **** jobopt_T *opt, jobopt_T *orig_opt UNUSED) { ! create_vterm(term, term->tl_rows, term->tl_cols); #if defined(FEAT_GUI) || defined(FEAT_TERMGUICOLORS) if (opt->jo_set2 & JO2_ANSI_COLORS) --- 5838,5845 ---- jobopt_T *opt, jobopt_T *orig_opt UNUSED) { ! if (create_vterm(term, term->tl_rows, term->tl_cols) == FAIL) ! return FAIL; #if defined(FEAT_GUI) || defined(FEAT_TERMGUICOLORS) if (opt->jo_set2 & JO2_ANSI_COLORS) *************** *** 5844,5850 **** static int create_pty_only(term_T *term, jobopt_T *opt) { ! create_vterm(term, term->tl_rows, term->tl_cols); term->tl_job = job_alloc(); if (term->tl_job == NULL) --- 5861,5868 ---- static int create_pty_only(term_T *term, jobopt_T *opt) { ! if (create_vterm(term, term->tl_rows, term->tl_cols) == FAIL) ! return FAIL; term->tl_job = job_alloc(); if (term->tl_job == NULL) *** ../vim-8.1.0632/src/libvterm/src/vterm.c 2018-12-15 14:49:30.800096933 +0100 --- src/libvterm/src/vterm.c 2018-12-24 21:26:14.100725256 +0100 *************** *** 1,5 **** --- 1,6 ---- #define DEFINE_INLINES + /* vim: set sw=2 : */ #include "vterm_internal.h" #include *************** *** 41,46 **** --- 42,49 ---- /* Need to bootstrap using the allocator function directly */ VTerm *vt = (*funcs->malloc)(sizeof(VTerm), allocdata); + if (vt == NULL) + return NULL; vt->allocator = funcs; vt->allocdata = allocdata; *************** *** 55,64 **** --- 58,78 ---- vt->parser.strbuffer_len = 500; /* should be able to hold an OSC string */ vt->parser.strbuffer_cur = 0; vt->parser.strbuffer = vterm_allocator_malloc(vt, vt->parser.strbuffer_len); + if (vt->parser.strbuffer == NULL) + { + vterm_allocator_free(vt, vt); + return NULL; + } vt->outbuffer_len = 200; vt->outbuffer_cur = 0; vt->outbuffer = vterm_allocator_malloc(vt, vt->outbuffer_len); + if (vt->outbuffer == NULL) + { + vterm_allocator_free(vt, vt->parser.strbuffer); + vterm_allocator_free(vt, vt); + return NULL; + } return vt; } *************** *** 82,90 **** return (*vt->allocator->malloc)(size, vt->allocdata); } INTERNAL void vterm_allocator_free(VTerm *vt, void *ptr) { ! (*vt->allocator->free)(ptr, vt->allocdata); } void vterm_get_size(const VTerm *vt, int *rowsp, int *colsp) --- 96,108 ---- return (*vt->allocator->malloc)(size, vt->allocdata); } + /* + * Free "ptr" unless it is NULL. + */ INTERNAL void vterm_allocator_free(VTerm *vt, void *ptr) { ! if (ptr) ! (*vt->allocator->free)(ptr, vt->allocdata); } void vterm_get_size(const VTerm *vt, int *rowsp, int *colsp) *** ../vim-8.1.0632/src/libvterm/src/state.c 2018-07-09 20:39:12.824845063 +0200 --- src/libvterm/src/state.c 2018-12-24 20:45:42.697972984 +0100 *************** *** 53,58 **** --- 53,60 ---- { VTermState *state = vterm_allocator_malloc(vt, sizeof(VTermState)); + if (state == NULL) + return NULL; state->vt = vt; state->rows = vt->rows; *************** *** 1693,1698 **** --- 1695,1704 ---- on_resize /* resize */ }; + /* + * Return the existing state or create a new one. + * Returns NULL when out of memory. + */ VTermState *vterm_obtain_state(VTerm *vt) { VTermState *state; *************** *** 1700,1705 **** --- 1706,1713 ---- return vt->state; state = vterm_state_new(vt); + if (state == NULL) + return NULL; vt->state = state; state->combine_chars_size = 16; *** ../vim-8.1.0632/src/libvterm/src/termscreen.c 2018-09-13 17:23:05.169150892 +0200 --- src/libvterm/src/termscreen.c 2018-12-24 21:29:52.730781805 +0100 *************** *** 1,5 **** --- 1,6 ---- #include "vterm_internal.h" + /* vim: set sw=2 : */ #include #include *************** *** 95,102 **** } } ! if(buffer) ! vterm_allocator_free(screen->vt, buffer); return new_buffer; } --- 96,102 ---- } } ! vterm_allocator_free(screen->vt, buffer); return new_buffer; } *************** *** 518,525 **** screen->rows = new_rows; screen->cols = new_cols; ! if(screen->sb_buffer) ! vterm_allocator_free(screen->vt, screen->sb_buffer); screen->sb_buffer = vterm_allocator_malloc(screen->vt, sizeof(VTermScreenCell) * new_cols); --- 518,524 ---- screen->rows = new_rows; screen->cols = new_cols; ! vterm_allocator_free(screen->vt, screen->sb_buffer); screen->sb_buffer = vterm_allocator_malloc(screen->vt, sizeof(VTermScreenCell) * new_cols); *************** *** 619,634 **** &setlineinfo /* setlineinfo */ }; static VTermScreen *screen_new(VTerm *vt) { VTermState *state = vterm_obtain_state(vt); VTermScreen *screen; int rows, cols; ! if(!state) return NULL; - screen = vterm_allocator_malloc(vt, sizeof(VTermScreen)); vterm_get_size(vt, &rows, &cols); --- 618,638 ---- &setlineinfo /* setlineinfo */ }; + /* + * Allocate a new screen and return it. + * Return NULL when out of memory. + */ static VTermScreen *screen_new(VTerm *vt) { VTermState *state = vterm_obtain_state(vt); VTermScreen *screen; int rows, cols; ! if (state == NULL) return NULL; screen = vterm_allocator_malloc(vt, sizeof(VTermScreen)); + if (screen == NULL) + return NULL; vterm_get_size(vt, &rows, &cols); *************** *** 646,655 **** screen->cbdata = NULL; screen->buffers[0] = realloc_buffer(screen, NULL, rows, cols); - screen->buffer = screen->buffers[0]; - screen->sb_buffer = vterm_allocator_malloc(screen->vt, sizeof(VTermScreenCell) * cols); vterm_state_set_callbacks(screen->state, &state_cbs, screen); --- 650,662 ---- screen->cbdata = NULL; screen->buffers[0] = realloc_buffer(screen, NULL, rows, cols); screen->buffer = screen->buffers[0]; screen->sb_buffer = vterm_allocator_malloc(screen->vt, sizeof(VTermScreenCell) * cols); + if (screen->buffer == NULL || screen->sb_buffer == NULL) + { + vterm_screen_free(screen); + return NULL; + } vterm_state_set_callbacks(screen->state, &state_cbs, screen); *************** *** 659,669 **** INTERNAL void vterm_screen_free(VTermScreen *screen) { vterm_allocator_free(screen->vt, screen->buffers[0]); ! if(screen->buffers[1]) ! vterm_allocator_free(screen->vt, screen->buffers[1]); ! vterm_allocator_free(screen->vt, screen->sb_buffer); - vterm_allocator_free(screen->vt, screen); } --- 666,673 ---- INTERNAL void vterm_screen_free(VTermScreen *screen) { vterm_allocator_free(screen->vt, screen->buffers[0]); ! vterm_allocator_free(screen->vt, screen->buffers[1]); vterm_allocator_free(screen->vt, screen->sb_buffer); vterm_allocator_free(screen->vt, screen); } *** ../vim-8.1.0632/src/version.c 2018-12-24 20:23:39.440716979 +0100 --- src/version.c 2018-12-24 21:34:22.808414341 +0100 *************** *** 801,802 **** --- 801,804 ---- { /* Add new patch number below this line */ + /**/ + 633, /**/ -- "Oh, no! NOT the Spanish Inquisition!" "NOBODY expects the Spanish Inquisition!!!" -- Monty Python sketch -- "Oh, no! NOT another option!" "EVERYBODY expects another option!!!" -- Discussion in vim-dev mailing list -- /// Bram Moolenaar -- Bram@Moolenaar.net -- http://www.Moolenaar.net \\\ /// sponsor Vim, vote for features -- http://www.Vim.org/sponsor/ \\\ \\\ an exciting new programming language -- http://www.Zimbu.org /// \\\ help me help AIDS victims -- http://ICCF-Holland.org ///