To: vim_dev@googlegroups.com Subject: Patch 8.1.0098 Fcc: outbox From: Bram Moolenaar Mime-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ------------ Patch 8.1.0098 Problem: Segfault when pattern with \z() is very slow. Solution: Check for NULL regprog. Add "nfa_fail" to test_override() to be able to test this. Fix that 'searchhl' resets called_emsg. Files: src/syntax.c, runtime/doc/eval.txt, src/evalfunc.c, src/vim.h, src/testdir/test_syntax.vim, src/globals.h, src/screen.c, src/regexp.c, src/regexp_nfa.c *** ../vim-8.1.0097/src/syntax.c 2018-05-20 13:35:40.193163458 +0200 --- src/syntax.c 2018-06-23 13:47:27.294238648 +0200 *************** *** 3327,3332 **** --- 3327,3338 ---- profile_start(&pt); #endif + if (rmp->regprog == NULL) + // This can happen if a previous call to vim_regexec_multi() tried to + // use the NFA engine, which resulted in NFA_TOO_EXPENSIVE, and + // compiling the pattern with the other engine fails. + return FALSE; + rmp->rmm_maxcol = syn_buf->b_p_smc; r = vim_regexec_multi(rmp, syn_win, syn_buf, lnum, col, #ifdef FEAT_RELTIME *** ../vim-8.1.0097/runtime/doc/eval.txt 2018-06-20 22:37:52.658911284 +0200 --- runtime/doc/eval.txt 2018-06-23 13:27:43.863676347 +0200 *************** *** 8694,8699 **** --- 8694,8701 ---- redraw disable the redrawing() function char_avail disable the char_avail() function starting reset the "starting" variable, see below + nfa_fail makes the NFA regexp engine fail to force a + fallback to the old engine ALL clear all overrides ({val} is not used) "starting" is to be used when a test should behave like *** ../vim-8.1.0097/src/evalfunc.c 2018-06-20 22:37:52.654911306 +0200 --- src/evalfunc.c 2018-06-23 13:29:32.603042364 +0200 *************** *** 13090,13099 **** --- 13090,13102 ---- save_starting = -1; } } + else if (STRCMP(name, (char_u *)"nfa_fail") == 0) + nfa_fail_for_testing = val; else if (STRCMP(name, (char_u *)"ALL") == 0) { disable_char_avail_for_testing = FALSE; disable_redraw_for_testing = FALSE; + nfa_fail_for_testing = FALSE; if (save_starting >= 0) { starting = save_starting; *** ../vim-8.1.0097/src/vim.h 2018-04-30 15:12:48.000000000 +0200 --- src/vim.h 2018-06-23 14:06:57.244449803 +0200 *************** *** 1013,1018 **** --- 1013,1019 ---- /* values for reg_do_extmatch */ # define REX_SET 1 /* to allow \z\(...\), */ # define REX_USE 2 /* to allow \z\1 et al. */ + # define REX_ALL (REX_SET | REX_USE) #endif /* Return values for fullpathcmp() */ *** ../vim-8.1.0097/src/testdir/test_syntax.vim 2018-02-24 19:33:19.000000000 +0100 --- src/testdir/test_syntax.vim 2018-06-23 14:11:40.474945470 +0200 *************** *** 562,564 **** --- 562,576 ---- let $COLORFGBG = '' call delete('Xtest.c') endfun + + " Using \z() in a region with NFA failing should not crash. + func Test_syn_wrong_z_one() + new + call setline(1, ['just some text', 'with foo and bar to match with']) + syn region FooBar start="foo\z(.*\)bar" end="\z1" + call test_override("nfa_fail", 1) + redraw! + redraw! + call test_override("ALL", 0) + bwipe! + endfunc *** ../vim-8.1.0097/src/globals.h 2018-06-19 17:49:20.296015375 +0200 --- src/globals.h 2018-06-23 13:30:00.290880945 +0200 *************** *** 1634,1639 **** --- 1634,1640 ---- /* flags set by test_override() */ EXTERN int disable_char_avail_for_testing INIT(= 0); EXTERN int disable_redraw_for_testing INIT(= 0); + EXTERN int nfa_fail_for_testing INIT(= 0); EXTERN int in_free_unref_items INIT(= FALSE); #endif *** ../vim-8.1.0097/src/screen.c 2018-06-17 16:23:29.341140642 +0200 --- src/screen.c 2018-06-23 14:02:46.329754550 +0200 *************** *** 7868,7873 **** --- 7868,7874 ---- linenr_T l; colnr_T matchcol; long nmatched; + int save_called_emsg = called_emsg; if (shl->lnum != 0) { *************** *** 7986,7991 **** --- 7987,7995 ---- break; /* useful match found */ } } + + // Restore called_emsg for assert_fails(). + called_emsg = save_called_emsg; } /* *** ../vim-8.1.0097/src/regexp.c 2018-02-13 16:28:11.000000000 +0100 --- src/regexp.c 2018-06-23 14:09:21.727692568 +0200 *************** *** 367,373 **** static char_u e_unmatchedpar[] = N_("E55: Unmatched %s)"); #ifdef FEAT_SYN_HL static char_u e_z_not_allowed[] = N_("E66: \\z( not allowed here"); ! static char_u e_z1_not_allowed[] = N_("E67: \\z1 et al. not allowed here"); #endif static char_u e_missing_sb[] = N_("E69: Missing ] after %s%%["); static char_u e_empty_sb[] = N_("E70: Empty %s%%[]"); --- 367,373 ---- static char_u e_unmatchedpar[] = N_("E55: Unmatched %s)"); #ifdef FEAT_SYN_HL static char_u e_z_not_allowed[] = N_("E66: \\z( not allowed here"); ! static char_u e_z1_not_allowed[] = N_("E67: \\z1 - \\z9 not allowed here"); #endif static char_u e_missing_sb[] = N_("E69: Missing ] after %s%%["); static char_u e_empty_sb[] = N_("E70: Empty %s%%[]"); *************** *** 2139,2145 **** switch (c) { #ifdef FEAT_SYN_HL ! case '(': if (reg_do_extmatch != REX_SET) EMSG_RET_NULL(_(e_z_not_allowed)); if (one_exactly) EMSG_ONE_RET_NULL; --- 2139,2145 ---- switch (c) { #ifdef FEAT_SYN_HL ! case '(': if ((reg_do_extmatch & REX_SET) == 0) EMSG_RET_NULL(_(e_z_not_allowed)); if (one_exactly) EMSG_ONE_RET_NULL; *************** *** 2158,2164 **** case '6': case '7': case '8': ! case '9': if (reg_do_extmatch != REX_USE) EMSG_RET_NULL(_(e_z1_not_allowed)); ret = regnode(ZREF + c - '0'); re_has_z = REX_USE; --- 2158,2164 ---- case '6': case '7': case '8': ! case '9': if ((reg_do_extmatch & REX_USE) == 0) EMSG_RET_NULL(_(e_z1_not_allowed)); ret = regnode(ZREF + c - '0'); re_has_z = REX_USE; *************** *** 8332,8339 **** /* * Match a regexp against multiple lines. ! * "rmp->regprog" is a compiled regexp as returned by vim_regcomp(). ! * Note: "rmp->regprog" may be freed and changed. * Uses curbuf for line count and 'iskeyword'. * * Return zero if there is no match. Return number of lines contained in the --- 8332,8339 ---- /* * Match a regexp against multiple lines. ! * "rmp->regprog" must be a compiled regexp as returned by vim_regcomp(). ! * Note: "rmp->regprog" may be freed and changed, even set to NULL. * Uses curbuf for line count and 'iskeyword'. * * Return zero if there is no match. Return number of lines contained in the *************** *** 8376,8382 **** --- 8376,8387 ---- #ifdef FEAT_EVAL report_re_switch(pat); #endif + // checking for \z misuse was already done when compiling for NFA, + // allow all here + reg_do_extmatch = REX_ALL; rmp->regprog = vim_regcomp(pat, re_flags); + reg_do_extmatch = 0; + if (rmp->regprog != NULL) result = rmp->regprog->engine->regexec_multi( rmp, win, buf, lnum, col, tm, timed_out); *** ../vim-8.1.0097/src/regexp_nfa.c 2018-06-22 21:42:26.743455036 +0200 --- src/regexp_nfa.c 2018-06-23 14:08:50.071858765 +0200 *************** *** 1482,1488 **** case '8': case '9': /* \z1...\z9 */ ! if (reg_do_extmatch != REX_USE) EMSG_RET_FAIL(_(e_z1_not_allowed)); EMIT(NFA_ZREF1 + (no_Magic(c) - '1')); /* No need to set nfa_has_backref, the sub-matches don't --- 1482,1488 ---- case '8': case '9': /* \z1...\z9 */ ! if ((reg_do_extmatch & REX_USE) == 0) EMSG_RET_FAIL(_(e_z1_not_allowed)); EMIT(NFA_ZREF1 + (no_Magic(c) - '1')); /* No need to set nfa_has_backref, the sub-matches don't *************** *** 1491,1497 **** break; case '(': /* \z( */ ! if (reg_do_extmatch != REX_SET) EMSG_RET_FAIL(_(e_z_not_allowed)); if (nfa_reg(REG_ZPAREN) == FAIL) return FAIL; /* cascaded error */ --- 1491,1497 ---- break; case '(': /* \z( */ ! if ((reg_do_extmatch & REX_SET) == 0) EMSG_RET_FAIL(_(e_z_not_allowed)); if (nfa_reg(REG_ZPAREN) == FAIL) return FAIL; /* cascaded error */ *************** *** 5692,5698 **** nextlist->n = 0; /* clear nextlist */ nextlist->has_pim = FALSE; ++nfa_listid; ! if (prog->re_engine == AUTOMATIC_ENGINE && nfa_listid >= NFA_MAX_STATES) { /* too many states, retry with old engine */ nfa_match = NFA_TOO_EXPENSIVE; --- 5692,5699 ---- nextlist->n = 0; /* clear nextlist */ nextlist->has_pim = FALSE; ++nfa_listid; ! if (prog->re_engine == AUTOMATIC_ENGINE ! && (nfa_listid >= NFA_MAX_STATES || nfa_fail_for_testing)) { /* too many states, retry with old engine */ nfa_match = NFA_TOO_EXPENSIVE; *** ../vim-8.1.0097/src/version.c 2018-06-22 21:42:26.747455046 +0200 --- src/version.c 2018-06-23 13:28:43.939326081 +0200 *************** *** 763,764 **** --- 763,766 ---- { /* Add new patch number below this line */ + /**/ + 98, /**/ -- I AM THANKFUL... ...for all the complaining I hear about the government because it means we have freedom of speech. /// Bram Moolenaar -- Bram@Moolenaar.net -- http://www.Moolenaar.net \\\ /// sponsor Vim, vote for features -- http://www.Vim.org/sponsor/ \\\ \\\ an exciting new programming language -- http://www.Zimbu.org /// \\\ help me help AIDS victims -- http://ICCF-Holland.org ///