To: vim_dev@googlegroups.com Subject: Patch 8.1.1476 Fcc: outbox From: Bram Moolenaar Mime-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ------------ Patch 8.1.1476 Problem: No statistics displayed after running tests. Solution: Summarize the test results. (Christian Brabandt, closes #4391) Also make it possible to report a skipped file. Files: src/Makefile, src/testdir/Makefile, src/testdir/summarize.vim, src/testdir/runtest.vim, src/testdir/test_arabic.vim, src/testdir/test_autochdir.vim, src/testdir/test_balloon.vim *** ../vim-8.1.1475/src/Makefile 2019-05-31 20:02:47.567231180 +0200 --- src/Makefile 2019-06-06 13:12:56.701571974 +0200 *************** *** 2117,2123 **** # TESTING # # Execute the test scripts and the unittests. ! test check: scripttests unittests test_libvterm # Execute the test scripts. Run these after compiling Vim, before installing. # This doesn't depend on $(VIMTARGET), because that won't work when configure --- 2118,2125 ---- # TESTING # # Execute the test scripts and the unittests. ! # Do the scripttests first, so that the summary shows last. ! test check: unittests test_libvterm scripttests # Execute the test scripts. Run these after compiling Vim, before installing. # This doesn't depend on $(VIMTARGET), because that won't work when configure *** ../vim-8.1.1475/src/testdir/Makefile 2019-02-21 21:50:42.603751312 +0100 --- src/testdir/Makefile 2019-06-06 12:56:44.810633118 +0200 *************** *** 12,18 **** # Comment out this line to see the verbose output of tests. # # Catches SwapExists to avoid hanging at the ATTENTION prompt. ! REDIR_TEST_TO_NULL = --cmd 'au SwapExists * let v:swapchoice = "e"' > /dev/null # Uncomment this line to use valgrind for memory leaks and extra warnings. # The output goes into a file "valgrind.testN" --- 12,18 ---- # Comment out this line to see the verbose output of tests. # # Catches SwapExists to avoid hanging at the ATTENTION prompt. ! # REDIR_TEST_TO_NULL = --cmd 'au SwapExists * let v:swapchoice = "e"' > /dev/null # Uncomment this line to use valgrind for memory leaks and extra warnings. # The output goes into a file "valgrind.testN" *************** *** 49,58 **** benchmark: $(SCRIPTS_BENCH) report: @echo @echo 'Test results:' @/bin/sh -c "if test -f test.log; \ ! then cat test.log; echo TEST FAILURE; exit 1; \ else echo ALL DONE; \ fi" --- 49,60 ---- benchmark: $(SCRIPTS_BENCH) report: + $(RUN_VIMTEST) $(NO_INITS) -S summarize.vim messages $(REDIR_TEST_TO_NULL) @echo @echo 'Test results:' + @cat test_result.log @/bin/sh -c "if test -f test.log; \ ! then echo TEST FAILURE; exit 1; \ else echo ALL DONE; \ fi" *************** *** 77,83 **** RUN_VIM = VIMRUNTIME=$(SCRIPTSOURCE); export VIMRUNTIME; $(VALGRIND) $(VIMPROG) -f $(GUI_FLAG) -u unix.vim $(NO_INITS) -s dotest.in clean: ! -rm -rf *.out *.failed *.res *.rej *.orig opt_test.vim test.log messages $(RM_ON_RUN) $(RM_ON_START) valgrind.* test1.out: test1.in -rm -rf $*.failed $(RM_ON_RUN) $(RM_ON_START) wrongtermsize --- 79,88 ---- RUN_VIM = VIMRUNTIME=$(SCRIPTSOURCE); export VIMRUNTIME; $(VALGRIND) $(VIMPROG) -f $(GUI_FLAG) -u unix.vim $(NO_INITS) -s dotest.in clean: ! -rm -rf *.out *.failed *.res *.rej *.orig ! -rm opt_test.vim test.log test_result.log messages ! -rm $(RM_ON_RUN) $(RM_ON_START) ! -rm valgrind.* test1.out: test1.in -rm -rf $*.failed $(RM_ON_RUN) $(RM_ON_START) wrongtermsize *** ../vim-8.1.1475/src/testdir/summarize.vim 2019-06-06 13:36:51.512235762 +0200 --- src/testdir/summarize.vim 2019-06-06 13:10:12.018902480 +0200 *************** *** 0 **** --- 1,60 ---- + if 1 + " This is executed with the eval feature + set nocp + func Count(match, type) + if a:type ==# 'executed' + let g:executed += (a:match+0) + elseif a:type ==# 'failed' + let g:failed = a:match+0 + elseif a:type ==# 'skipped' + let g:skipped += 1 + call extend(g:skipped_output, ["\t".a:match]) + endif + endfunc + + let g:executed = 0 + let g:skipped = 0 + let g:failed = 0 + let g:skipped_output = [] + let g:failed_output = [] + let output = [""] + + try + " This uses the :s command to just fetch and process the output of the + " tests, it doesn't acutally replay anything + %s/^Executed\s\+\zs\d\+\ze\s\+tests/\=Count(submatch(0),'executed')/egn + %s/^SKIPPED \zs.*/\=Count(submatch(0), 'skipped')/egn + %s/^\(\d\+\)\s\+FAILED:/\=Count(submatch(1), 'failed')/egn + + call extend(output, ["Skipped:"]) + call extend(output, skipped_output) + + call extend(output, [ + \ "", + \ "-------------------------------", + \ printf("Executed: %5d Tests", g:executed), + \ printf(" Skipped: %5d Tests", g:skipped), + \ printf(" %s: %5d Tests", g:failed == 0 ? 'Failed' : 'FAILED', g:failed), + \ "", + \ ]) + if filereadable('test.log') + " outputs and indents the failed test result + call extend(output, ["", "Failures: "]) + let failed_output = filter(readfile('test.log'), { v,k -> !empty(k)}) + call extend(output, map(failed_output, { v,k -> "\t".k})) + " Add a final newline + call extend(output, [""]) + endif + + catch " Catch-all + finally + call writefile(output, 'test_result.log') " overwrites an existing file + q! + endtry + endif + + " This is executed without the eval feature + %d + r test.log + w test_result.log + q! *** ../vim-8.1.1475/src/testdir/runtest.vim 2019-02-22 14:38:46.447452832 +0100 --- src/testdir/runtest.vim 2019-06-06 13:17:50.755349142 +0200 *************** *** 268,273 **** --- 268,276 ---- else try source % + catch /^\cskipped/ + call add(s:messages, ' Skipped') + call add(s:skipped, 'SKIPPED ' . expand('%') . ': ' . substitute(v:exception, '^\S*\s\+', '', '')) catch let s:fail += 1 call add(s:errors, 'Caught exception: ' . v:exception . ' @ ' . v:throwpoint) *** ../vim-8.1.1475/src/testdir/test_arabic.vim 2019-03-23 12:59:36.262661906 +0100 --- src/testdir/test_arabic.vim 2019-06-06 13:24:06.741229849 +0200 *************** *** 3,9 **** " functional tests that check the shaping works with real text. if !has('arabic') ! finish endif source view_util.vim --- 3,9 ---- " functional tests that check the shaping works with real text. if !has('arabic') ! throw 'Skipped: arabic feature missing' endif source view_util.vim *** ../vim-8.1.1475/src/testdir/test_autochdir.vim 2018-12-16 15:37:58.866807609 +0100 --- src/testdir/test_autochdir.vim 2019-06-06 13:25:39.432658711 +0200 *************** *** 1,7 **** " Test 'autochdir' behavior if !exists("+autochdir") ! finish endif func Test_set_filename() --- 1,7 ---- " Test 'autochdir' behavior if !exists("+autochdir") ! throw 'Skipped: autochdir feature missing' endif func Test_set_filename() *** ../vim-8.1.1475/src/testdir/test_balloon.vim 2019-05-20 22:12:30.720442793 +0200 --- src/testdir/test_balloon.vim 2019-06-06 13:35:26.168814971 +0200 *************** *** 1,11 **** " Tests for 'balloonevalterm'. ! " Tests that only work in the terminal. ! if has('balloon_eval_term') && !has('gui_running') source screendump.vim if !CanRunVimInTerminal() ! finish endif let s:common_script =<< [CODE] --- 1,15 ---- " Tests for 'balloonevalterm'. ! if !has('balloon_eval_term') ! throw 'Skipped: balloon_eval_term feature missing' ! endif ! ! " A few tests only work in the terminal. ! if !has('gui_running') source screendump.vim if !CanRunVimInTerminal() ! throw 'Skipped: cannot run Vim in a terminal window' endif let s:common_script =<< [CODE] *** ../vim-8.1.1475/src/version.c 2019-06-06 12:49:25.597017931 +0200 --- src/version.c 2019-06-06 12:53:55.112790029 +0200 *************** *** 769,770 **** --- 769,772 ---- { /* Add new patch number below this line */ + /**/ + 1476, /**/ -- # echo reboot >universe # chmod +x universe # ./universe /// Bram Moolenaar -- Bram@Moolenaar.net -- http://www.Moolenaar.net \\\ /// sponsor Vim, vote for features -- http://www.Vim.org/sponsor/ \\\ \\\ an exciting new programming language -- http://www.Zimbu.org /// \\\ help me help AIDS victims -- http://ICCF-Holland.org ///