[PATCH 1 of 2] run-tests: implement partial skipping for *.t tests

Adrian Buehlmann adrian at cadifra.com
Thu May 31 09:06:03 CDT 2012


# HG changeset patch
# User Adrian Buehlmann <adrian at cadifra.com>
# Date 1338467696 -7200
# Node ID ca1dcebca73680d1b874e51e7515b696c095a4a1
# Parent  0e488ac1f0c4b80f731b3ba5939ee1d146699ac5
run-tests: implement partial skipping for *.t tests

*.t tests are now executed and outputs compared until a line

  $ "$TESTDIR/hghave" .. || exit 80

exits the test. So, *.t tests may now both fail and be skipped.

This change enables grouping test cases inside a testfile before/after a
hghave check, thus providing a finer granularity for skipping.

Example:

  $ python run-tests.py --local test-clone-failure2.t

  --- c:\users\adi\hgrepos\hg-main\tests\test-clone-failure2.t
  +++ c:\users\adi\hgrepos\hg-main\tests\test-clone-failure2.t.err
  @@ -7,7 +7,6 @@
     $ hg init q
     $ hg clone q
     destination directory: q
  -  What?
     abort: destination 'q' is not empty
     [255]


  ERROR: c:\users\adi\hgrepos\hg-main\tests\test-clone-failure2.t output changed
  s
  Skipped test-clone-failure2.t: missing feature: unix-style permissions
  Failed test-clone-failure2.t: output changed
  # Ran 1 tests, 1 skipped, 1 failed.

In this case, the test was run, the parts that were executed failed, and another
(not reported) part of the test was skipped. So, the test is both reported as
failed *and* skipped. The reason why parts of the test were skipped ("missing
feature: unix-style permissions") is additionally reported.

Existing *.t testfiles all have the "hghave" checks / exits at the beginning of
the *.t file, so no test steps will be executed.

diff --git a/tests/run-tests.py b/tests/run-tests.py
--- a/tests/run-tests.py
+++ b/tests/run-tests.py
@@ -656,15 +656,22 @@
         cmd = '%s "%s"' % (options.shell, name)
         vlog("# Running", cmd)
         exitcode, output = run(cmd, wd, options, replacements)
-        # do not merge output if skipped, return hghave message instead
-        # similarly, with --debug, output is None
-        if exitcode == SKIPPED_STATUS or output is None:
-            return exitcode, output
+        if output is None: # with --debug, output is None
+            return exitcode, None, None
     finally:
         os.remove(name)
 
     # Merge the script output back into a unified test
 
+    skipped = []
+    if exitcode == SKIPPED_STATUS:
+        # remove skipped message lines from the end of output
+        for l in reversed(output):
+            if salt in l:
+                break
+            skipped.insert(0, l)
+        output = output[:-len(skipped)]
+
     pos = -1
     postout = []
     ret = 0
@@ -700,10 +707,12 @@
                 postout += after.pop(pos)
             pos = int(lcmd.split()[0])
 
-    if pos in after:
+    if skipped:
+        postout.extend(t[pos + 1:])
+    elif pos in after:
         postout += after.pop(pos)
 
-    return exitcode, postout
+    return exitcode, postout, skipped
 
 wifexited = getattr(os, "WIFEXITED", lambda x: False)
 def run(cmd, wd, options, replacements):
@@ -896,12 +905,19 @@
         replacements.append((re.escape(testtmp), '$TESTTMP'))
 
     os.mkdir(testtmp)
-    ret, out = runner(testpath, testtmp, options, replacements)
+    res = runner(testpath, testtmp, options, replacements)
+    ret, out = res[:2]
+    if test.endswith(".t"):
+        skipout = res[2]
+    else:
+        skipout = out
+
     vlog("# Ret was:", ret)
 
     mark = '.'
 
     skipped = (ret == SKIPPED_STATUS)
+    fullskipped = skipped and not test.endswith(".t")
 
     # If we're not in --debug mode and reference output file exists,
     # check test output against it.
@@ -914,7 +930,7 @@
     else:
         refout = []
 
-    if (ret != 0 or out != refout) and not skipped and not options.debug:
+    if ((ret != 0 and not skipped) or out != refout) and not options.debug:
         # Save errors to a file for diagnosis
         f = open(err, "wb")
         for line in out:
@@ -927,7 +943,7 @@
             missing = ['unknown']
             failed = None
         else:
-            missing, failed = parsehghaveoutput(out)
+            missing, failed = parsehghaveoutput(skipout)
         if not missing:
             missing = ['irrelevant']
         if failed:
@@ -935,10 +951,11 @@
             skipped = False
         else:
             skip(missing[-1])
-    elif ret == 'timeout':
+
+    if ret == 'timeout':
         mark = 't'
         fail("timed out", ret)
-    elif out != refout:
+    elif out != refout and not fullskipped:
         mark = '!'
         if not options.nodiff:
             iolock.acquire()
@@ -947,15 +964,15 @@
             else:
                 showdiff(refout, out, ref, err)
             iolock.release()
-        if ret:
+        if ret and not skipped:
             fail("output changed and returned error code %d" % ret, ret)
         else:
             fail("output changed", ret)
         ret = 1
-    elif ret:
+    elif ret and not skipped:
         mark = '!'
         fail("returned error code %d" % ret, ret)
-    else:
+    elif not skipped:
         success()
 
     if not options.verbose:


More information about the Mercurial-devel mailing list