[PATCH 1 of 2] run-tests: implement partial skipping for *.t tests
Adrian Buehlmann
adrian at cadifra.com
Thu May 31 09:06:03 CDT 2012
# HG changeset patch
# User Adrian Buehlmann <adrian at cadifra.com>
# Date 1338467696 -7200
# Node ID ca1dcebca73680d1b874e51e7515b696c095a4a1
# Parent 0e488ac1f0c4b80f731b3ba5939ee1d146699ac5
run-tests: implement partial skipping for *.t tests
*.t tests are now executed and outputs compared until a line
$ "$TESTDIR/hghave" .. || exit 80
exits the test. So, *.t tests may now both fail and be skipped.
This change enables grouping test cases inside a testfile before/after a
hghave check, thus providing a finer granularity for skipping.
Example:
$ python run-tests.py --local test-clone-failure2.t
--- c:\users\adi\hgrepos\hg-main\tests\test-clone-failure2.t
+++ c:\users\adi\hgrepos\hg-main\tests\test-clone-failure2.t.err
@@ -7,7 +7,6 @@
$ hg init q
$ hg clone q
destination directory: q
- What?
abort: destination 'q' is not empty
[255]
ERROR: c:\users\adi\hgrepos\hg-main\tests\test-clone-failure2.t output changed
s
Skipped test-clone-failure2.t: missing feature: unix-style permissions
Failed test-clone-failure2.t: output changed
# Ran 1 tests, 1 skipped, 1 failed.
In this case, the test was run, the parts that were executed failed, and another
(not reported) part of the test was skipped. So, the test is both reported as
failed *and* skipped. The reason why parts of the test were skipped ("missing
feature: unix-style permissions") is additionally reported.
Existing *.t testfiles all have the "hghave" checks / exits at the beginning of
the *.t file, so no test steps will be executed.
diff --git a/tests/run-tests.py b/tests/run-tests.py
--- a/tests/run-tests.py
+++ b/tests/run-tests.py
@@ -656,15 +656,22 @@
cmd = '%s "%s"' % (options.shell, name)
vlog("# Running", cmd)
exitcode, output = run(cmd, wd, options, replacements)
- # do not merge output if skipped, return hghave message instead
- # similarly, with --debug, output is None
- if exitcode == SKIPPED_STATUS or output is None:
- return exitcode, output
+ if output is None: # with --debug, output is None
+ return exitcode, None, None
finally:
os.remove(name)
# Merge the script output back into a unified test
+ skipped = []
+ if exitcode == SKIPPED_STATUS:
+ # remove skipped message lines from the end of output
+ for l in reversed(output):
+ if salt in l:
+ break
+ skipped.insert(0, l)
+ output = output[:-len(skipped)]
+
pos = -1
postout = []
ret = 0
@@ -700,10 +707,12 @@
postout += after.pop(pos)
pos = int(lcmd.split()[0])
- if pos in after:
+ if skipped:
+ postout.extend(t[pos + 1:])
+ elif pos in after:
postout += after.pop(pos)
- return exitcode, postout
+ return exitcode, postout, skipped
wifexited = getattr(os, "WIFEXITED", lambda x: False)
def run(cmd, wd, options, replacements):
@@ -896,12 +905,19 @@
replacements.append((re.escape(testtmp), '$TESTTMP'))
os.mkdir(testtmp)
- ret, out = runner(testpath, testtmp, options, replacements)
+ res = runner(testpath, testtmp, options, replacements)
+ ret, out = res[:2]
+ if test.endswith(".t"):
+ skipout = res[2]
+ else:
+ skipout = out
+
vlog("# Ret was:", ret)
mark = '.'
skipped = (ret == SKIPPED_STATUS)
+ fullskipped = skipped and not test.endswith(".t")
# If we're not in --debug mode and reference output file exists,
# check test output against it.
@@ -914,7 +930,7 @@
else:
refout = []
- if (ret != 0 or out != refout) and not skipped and not options.debug:
+ if ((ret != 0 and not skipped) or out != refout) and not options.debug:
# Save errors to a file for diagnosis
f = open(err, "wb")
for line in out:
@@ -927,7 +943,7 @@
missing = ['unknown']
failed = None
else:
- missing, failed = parsehghaveoutput(out)
+ missing, failed = parsehghaveoutput(skipout)
if not missing:
missing = ['irrelevant']
if failed:
@@ -935,10 +951,11 @@
skipped = False
else:
skip(missing[-1])
- elif ret == 'timeout':
+
+ if ret == 'timeout':
mark = 't'
fail("timed out", ret)
- elif out != refout:
+ elif out != refout and not fullskipped:
mark = '!'
if not options.nodiff:
iolock.acquire()
@@ -947,15 +964,15 @@
else:
showdiff(refout, out, ref, err)
iolock.release()
- if ret:
+ if ret and not skipped:
fail("output changed and returned error code %d" % ret, ret)
else:
fail("output changed", ret)
ret = 1
- elif ret:
+ elif ret and not skipped:
mark = '!'
fail("returned error code %d" % ret, ret)
- else:
+ elif not skipped:
success()
if not options.verbose:
More information about the Mercurial-devel
mailing list