[PATCH 2 of 3] run-tests: support multiple cases in .t test
Gregory Szorc
gregory.szorc at gmail.com
Thu May 4 03:07:15 EDT 2017
On Tue, May 2, 2017 at 7:01 PM, Jun Wu <quark at fb.com> wrote:
> # HG changeset patch
> # User Jun Wu <quark at fb.com>
> # Date 1493488388 25200
> # Sat Apr 29 10:53:08 2017 -0700
> # Node ID 6663e696ae0c2a1b8f6fd735dfcf9b99fb4dac73
> # Parent f32a5c7a590fd5d8a9d6c8195b6171331ec9f3a8
> # Available At https://bitbucket.org/quark-zju/hg-draft
> # hg pull https://bitbucket.org/quark-zju/hg-draft -r
> 6663e696ae0c
> run-tests: support multiple cases in .t test
>
> Sometimes we want to run similar tests with slightly different
> configurations. Previously we duplicate the test files. This patch
> introduces special "#case" syntax that allows a single .t file to contain
> multiple test cases.
>
> For example, if a test should behave the same with or without an
> experimental flag, we can add the following to the .t header:
>
> #case default
> ....
> #case experimental-a
> $ cat >> $HGRCPATH << EOF
> > [experimental]
> > feature=a
> > EOF
> #endcase
>
> The "experimental-a" block won't be executed when running the "default"
> test
> case.
>
> In the test body, if there are case-dependent outputs, we can also gate
> them
> using the same syntax:
>
> #case default
> $ hg strip X
> ... save backup bundle ...
> #case experimental-a
> $ hg strip X
> ... pruned ...
> #endcase
>
> The implementation allows "#case" to specify multiple values, to make it
> easier to test combinations affected by multiple factors. For example:
>
> #case A B
> $ flag="$flag --config experimental.factor-a=1"
> #case A C
> $ flag="$flag --config experimental.factor-b=1"
> #case D
> #endcase
>
> A later patch will make use of this feature to simplify test-hardlink*.t.
>
I approve of this feature! This should make it vastly easier to test
configuration variations (especially wire protocol interop).
I performed a half review and don't see anything obviously bad with this
patch: it seems reasonable to me. But I don't have run-tests.py paged in,
so I may have missed something.
>
> diff --git a/tests/run-tests.py b/tests/run-tests.py
> --- a/tests/run-tests.py
> +++ b/tests/run-tests.py
> @@ -592,4 +592,5 @@ class Test(unittest.TestCase):
> self.name = _strpath(self.bname)
> self._testdir = os.path.dirname(path)
> + self._tmpname = os.path.basename(path)
> self.errpath = os.path.join(self._testdir, b'%s.err' % self.bname)
>
> @@ -651,5 +652,5 @@ class Test(unittest.TestCase):
> raise
>
> - name = os.path.basename(self.path)
> + name = self._tmpname
> self._testtmp = os.path.join(self._threadtmp, name)
> os.mkdir(self._testtmp)
> @@ -1061,4 +1062,16 @@ class TTest(Test):
> ESCAPEMAP.update({b'\\': b'\\\\', b'\r': br'\r'})
>
> + def __init__(self, *args, **kwds):
> + # accept an extra "case" parameter
> + case = None
> + if 'case' in kwds:
> + case = kwds.pop('case')
> + self._case = case
> + super(TTest, self).__init__(*args, **kwds)
> + if case:
> + self.name += b' (case %s)' % case
> + self.errpath = b'%s.%s.err' % (self.errpath[:-4], case)
> + self._tmpname += b'-%s' % case
> +
> @property
> def refpath(self):
> @@ -1147,4 +1160,7 @@ class TTest(Test):
> inpython = False
>
> + # What "case" we're currently in
> + incases = None
> +
> if self._debug:
> script.append(b'set -x\n')
> @@ -1158,4 +1174,13 @@ class TTest(Test):
> if not l.endswith(b'\n'):
> l += b'\n'
> + # case handling
> + if l.startswith(b'#case '):
> + incases = set(l[6:].split())
> + elif l.startswith(b'#endcase'):
> + incases = None
> + if incases is not None and self._case not in incases:
> + after.setdefault(pos, []).append(l)
> + continue
> + # normal parsing
> if l.startswith(b'#require'):
> lsplit = l.split()
> @@ -2269,7 +2294,27 @@ class TestRunner(object):
> args = os.listdir(b'.')
>
> - return [{'path': t} for t in args
> - if os.path.basename(t).startswith(b'test-')
> - and (t.endswith(b'.py') or t.endswith(b'.t'))]
> + tests = []
> + for t in args:
> + if not (os.path.basename(t).startswith(b'test-')
> + and (t.endswith(b'.py') or t.endswith(b'.t'))):
> + continue
> + if t.endswith(b'.t'):
> + # a .t file may contain multiple test cases
> + # "#case X" indicates a new test case
> + cases = set()
> + try:
> + with open(t, 'rb') as f:
> + for l in f:
> + if l.startswith(b'#case '):
> + cases.update(l[6:].split())
> + except IOError:
> + pass
> + if not cases:
> + tests.append({'path': t})
> + else:
> + tests += [{'path': t, 'case': c} for c in
> sorted(cases)]
> + else:
> + tests.append({'path': t})
> + return tests
>
> def _runtests(self, tests):
> @@ -2277,4 +2322,7 @@ class TestRunner(object):
> # convert a test back to its description dict
> desc = {'path': test.path}
> + case = getattr(test, '_case', None)
> + if case:
> + desc['case'] = case
> return self._gettest(desc, i)
>
> @@ -2375,4 +2423,10 @@ class TestRunner(object):
> tmpdir = os.path.join(self._hgtmp, b'child%d' % count)
>
> + # extra keyword parameters. 'case' is used by .t tests
> + kwds = {}
> + for key in ['case']:
> + if key in test:
> + kwds[key] = test[key]
>
Nit: you don't need the for loop here.
> +
> t = testcls(refpath, tmpdir,
> keeptmpdir=self.options.keep_tmpdir,
> @@ -2385,5 +2439,5 @@ class TestRunner(object):
> hgcommand=self._hgcommand,
> usechg=bool(self.options.with_chg or
> self.options.chg),
> - useipv6=useipv6)
> + useipv6=useipv6, **kwds)
> t.should_reload = True
> return t
> diff --git a/tests/test-run-tests.t b/tests/test-run-tests.t
> --- a/tests/test-run-tests.t
> +++ b/tests/test-run-tests.t
> @@ -901,2 +901,51 @@ support for bisecting failed tests autom
> python hash seed: * (glob)
> [1]
> +
> + $ cd ..
> +
> +Test cases in .t files
> +======================
> + $ mkdir cases
> + $ cd cases
> + $ cat > test-cases-abc.t <<'EOF'
> + > #case B
> + > $ V=B
> + > #case A
> + > $ V=A
> + > #case C
> + > $ V=C
> + > #endcase
> + > $ echo $V | sed 's/A/C/'
> + > C
> + > #case C
> + > $ [ $V == C ]
> + > #case A
> + > $ [ $V == C ]
> + > [1]
> + > #case A B
> + > $ [ $V == C ]
> + > [1]
> + > #endcase
> + > $ [ $V == D ]
> + > [1]
> + > EOF
> + $ rt
> + .
> + --- $TESTTMP/anothertests/cases/test-cases-abc.t
> + +++ $TESTTMP/anothertests/cases/test-cases-abc.t.B.err
> + @@ -6,7 +6,7 @@
> + $ V=C
> + #endcase
> + $ echo $V | sed 's/A/C/'
> + - C
> + + B
> + #case C
> + $ [ $V == C ]
> + #case A
> +
> + ERROR: test-cases-abc.t (case B) output changed
> + !.
> + Failed test-cases-abc.t (case B): output changed
> + # Ran 3 tests, 0 skipped, 0 warned, 1 failed.
> + python hash seed: * (glob)
> + [1]
>
Could you please add a test variation running with -v so we have better
test coverage of cases in test names?
> _______________________________________________
> Mercurial-devel mailing list
> Mercurial-devel at mercurial-scm.org
> https://www.mercurial-scm.org/mailman/listinfo/mercurial-devel
>
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://www.mercurial-scm.org/pipermail/mercurial-devel/attachments/20170504/69275381/attachment.html>
More information about the Mercurial-devel
mailing list