[PATCH 2 of 3] run-tests: support multiple cases in .t test

Pierre-Yves David pierre-yves.david at ens-lyon.org
Wed May 3 09:07:10 EDT 2017


I've not looked at the implementation yet, but I'm greatly in favor of 
having this kind of capability available. I found myself needing them on 
a regular basis.

On 05/03/2017 04:01 AM, Jun Wu wrote:
> # HG changeset patch
> # User Jun Wu <quark at fb.com>
> # Date 1493488388 25200
> #      Sat Apr 29 10:53:08 2017 -0700
> # Node ID 6663e696ae0c2a1b8f6fd735dfcf9b99fb4dac73
> # Parent  f32a5c7a590fd5d8a9d6c8195b6171331ec9f3a8
> # Available At https://bitbucket.org/quark-zju/hg-draft
> #              hg pull https://bitbucket.org/quark-zju/hg-draft -r 6663e696ae0c
> run-tests: support multiple cases in .t test
>
> Sometimes we want to run similar tests with slightly different
> configurations. Previously we duplicate the test files. This patch
> introduces special "#case" syntax that allows a single .t file to contain
> multiple test cases.
>
> For example, if a test should behave the same with or without an
> experimental flag, we can add the following to the .t header:
>
>     #case default
>        ....
>     #case experimental-a
>       $ cat >> $HGRCPATH << EOF
>       > [experimental]
>       > feature=a
>       > EOF
>     #endcase
>
> The "experimental-a" block won't be executed when running the "default" test
> case.
>
> In the test body, if there are case-dependent outputs, we can also gate them
> using the same syntax:
>
>     #case default
>       $ hg strip X
>       ... save backup bundle ...
>     #case experimental-a
>       $ hg strip X
>       ... pruned ...
>     #endcase
>
> The implementation allows "#case" to specify multiple values, to make it
> easier to test combinations affected by multiple factors. For example:
>
>     #case A B
>       $ flag="$flag --config experimental.factor-a=1"
>     #case A C
>       $ flag="$flag --config experimental.factor-b=1"
>     #case D
>     #endcase
>
> A later patch will make use of this feature to simplify test-hardlink*.t.
>
> diff --git a/tests/run-tests.py b/tests/run-tests.py
> --- a/tests/run-tests.py
> +++ b/tests/run-tests.py
> @@ -592,4 +592,5 @@ class Test(unittest.TestCase):
>          self.name = _strpath(self.bname)
>          self._testdir = os.path.dirname(path)
> +        self._tmpname = os.path.basename(path)
>          self.errpath = os.path.join(self._testdir, b'%s.err' % self.bname)
>
> @@ -651,5 +652,5 @@ class Test(unittest.TestCase):
>                  raise
>
> -        name = os.path.basename(self.path)
> +        name = self._tmpname
>          self._testtmp = os.path.join(self._threadtmp, name)
>          os.mkdir(self._testtmp)
> @@ -1061,4 +1062,16 @@ class TTest(Test):
>      ESCAPEMAP.update({b'\\': b'\\\\', b'\r': br'\r'})
>
> +    def __init__(self, *args, **kwds):
> +        # accept an extra "case" parameter
> +        case = None
> +        if 'case' in kwds:
> +            case = kwds.pop('case')
> +        self._case = case
> +        super(TTest, self).__init__(*args, **kwds)
> +        if case:
> +            self.name += b' (case %s)' % case
> +            self.errpath = b'%s.%s.err' % (self.errpath[:-4], case)
> +            self._tmpname += b'-%s' % case
> +
>      @property
>      def refpath(self):
> @@ -1147,4 +1160,7 @@ class TTest(Test):
>          inpython = False
>
> +        # What "case" we're currently in
> +        incases = None
> +
>          if self._debug:
>              script.append(b'set -x\n')
> @@ -1158,4 +1174,13 @@ class TTest(Test):
>              if not l.endswith(b'\n'):
>                  l += b'\n'
> +            # case handling
> +            if l.startswith(b'#case '):
> +                incases = set(l[6:].split())
> +            elif l.startswith(b'#endcase'):
> +                incases = None
> +            if incases is not None and self._case not in incases:
> +                after.setdefault(pos, []).append(l)
> +                continue
> +            # normal parsing
>              if l.startswith(b'#require'):
>                  lsplit = l.split()
> @@ -2269,7 +2294,27 @@ class TestRunner(object):
>                  args = os.listdir(b'.')
>
> -        return [{'path': t} for t in args
> -                if os.path.basename(t).startswith(b'test-')
> -                    and (t.endswith(b'.py') or t.endswith(b'.t'))]
> +        tests = []
> +        for t in args:
> +            if not (os.path.basename(t).startswith(b'test-')
> +                    and (t.endswith(b'.py') or t.endswith(b'.t'))):
> +                continue
> +            if t.endswith(b'.t'):
> +                # a .t file may contain multiple test cases
> +                # "#case X" indicates a new test case
> +                cases = set()
> +                try:
> +                    with open(t, 'rb') as f:
> +                        for l in f:
> +                            if l.startswith(b'#case '):
> +                                cases.update(l[6:].split())
> +                except IOError:
> +                    pass
> +                if not cases:
> +                    tests.append({'path': t})
> +                else:
> +                    tests += [{'path': t, 'case': c} for c in sorted(cases)]
> +            else:
> +                tests.append({'path': t})
> +        return tests
>
>      def _runtests(self, tests):
> @@ -2277,4 +2322,7 @@ class TestRunner(object):
>              # convert a test back to its description dict
>              desc = {'path': test.path}
> +            case = getattr(test, '_case', None)
> +            if case:
> +                desc['case'] = case
>              return self._gettest(desc, i)
>
> @@ -2375,4 +2423,10 @@ class TestRunner(object):
>          tmpdir = os.path.join(self._hgtmp, b'child%d' % count)
>
> +        # extra keyword parameters. 'case' is used by .t tests
> +        kwds = {}
> +        for key in ['case']:
> +            if key in test:
> +                kwds[key] = test[key]
> +
>          t = testcls(refpath, tmpdir,
>                      keeptmpdir=self.options.keep_tmpdir,
> @@ -2385,5 +2439,5 @@ class TestRunner(object):
>                      hgcommand=self._hgcommand,
>                      usechg=bool(self.options.with_chg or self.options.chg),
> -                    useipv6=useipv6)
> +                    useipv6=useipv6, **kwds)
>          t.should_reload = True
>          return t
> diff --git a/tests/test-run-tests.t b/tests/test-run-tests.t
> --- a/tests/test-run-tests.t
> +++ b/tests/test-run-tests.t
> @@ -901,2 +901,51 @@ support for bisecting failed tests autom
>    python hash seed: * (glob)
>    [1]
> +
> +  $ cd ..
> +
> +Test cases in .t files
> +======================
> +  $ mkdir cases
> +  $ cd cases
> +  $ cat > test-cases-abc.t <<'EOF'
> +  > #case B
> +  >   $ V=B
> +  > #case A
> +  >   $ V=A
> +  > #case C
> +  >   $ V=C
> +  > #endcase
> +  >   $ echo $V | sed 's/A/C/'
> +  >   C
> +  > #case C
> +  >   $ [ $V == C ]
> +  > #case A
> +  >   $ [ $V == C ]
> +  >   [1]
> +  > #case A B
> +  >   $ [ $V == C ]
> +  >   [1]
> +  > #endcase
> +  >   $ [ $V == D ]
> +  >   [1]
> +  > EOF
> +  $ rt
> +  .
> +  --- $TESTTMP/anothertests/cases/test-cases-abc.t
> +  +++ $TESTTMP/anothertests/cases/test-cases-abc.t.B.err
> +  @@ -6,7 +6,7 @@
> +     $ V=C
> +   #endcase
> +     $ echo $V | sed 's/A/C/'
> +  -  C
> +  +  B
> +   #case C
> +     $ [ $V == C ]
> +   #case A
> +
> +  ERROR: test-cases-abc.t (case B) output changed
> +  !.
> +  Failed test-cases-abc.t (case B): output changed
> +  # Ran 3 tests, 0 skipped, 0 warned, 1 failed.
> +  python hash seed: * (glob)
> +  [1]
> _______________________________________________
> Mercurial-devel mailing list
> Mercurial-devel at mercurial-scm.org
> https://www.mercurial-scm.org/mailman/listinfo/mercurial-devel
>

-- 
Pierre-Yves David


More information about the Mercurial-devel mailing list