[PATCH V5] run-tests: support multiple cases in .t test

Jun Wu quark at fb.com
Thu May 18 17:52:21 EDT 2017


Sorry for the fast resend. But this one does not introduce new Python3
compatibility issues.

Excerpts from Jun Wu's message of 2017-05-18 14:34:08 -0700:
> # HG changeset patch
> # User Jun Wu <quark at fb.com>
> # Date 1495001431 25200
> #      Tue May 16 23:10:31 2017 -0700
> # Node ID d11883f5611d11c7614624b4a0c6e628bcd4499f
> # Parent  0d6b3572ad924103128bb9cd296000fc6fd821ef
> # Available At https://bitbucket.org/quark-zju/hg-draft 
> #              hg pull https://bitbucket.org/quark-zju/hg-draft  -r d11883f5611d
> run-tests: support multiple cases in .t test
> 
> Sometimes we want to run similar tests with slightly different
> configurations. Previously we duplicate the test files. This patch
> introduces special "#testcases" syntax that allows a single .t file to
> contain multiple test cases.
> 
> Defined cases could be tested using "#if".
> 
> For example, if a test should behave the same with or without an
> experimental flag, we can add the following to the .t header:
> 
>     #testcases default experimental-a
>     #if experimental-a
>       $ cat >> $HGRCPATH << EOF
>       > [experimental]
>       > feature=a
>       > EOF
>     #endif
> 
> The "experimental-a" block won't be executed when running the "default" test
> case.
> 
> diff --git a/tests/run-tests.py b/tests/run-tests.py
> --- a/tests/run-tests.py
> +++ b/tests/run-tests.py
> @@ -217,4 +217,20 @@ def parselistfiles(files, listtype, warn
>      return entries
>  
> +def parsettestcases(path):
> +    """read a .t test file, return a set of test case names
> +
> +    If path does not exist, return an empty set.
> +    """
> +    cases = set()
> +    try:
> +        with open(path, 'rb') as f:
> +            for l in f:
> +                if l.startswith(b'#testcases '):
> +                    cases.update(l[11:].split())
> +    except IOError as ex:
> +        if ex.errno != errno.ENOENT:
> +            raise
> +    return cases
> +
>  def getparser():
>      """Obtain the OptionParser used by the CLI."""
> @@ -588,4 +604,5 @@ class Test(unittest.TestCase):
>          self.name = _strpath(self.bname)
>          self._testdir = os.path.dirname(path)
> +        self._tmpname = os.path.basename(path)
>          self.errpath = os.path.join(self._testdir, b'%s.err' % self.bname)
>  
> @@ -647,5 +664,5 @@ class Test(unittest.TestCase):
>                  raise
>  
> -        name = os.path.basename(self.path)
> +        name = self._tmpname
>          self._testtmp = os.path.join(self._threadtmp, name)
>          os.mkdir(self._testtmp)
> @@ -1056,4 +1073,17 @@ class TTest(Test):
>      ESCAPEMAP.update({b'\\': b'\\\\', b'\r': br'\r'})
>  
> +    def __init__(self, path, *args, **kwds):
> +        # accept an extra "case" parameter
> +        case = None
> +        if 'case' in kwds:
> +            case = kwds.pop('case')
> +        self._case = case
> +        self._allcases = parsettestcases(path)
> +        super(TTest, self).__init__(path, *args, **kwds)
> +        if case:
> +            self.name = '%s (case %s)' % (self.name, _strpath(case))
> +            self.errpath = b'%s.%s.err' % (self.errpath[:-4], case)
> +            self._tmpname += b'-%s' % case
> +
>      @property
>      def refpath(self):
> @@ -1111,4 +1141,18 @@ class TTest(Test):
>          return True, None
>  
> +    def _iftest(self, args):
> +        # implements "#if"
> +        reqs = []
> +        for arg in args:
> +            if arg.startswith(b'no-') and arg[3:] in self._allcases:
> +                if arg[3:] == self._case:
> +                    return False
> +            elif arg in self._allcases:
> +                if arg != self._case:
> +                    return False
> +            else:
> +                reqs.append(arg)
> +        return self._hghave(reqs)[0]
> +
>      def _parsetest(self, lines):
>          # We generate a shell script which outputs unique markers to line
> @@ -1168,5 +1212,5 @@ class TTest(Test):
>                  if skipping is not None:
>                      after.setdefault(pos, []).append('  !!! nested #if\n')
> -                skipping = not self._hghave(lsplit[1:])[0]
> +                skipping = not self._iftest(lsplit[1:])
>                  after.setdefault(pos, []).append(l)
>              elif l.startswith(b'#else'):
> @@ -2264,7 +2308,19 @@ class TestRunner(object):
>                  args = os.listdir(b'.')
>  
> -        return [{'path': t} for t in args
> -                if os.path.basename(t).startswith(b'test-')
> -                    and (t.endswith(b'.py') or t.endswith(b'.t'))]
> +        tests = []
> +        for t in args:
> +            if not (os.path.basename(t).startswith(b'test-')
> +                    and (t.endswith(b'.py') or t.endswith(b'.t'))):
> +                continue
> +            if t.endswith(b'.t'):
> +                # .t file may contain multiple test cases
> +                cases = sorted(parsettestcases(t))
> +                if cases:
> +                    tests += [{'path': t, 'case': c} for c in sorted(cases)]
> +                else:
> +                    tests.append({'path': t})
> +            else:
> +                tests.append({'path': t})
> +        return tests
>  
>      def _runtests(self, testdescs):
> @@ -2272,4 +2328,7 @@ class TestRunner(object):
>              # convert a test back to its description dict
>              desc = {'path': test.path}
> +            case = getattr(test, '_case', None)
> +            if case:
> +                desc['case'] = case
>              return self._gettest(desc, i)
>  
> @@ -2287,5 +2346,10 @@ class TestRunner(object):
>                  orig = list(testdescs)
>                  while testdescs:
> -                    if os.path.exists(testdescs[0]['path'] + ".err"):
> +                    desc = testdescs[0]
> +                    if 'case' in desc:
> +                        errpath = b'%s.%s.err' % (desc['path'], desc['case'])
> +                    else:
> +                        errpath = b'%s.err' % desc['path']
> +                    if os.path.exists(errpath):
>                          break
>                      testdescs.pop(0)
> @@ -2370,4 +2434,7 @@ class TestRunner(object):
>          tmpdir = os.path.join(self._hgtmp, b'child%d' % count)
>  
> +        # extra keyword parameters. 'case' is used by .t tests
> +        kwds = dict((k, testdesc[k]) for k in ['case'] if k in testdesc)
> +
>          t = testcls(refpath, tmpdir,
>                      keeptmpdir=self.options.keep_tmpdir,
> @@ -2380,5 +2447,5 @@ class TestRunner(object):
>                      hgcommand=self._hgcommand,
>                      usechg=bool(self.options.with_chg or self.options.chg),
> -                    useipv6=useipv6)
> +                    useipv6=useipv6, **kwds)
>          t.should_reload = True
>          return t
> diff --git a/tests/test-run-tests.t b/tests/test-run-tests.t
> --- a/tests/test-run-tests.t
> +++ b/tests/test-run-tests.t
> @@ -901,2 +901,77 @@ support for bisecting failed tests autom
>    python hash seed: * (glob)
>    [1]
> +
> +  $ cd ..
> +
> +Test cases in .t files
> +======================
> +  $ mkdir cases
> +  $ cd cases
> +  $ cat > test-cases-abc.t <<'EOF'
> +  > #testcases A B C
> +  >   $ V=B
> +  > #if A
> +  >   $ V=A
> +  > #endif
> +  > #if C
> +  >   $ V=C
> +  > #endif
> +  >   $ echo $V | sed 's/A/C/'
> +  >   C
> +  > #if C
> +  >   $ [ $V = C ]
> +  > #endif
> +  > #if A
> +  >   $ [ $V = C ]
> +  >   [1]
> +  > #endif
> +  > #if no-C
> +  >   $ [ $V = C ]
> +  >   [1]
> +  > #endif
> +  >   $ [ $V = D ]
> +  >   [1]
> +  > EOF
> +  $ rt
> +  .
> +  --- $TESTTMP/anothertests/cases/test-cases-abc.t
> +  +++ $TESTTMP/anothertests/cases/test-cases-abc.t.B.err
> +  @@ -7,7 +7,7 @@
> +     $ V=C
> +   #endif
> +     $ echo $V | sed 's/A/C/'
> +  -  C
> +  +  B
> +   #if C
> +     $ [ $V = C ]
> +   #endif
> +  
> +  ERROR: test-cases-abc.t (case B) output changed
> +  !.
> +  Failed test-cases-abc.t (case B): output changed
> +  # Ran 3 tests, 0 skipped, 0 warned, 1 failed.
> +  python hash seed: * (glob)
> +  [1]
> +
> +--restart works
> +
> +  $ rt --restart
> +  
> +  --- $TESTTMP/anothertests/cases/test-cases-abc.t
> +  +++ $TESTTMP/anothertests/cases/test-cases-abc.t.B.err
> +  @@ -7,7 +7,7 @@
> +     $ V=C
> +   #endif
> +     $ echo $V | sed 's/A/C/'
> +  -  C
> +  +  B
> +   #if C
> +     $ [ $V = C ]
> +   #endif
> +  
> +  ERROR: test-cases-abc.t (case B) output changed
> +  !.
> +  Failed test-cases-abc.t (case B): output changed
> +  # Ran 2 tests, 0 skipped, 0 warned, 1 failed.
> +  python hash seed: * (glob)
> +  [1]


More information about the Mercurial-devel mailing list