[PATCH] run-tests: add support for automatically bisecting test failures

[Kevin Bullock]

> On Mar 19, 2016, at 11:40, Augie Fackler <raf at durin42.com> wrote:
> 
> # HG changeset patch
> # User Augie Fackler <augie at google.com>
> # Date 1458411970 14400
> #      Sat Mar 19 14:26:10 2016 -0400
> # Node ID 5b6812f29c7dd80b827ea46c6c4ae611401f1985
> # Parent  b7a31068cc80be5b848deaa6bda9ae117fcc13c8
> run-tests: add support for automatically bisecting test failures
> 
> diff --git a/tests/run-tests.py b/tests/run-tests.py
> --- a/tests/run-tests.py
> +++ b/tests/run-tests.py
> @@ -265,6 +265,10 @@ def getparser():
>                       help='allow extremely slow tests')
>     parser.add_option('--showchannels', action='store_true',
>                       help='show scheduling channels')
> +    parser.add_option('--known-good-rev', type="string",
> +                      metavar="known_good_rev",
> +                      help=("Automatically bisect any failures using this "
> +                            "revision as a known-good revision."))
> 
>     for option, (envvar, default) in defaults.items():
>         defaults[option] = type(default)(os.environ.get(envvar, default))
> @@ -1774,6 +1778,40 @@ class TextTestRunner(unittest.TextTestRu
>             self._runner._checkhglib('Tested')
> 
>             savetimes(self._runner._testdir, result)
> +
> +            if failed and self._runner.options.known_good_rev:
> +                def nooutput(args):
> +                    p = subprocess.Popen(args, stderr=subprocess.STDOUT,
> +                                         stdout=subprocess.PIPE)
> +                    p.stdout.read()
> +                    p.wait()
> +                for test, msg in result.failures:
> +                    nooutput(['hg', 'bisect', '--reset']),
> +                    nooutput(['hg', 'bisect', '--bad', '.'])
> +                    nooutput(['hg', 'bisect', '--good',
> +                              self._runner.options.known_good_rev])
> +                    # TODO: we probably need to forward some options
> +                    # that alter hg's behavior inside the tests.
> +                    rtc = '%s %s %s' % (sys.executable, sys.argv[0], test)
> +                    sub = subprocess.Popen(['hg', 'bisect', '--command', rtc],
> +                                           stderr=subprocess.STDOUT,
> +                                           stdout=subprocess.PIPE)
> +                    data = sub.stdout.read()
> +                    sub.wait()
> +                    m = re.search(
> +                        (r'\nThe first (?P<goodbad>bad|good) revision '
> +                         r'is:\nchangeset: +\d:(?P<node>[a-f0-9]+)\n.*\n'
> +                         r'summary: +(?P<summary>[^\n]+)\n'),
> +                        data, (re.MULTILINE | re.DOTALL))
> +                    if m:
> +                        dat = m.groupdict()
> +                        verb = 'broken' if dat['goodbad'] == 'bad' else 'fixed'
> +                        self.stream.writeln(
> +                            '%s %s by %s (%s)' % (
> +                                test, verb, dat['node'], dat['summary']))
Nit: `continue` here and then outdent the `else` clause? Or it might make sense to flip the conditional to a guard clause

if not m:
    self.stream.writeln(...)
    continue
dat = m.groupdict()
...

LGTM otherwise.

pacem in terris / мир / शान्ति / ‎‫سَلاَم‬ / 平和
Kevin R. Bullock