[PATCH 2 of 4 runs-per-test-flag] run-tests: avoid running the same test instance concurrently

[Augie Fackler]

# HG changeset patch
# User Augie Fackler <augie at google.com>
# Date 1426265236 14400
#      Fri Mar 13 12:47:16 2015 -0400
# Node ID 3c7d06b719c5abfd508ca4d609a9a8c64fafaed8
# Parent  06e650cb8598b9503f81c2b3ae4e2d6847701d41
run-tests: avoid running the same test instance concurrently

There's a fair amount of mutable state stored on test case
instances. That causes many weird failures if you try to do something
like `run-tests.py -j16 --loop test-help.t`. The quick fix is this
slightly weird test-reloading dance, which ensures that every time a
test is executed it runs on a fresh instance of the TestCase subclass.

diff --git a/tests/run-tests.py b/tests/run-tests.py
--- a/tests/run-tests.py
+++ b/tests/run-tests.py
@@ -1291,6 +1291,7 @@ class TestSuite(unittest.TestSuite):
 
     def __init__(self, testdir, jobs=1, whitelist=None, blacklist=None,
                  retest=False, keywords=None, loop=False, runs_per_test=1,
+                 loadtest=None,
                  *args, **kwargs):
         """Create a new instance that can run tests with a configuration.
 
@@ -1326,13 +1327,20 @@ class TestSuite(unittest.TestSuite):
         self._keywords = keywords
         self._loop = loop
         self._runs_per_test = runs_per_test
+        self._loadtest = loadtest
 
     def run(self, result):
         # We have a number of filters that need to be applied. We do this
         # here instead of inside Test because it makes the running logic for
         # Test simpler.
         tests = []
+        num_tests = [0]
         for test in self._tests:
+            def get():
+                num_tests[0] += 1
+                if getattr(test, 'should_reload', False):
+                    return self._loadtest(test.name, num_tests[0])
+                return test
             if not os.path.exists(test.path):
                 result.addSkip(test, "Doesn't exist")
                 continue
@@ -1360,7 +1368,7 @@ class TestSuite(unittest.TestSuite):
                     if ignored:
                         continue
             for _ in xrange(self._runs_per_test):
-                tests.append(test)
+                tests.append(get())
 
         runtests = list(tests)
         done = queue.Queue()
@@ -1389,7 +1397,11 @@ class TestSuite(unittest.TestSuite):
                 if tests and not running == self._jobs:
                     test = tests.pop(0)
                     if self._loop:
-                        tests.append(test)
+                        if getattr(test, 'should_reload', False):
+                            num_tests[0] += 1
+                            tests.append(self._loadtest(test.name, num_tests[0]))
+                        else:
+                            tests.append(test)
                     t = threading.Thread(target=job, name=test.name,
                                          args=(test, result))
                     t.start()
@@ -1733,7 +1745,7 @@ class TestRunner(object):
                               keywords=self.options.keywords,
                               loop=self.options.loop,
                               runs_per_test=self.options.runs_per_test,
-                              tests=tests)
+                              tests=tests, loadtest=self._gettest)
             verbosity = 1
             if self.options.verbose:
                 verbosity = 2
@@ -1773,14 +1785,16 @@ class TestRunner(object):
         refpath = os.path.join(self._testdir, test)
         tmpdir = os.path.join(self._hgtmp, 'child%d' % count)
 
-        return testcls(refpath, tmpdir,
-                       keeptmpdir=self.options.keep_tmpdir,
-                       debug=self.options.debug,
-                       timeout=self.options.timeout,
-                       startport=self.options.port + count * 3,
-                       extraconfigopts=self.options.extra_config_opt,
-                       py3kwarnings=self.options.py3k_warnings,
-                       shell=self.options.shell)
+        t = testcls(refpath, tmpdir,
+                    keeptmpdir=self.options.keep_tmpdir,
+                    debug=self.options.debug,
+                    timeout=self.options.timeout,
+                    startport=self.options.port + count * 3,
+                    extraconfigopts=self.options.extra_config_opt,
+                    py3kwarnings=self.options.py3k_warnings,
+                    shell=self.options.shell)
+        t.should_reload = True
+        return t
 
     def _cleanup(self):
         """Clean up state from this test invocation."""