D1477: run-tests: mechanism to report exceptions during test execution

[indygreg (Gregory Szorc)]

This revision was automatically updated to reflect the committed changes.
Closed by commit rHGbd8875b6473c: run-tests: mechanism to report exceptions during test execution (authored by indygreg, committed by ).

REPOSITORY
  rHG Mercurial

CHANGES SINCE LAST UPDATE
  https://phab.mercurial-scm.org/D1477?vs=3717&id=4073

REVISION DETAIL
  https://phab.mercurial-scm.org/D1477

AFFECTED FILES
  .hgignore
  tests/logexceptions.py
  tests/run-tests.py

CHANGE DETAILS

diff --git a/tests/run-tests.py b/tests/run-tests.py
--- a/tests/run-tests.py
+++ b/tests/run-tests.py
@@ -46,6 +46,7 @@
 from __future__ import absolute_import, print_function
 
 import argparse
+import collections
 import difflib
 import distutils.version as version
 import errno
@@ -373,7 +374,7 @@
         help="install and use chg wrapper in place of hg")
     hgconf.add_argument("--compiler",
         help="compiler to build with")
-    hgconf.add_argument('--extra-config-opt', action="append",
+    hgconf.add_argument('--extra-config-opt', action="append", default=[],
         help='set the given config opt in the test hgrc')
     hgconf.add_argument("-l", "--local", action="store_true",
         help="shortcut for --with-hg=<testdir>/../hg, "
@@ -404,6 +405,8 @@
         help="colorisation: always|auto|never (default: auto)")
     reporting.add_argument("-c", "--cover", action="store_true",
         help="print a test coverage report")
+    reporting.add_argument('--exceptions', action='store_true',
+        help='log all exceptions and generate an exception report')
     reporting.add_argument("-H", "--htmlcov", action="store_true",
         help="create an HTML report of the coverage of the files")
     reporting.add_argument("--json", action="store_true",
@@ -2115,6 +2118,18 @@
                     os.environ['PYTHONHASHSEED'])
             if self._runner.options.time:
                 self.printtimes(result.times)
+
+            if self._runner.options.exceptions:
+                exceptions = aggregateexceptions(
+                    os.path.join(self._runner._outputdir, b'exceptions'))
+                total = sum(exceptions.values())
+
+                self.stream.writeln('Exceptions Report:')
+                self.stream.writeln('%d total from %d frames' %
+                                    (total, len(exceptions)))
+                for (frame, line, exc), count in exceptions.most_common():
+                    self.stream.writeln('%d\t%s: %s' % (count, frame, exc))
+
             self.stream.flush()
 
         return result
@@ -2501,6 +2516,23 @@
 
         self._coveragefile = os.path.join(self._testdir, b'.coverage')
 
+        if self.options.exceptions:
+            exceptionsdir = os.path.join(self._outputdir, b'exceptions')
+            try:
+                os.makedirs(exceptionsdir)
+            except OSError as e:
+                if e.errno != errno.EEXIST:
+                    raise
+
+            # Remove all existing exception reports.
+            for f in os.listdir(exceptionsdir):
+                os.unlink(os.path.join(exceptionsdir, f))
+
+            osenvironb[b'HGEXCEPTIONSDIR'] = exceptionsdir
+            logexceptions = os.path.join(self._testdir, b'logexceptions.py')
+            self.options.extra_config_opt.append(
+                'extensions.logexceptions=%s' % logexceptions.decode('utf-8'))
+
         vlog("# Using TESTDIR", self._testdir)
         vlog("# Using RUNTESTDIR", osenvironb[b'RUNTESTDIR'])
         vlog("# Using HGTMP", self._hgtmp)
@@ -2953,6 +2985,24 @@
                 print("WARNING: Did not find prerequisite tool: %s " %
                       p.decode("utf-8"))
 
+def aggregateexceptions(path):
+    exceptions = collections.Counter()
+
+    for f in os.listdir(path):
+        with open(os.path.join(path, f), 'rb') as fh:
+            data = fh.read().split(b'\0')
+            if len(data) != 4:
+                continue
+
+            exc, mainframe, hgframe, hgline = data
+            exc = exc.decode('utf-8')
+            mainframe = mainframe.decode('utf-8')
+            hgframe = hgframe.decode('utf-8')
+            hgline = hgline.decode('utf-8')
+            exceptions[(hgframe, hgline, exc)] += 1
+
+    return exceptions
+
 if __name__ == '__main__':
     runner = TestRunner()
 
diff --git a/tests/logexceptions.py b/tests/logexceptions.py
new file mode 100644
--- /dev/null
+++ b/tests/logexceptions.py
@@ -0,0 +1,73 @@
+# logexceptions.py - Write files containing info about Mercurial exceptions
+#
+# Copyright 2017 Matt Mackall <mpm at selenic.com>
+#
+# This software may be used and distributed according to the terms of the
+# GNU General Public License version 2 or any later version.
+
+from __future__ import absolute_import
+
+import inspect
+import os
+import sys
+import traceback
+import uuid
+
+from mercurial import (
+    dispatch,
+    extensions,
+)
+
+def handleexception(orig, ui):
+    res = orig(ui)
+
+    if not ui.environ.get(b'HGEXCEPTIONSDIR'):
+        return res
+
+    dest = os.path.join(ui.environ[b'HGEXCEPTIONSDIR'],
+                        str(uuid.uuid4()).encode('ascii'))
+
+    exc_type, exc_value, exc_tb = sys.exc_info()
+
+    stack = []
+    tb = exc_tb
+    while tb:
+        stack.append(tb)
+        tb = tb.tb_next
+    stack.reverse()
+
+    hgframe = 'unknown'
+    hgline = 'unknown'
+
+    # Find the first Mercurial frame in the stack.
+    for tb in stack:
+        mod = inspect.getmodule(tb)
+        if not mod.__name__.startswith(('hg', 'mercurial')):
+            continue
+
+        frame = tb.tb_frame
+
+        try:
+            with open(inspect.getsourcefile(tb), 'r') as fh:
+                hgline = fh.readlines()[frame.f_lineno - 1].strip()
+        except (IndexError, OSError):
+            pass
+
+        hgframe = '%s:%d' % (frame.f_code.co_filename, frame.f_lineno)
+        break
+
+    primary = traceback.extract_tb(exc_tb)[-1]
+    primaryframe = '%s:%d' % (primary.filename, primary.lineno)
+
+    with open(dest, 'wb') as fh:
+        parts = [
+            str(exc_value),
+            primaryframe,
+            hgframe,
+            hgline,
+        ]
+        fh.write(b'\0'.join(p.encode('utf-8', 'replace') for p in parts))
+
+def extsetup(ui):
+    extensions.wrapfunction(dispatch, 'handlecommandexception',
+                            handleexception)
diff --git a/.hgignore b/.hgignore
--- a/.hgignore
+++ b/.hgignore
@@ -24,6 +24,7 @@
 tests/.hypothesis
 tests/hypothesis-generated
 tests/annotated
+tests/exceptions
 tests/*.err
 tests/htmlcov
 build



To: indygreg, #hg-reviewers, yuja
Cc: mercurial-devel