[PATCH V3] testing: generate tests operations using Hypothesis
David R. MacIver
david at drmaciver.com
Wed Feb 24 11:00:51 UTC 2016
# HG changeset patch
# User David R. MacIver <david at drmaciver.com>
# Date 1456235047 0
# Tue Feb 23 13:44:07 2016 +0000
# Node ID d49bca9cd31107685dba9f4e881ea1ff514f2065
# Parent 83fc0c0556644d742e3d6eecfb38e742005799cc
testing: generate tests operations using Hypothesis
The idea of this patch is to expand the use of Hypothesis
within Mercurial to use its concept of "stateful testing".
The result is a test which runs a sequence of operations
against a family of Mercurial repositories. Each operation
is given a set of allowed ways it can fail. Any other
non-zero exit code is a test failure.
At the end, the whole sequence is then reverified by
generating a .t test and testing it again in pure
mode (this is also useful for catching non-determinism
bugs).
This has proven reasonably effective at finding bugs,
and has identified two problems in the shelve extension
already (issue5113 and issue5112).
diff -r 83fc0c055664 -r d49bca9cd311 .hgignore
--- a/.hgignore Tue Jan 19 18:20:13 2016 +0000
+++ b/.hgignore Tue Feb 23 13:44:07 2016 +0000
@@ -21,6 +21,7 @@
.\#*
tests/.coverage*
tests/.testtimes*
+tests/.hypothesis
tests/annotated
tests/*.err
tests/htmlcov
diff -r 83fc0c055664 -r d49bca9cd311 tests/test-verify-repo-operations.py
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/tests/test-verify-repo-operations.py Tue Feb 23 13:44:07 2016 +0000
@@ -0,0 +1,545 @@
+from __future__ import print_function, absolute_import
+
+"""Fuzz testing for operations against a Mercurial repository
+
+This uses Hypothesis's stateful testing to generate random repository
+operations and test Mercurial using them, both to see if there are any
+unexpected errors and to compare different versions of it."""
+
+import base64
+from contextlib import contextmanager
+import errno
+import os
+import pipes
+import shutil
+import silenttestrunner
+import subprocess
+
+from hypothesis.extra.datetime import datetimes
+from hypothesis.errors import HypothesisException
+from hypothesis.stateful import (
+ rule, RuleBasedStateMachine, Bundle, precondition)
+import hypothesis.strategies as st
+from hypothesis import settings, note
+from hypothesis.configuration import set_hypothesis_home_dir
+from hypothesis.database import ExampleDatabase
+
+testdir = os.path.abspath(os.environ["TESTDIR"])
+
+# We store Hypothesis examples here rather in the temporary test directory
+# so that when rerunning a failing test this always results in refinding the
+# previous failure. This directory is in .hgignore and should not be checked in
+# but is useful to have for development.
+set_hypothesis_home_dir(os.path.join(testdir, ".hypothesis"))
+
+runtests = os.path.join(testdir, "run-tests.py")
+testtmp = os.environ["TESTTMP"]
+assert os.path.isdir(testtmp)
+
+generatedtests = os.path.join(testdir, "hypothesis-generated")
+
+try:
+ os.makedirs(generatedtests)
+except OSError:
+ pass
+
+# We write out generated .t files to a file in order to ease debugging and to
+# give a starting point for turning failures Hypothesis finds into normal
+# tests. In order to ensure that multiple copies of this test can be run in
+# parallel we use atomic file create to ensure that we always get a unique
+# name.
+file_index = 0
+while True:
+ file_index += 1
+ savefile = os.path.join(generatedtests, "test-generated-%d.t" % (
+ file_index,
+ ))
+ try:
+ os.close(os.open(savefile, os.O_CREAT | os.O_EXCL | os.O_WRONLY))
+ break
+ except OSError as e:
+ if e.errno != errno.EEXIST:
+ raise
+assert os.path.exists(savefile)
+
+hgrc = os.path.join(".hg", "hgrc")
+
+pathcharacters = (
+ "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789"
+ "[]^_`;=@{}~ !#$%&'()+,-"
+)
+
+files = st.text(pathcharacters, min_size=1).map(lambda x: x.strip()).filter(
+ bool).map(lambda s: s.encode('ascii'))
+
+safetext = st.text(st.characters(
+ min_codepoint=1, max_codepoint=127,
+ blacklist_categories=('Cc', 'Cs')), min_size=1).map(
+ lambda s: s.encode('utf-8')
+)
+
+ at contextmanager
+def acceptableerrors(*args):
+ """Sometimes we know an operation we're about to perform might fail, and
+ we're OK with some of the failures. In those cases this may be used as a
+ context manager and will swallow expected failures, as identified by
+ substrings of the error message Mercurial emits."""
+ try:
+ yield
+ except subprocess.CalledProcessError as e:
+ if not any(a in e.output for a in args):
+ note(e.output)
+ raise
+
+reponames = st.text("abcdefghijklmnopqrstuvwxyz01234556789", min_size=1).map(
+ lambda s: s.encode('ascii')
+)
+
+class verifyingstatemachine(RuleBasedStateMachine):
+ """This defines the set of acceptable operations on a Mercurial repository
+ using Hypothesis's RuleBasedStateMachine.
+
+ The general concept is that we manage multiple repositories inside a
+ repos/ directory in our temporary test location. Some of these are freshly
+ inited, some are clones of the others. Our current working directory is
+ always inside one of these repositories while the tests are running.
+
+ Hypothesis then performs a series of operations against these repositories,
+ including hg commands, generating contents and editing the .hgrc file.
+ If these operations fail in unexpected ways or behave differently in
+ different configurations of Mercurial, the test will fail and a minimized
+ .t test file will be written to the hypothesis-generated directory to
+ exhibit that failure.
+
+ Operations are defined as methods with @rule() decorators. See the
+ Hypothesis documentation at
+ http://hypothesis.readthedocs.org/en/release/stateful.html for more
+ details."""
+
+ # A bundle is a reusable collection of data that rules have previously
+ # generated which may be provided as arguments to future rules.
+ repos = Bundle('repos')
+ paths = Bundle('paths')
+ contents = Bundle('contents')
+ branches = Bundle('branches')
+ committimes = Bundle('committimes')
+
+ def __init__(self):
+ super(verifyingstatemachine, self).__init__()
+ self.repodir = os.path.join(testtmp, "repos")
+ assert self.repodir.startswith("/tmp/")
+ if os.path.exists(self.repodir):
+ shutil.rmtree(self.repodir)
+ os.chdir(testtmp)
+ self.log = []
+ self.failed = False
+
+ self.mkdirp("repos")
+ self.cd("repos")
+ self.mkdirp("repo1")
+ self.cd("repo1")
+ self.hg("init")
+ self.extensions = {}
+
+ def teardown(self):
+ """On teardown we clean up after ourselves as usual, but we also
+ do some additional testing: We generate a .t file based on our test
+ run using run-test.py -i to get the correct output.
+
+ We then test it in a number of other configurations, verifying that
+ each passes the same test."""
+ super(verifyingstatemachine, self).teardown()
+ try:
+ shutil.rmtree(self.repodir)
+ except OSError:
+ pass
+ ttest = "\n".join(" " + l for l in self.log)
+ os.chdir(testtmp)
+ path = os.path.join(testtmp, "test-generated.t")
+ with open(path, 'w') as o:
+ o.write(ttest)
+ o.write('\n')
+ o.close()
+ with open("/dev/null", "w") as devnull:
+ rewriter = subprocess.Popen(
+ [runtests, "--local", "-i", path], stdin=subprocess.PIPE,
+ stdout=devnull, stderr=devnull,
+ )
+ rewriter.communicate("yes")
+ with open(path, 'r') as i:
+ ttest = i.read()
+
+ e = None
+ if not self.failed:
+ try:
+ output = subprocess.check_output([
+ runtests, path, "--local", "--pure"
+ ], stderr=subprocess.STDOUT)
+ assert "Ran 1 test" in output, output
+ except subprocess.CalledProcessError as e:
+ note(e.output)
+ finally:
+ os.unlink(path)
+ try:
+ os.unlink(path + ".err")
+ except OSError:
+ pass
+ if self.failed or e is not None:
+ with open(savefile, "wb") as o:
+ o.write(ttest)
+ if e is not None:
+ raise e
+
+ @property
+ def currentrepo(self):
+ return os.path.basename(os.getcwd())
+
+ def mkdirp(self, path):
+ if os.path.exists(path):
+ return
+ self.log.append(
+ "$ mkdir -p -- %s" % (pipes.quote(os.path.relpath(path)),))
+ os.makedirs(path)
+
+ def cd(self, path):
+ path = os.path.relpath(path)
+ if path == ".":
+ return
+ os.chdir(path)
+ self.log.append("$ cd -- %s" % (pipes.quote(path),))
+
+ def hasextension(self, extension):
+ repo = self.currentrepo
+ return repo in self.extensions and extension in self.extensions[repo]
+
+ @rule(extension=st.sampled_from((
+ 'shelve', 'mq'
+ )))
+ def addextension(self, extension):
+ extensions = self.extensions.setdefault(self.currentrepo, set())
+ if extension in extensions:
+ return
+ extensions.add(extension)
+ if not os.path.exists(hgrc):
+ self.command("touch", hgrc)
+ with open(hgrc, "r") as i:
+ existing = i.read()
+ if "[extensions]" not in existing:
+ with open(hgrc, "a") as o:
+ o.write("[extensions]\n")
+ self.log.append("$ echo '[extensions]' >> %s" % (hgrc,))
+ with open(hgrc, 'a') as o:
+ line = "%s=" % (extension,)
+ o.write(line)
+ o.write("\n")
+ self.log.append("$ echo %s >> %s" % (line, hgrc))
+
+ @rule()
+ def dumprc(self):
+ if os.path.exists(hgrc):
+ self.command("cat", hgrc)
+
+ def hg(self, *args):
+ self.command("hg", *args)
+
+ def command(self, *args):
+ self.log.append("$ " + ' '.join(map(pipes.quote, args)))
+ subprocess.check_output(args, stderr=subprocess.STDOUT)
+
+ def execute_step(self, step):
+ try:
+ return super(verifyingstatemachine, self).execute_step(step)
+ except (HypothesisException, KeyboardInterrupt):
+ raise
+ except Exception:
+ self.failed = True
+ raise
+
+ @rule(
+ target=paths,
+ source=st.lists(files, min_size=1).map(
+ lambda l: os.path.join(*l)).filter(
+ lambda x: x and x[0] != "/" and x[-1] != "/"))
+ def genpath(self, source):
+ return source
+
+ @rule(
+ target=committimes,
+ when=datetimes(min_year=1970, max_year=2038) | st.none())
+ def gentime(self, when):
+ return when
+
+ @rule(
+ target=repos,
+ source=repos,
+ name=reponames,
+ )
+ def clone(self, source, name):
+ if not os.path.exists(os.path.join("..", name)):
+ self.cd("..")
+ self.hg("clone", source, name)
+ self.cd(name)
+ return name
+
+ @rule(
+ target=repos,
+ name=reponames,
+ )
+ def fresh(self, name):
+ if not os.path.exists(os.path.join("..", name)):
+ self.cd("..")
+ self.mkdirp(name)
+ self.cd(name)
+ self.hg("init")
+ return name
+
+ @rule(name=repos)
+ def switch(self, name):
+ self.cd(os.path.join("..", name))
+ assert self.currentrepo == name
+ assert os.path.exists(".hg")
+
+ @rule(target=repos)
+ def origin(self):
+ return "repo1"
+
+ @rule()
+ def pull(self, repo=repos):
+ with acceptableerrors("repository default not found"):
+ self.hg("pull")
+
+ @rule(newbranch=st.booleans())
+ def push(self, newbranch):
+ with acceptableerrors(
+ "default repository not configured",
+ "no changes found",
+ ):
+ if newbranch:
+ self.hg("push", "--new-branch")
+ else:
+ with acceptableerrors(
+ "creates new branches"
+ ):
+ self.hg("push")
+
+ @rule(
+ target=contents,
+ content=st.one_of(
+ st.binary(),
+ st.text().map(lambda x: x.encode('utf-8'))
+ ))
+ def gencontent(self, content):
+ return content
+
+ @rule(
+ target=branches,
+ name=safetext,
+ )
+ def genbranch(self, name):
+ return name
+
+ @rule(target=paths, source=paths)
+ def lowerpath(self, source):
+ return source.lower()
+
+ @rule(target=paths, source=paths)
+ def upperpath(self, source):
+ return source.upper()
+
+ @rule()
+ def log(self):
+ self.hg("log")
+
+ @rule()
+ def verify(self):
+ self.hg("verify")
+
+ @rule()
+ def diff(self):
+ self.hg("diff", "--nodates")
+
+ @rule()
+ def status(self):
+ self.hg("status")
+
+ @rule()
+ def checkbranch(self):
+ self.hg("branch")
+
+ @rule(branch=branches)
+ def switchbranch(self, branch):
+ with acceptableerrors(
+ 'cannot use an integer as a name',
+ 'be used in a name',
+ 'a branch of the same name already exists',
+ 'is reserved',
+ ):
+ self.hg("branch", "--", branch)
+
+ @rule(branch=branches, c=st.booleans())
+ def update(self, branch, c):
+ with acceptableerrors(
+ 'unknown revision',
+ 'parse error',
+ ):
+ if c:
+ self.hg("update", "-C", "--", branch)
+ else:
+ self.hg("update", "--", branch)
+
+ @rule(s=st.none() | st.integers(0, 100))
+ def addremove(self, s):
+ args = ["addremove"]
+ if s is not None:
+ args.extend(["-s", str(s)])
+ self.hg(*args)
+
+ @rule(path=paths)
+ def addpath(self, path):
+ if os.path.exists(path):
+ self.hg("add", "--", path)
+
+ @rule(path=paths)
+ def forgetpath(self, path):
+ if os.path.exists(path):
+ with acceptableerrors("file is already untracked"):
+ self.hg("forget", "--", path)
+
+ @rule(path=paths)
+ def removepath(self, path):
+ if os.path.exists(path):
+ with acceptableerrors(
+ 'file is untracked', 'file has been marked for add',
+ 'file is modified',
+ ):
+ self.hg("remove", "--", path)
+
+ @rule(
+ message=safetext,
+ amend=st.booleans(),
+ addremove=st.booleans(),
+ secret=st.booleans(),
+ close_branch=st.booleans(),
+ when=committimes,
+ )
+ def maybecommit(
+ self, message, amend, when, addremove, secret, close_branch
+ ):
+ command = ["commit"]
+ errors = ["nothing changed"]
+ if amend:
+ errors.append("cannot amend public changesets")
+ command.append("--amend")
+ command.append("-m" + pipes.quote(message))
+ if secret:
+ command.append("--secret")
+ if close_branch:
+ command.append("--close-branch")
+ errors.append("can only close branch heads")
+ if addremove:
+ command.append("--addremove")
+ if when is not None:
+ if when.year == 1970:
+ errors.append('negative date value')
+ if when.year == 2038:
+ errors.append('exceeds 32 bits')
+ command.append("--date=%s" % (
+ when.strftime('%Y-%m-%d %H:%M:%S %z'),))
+
+ with acceptableerrors(*errors):
+ self.hg(*command)
+
+ @rule(path=paths, content=contents)
+ def writecontent(self, path, content):
+ self.unadded_changes = True
+ if os.path.isdir(path):
+ return
+ parent = os.path.dirname(path)
+ if parent:
+ try:
+ self.mkdirp(parent)
+ except OSError:
+ return
+ with open(path, 'wb') as o:
+ o.write(content)
+ self.log.append("$ echo %s | base64 --decode > %s" % (
+ pipes.quote(base64.b64encode(content)),
+ pipes.quote(path),
+ ))
+
+ @rule()
+ def export(self):
+ self.hg("export")
+
+ @rule()
+ @precondition(lambda self: self.hasextension("shelve"))
+ def shelve(self):
+ with acceptableerrors("nothing changed"):
+ self.hg("shelve")
+
+ @rule()
+ @precondition(lambda self: self.hasextension("shelve"))
+ def unshelve(self):
+ with acceptableerrors("no shelved changes to apply"):
+ self.hg("unshelve")
+
+class writeonlydatabase(ExampleDatabase):
+ def __init__(self, underlying):
+ super(ExampleDatabase, self).__init__()
+ self.underlying = underlying
+
+ def fetch(self, key):
+ return ()
+
+ def save(self, key, value):
+ self.underlying.save(key, value)
+
+ def delete(self, key, value):
+ self.underlying.delete(key, value)
+
+ def close(self):
+ self.underlying.close()
+
+settings.register_profile(
+ 'default', settings(
+ timeout=300,
+ stateful_step_count=100,
+ max_examples=20,
+ )
+)
+
+settings.register_profile(
+ 'fast', settings(
+ timeout=10,
+ stateful_step_count=20,
+ max_examples=5,
+ min_satisfying_examples=1,
+ max_shrinks=0,
+ )
+)
+
+settings.register_profile(
+ 'continuous', settings(
+ timeout=-1,
+ stateful_step_count=10000,
+ max_examples=10 ** 8,
+ max_iterations=10 ** 8,
+ database=writeonlydatabase(settings.default.database)
+ )
+)
+
+settings.load_profile(os.getenv('HYPOTHESIS_PROFILE', 'default'))
+
+verifyingtest = verifyingstatemachine.TestCase
+
+verifyingtest.settings = settings.default
+
+if __name__ == '__main__':
+ try:
+ silenttestrunner.main(__name__)
+ finally:
+ # So as to prevent proliferation of useless test files, if we never
+ # actually wrote a failing test we clean up after ourselves and delete
+ # the file for doing so that we owned.
+ if os.path.exists(savefile) and os.path.getsize(savefile) == 0:
+ os.unlink(savefile)
More information about the Mercurial-devel
mailing list