[PATCH 5 of 8 zstd-revlogs] perf: support multiple compression engines in perfrevlogchunks

[Gregory Szorc]

# HG changeset patch
# User Gregory Szorc <gregory.szorc at gmail.com>
# Date 1483387328 28800
#      Mon Jan 02 12:02:08 2017 -0800
# Node ID 9079e3d9ed5d7dbb1f108363c4dc37f5cdd4c7b7
# Parent  fd6d4d5fd1a60507c148b31113a845c99663853e
perf: support multiple compression engines in perfrevlogchunks

Now that the revlog has a reference to a compressor, it is
possible to swap in other compression engines. So, teach
`hg perfrevlogchunks` to do that.

The default behavior of `hg perfrevlogchunks` is now to measure the
compression performance of all compression engines implementing the
revlog compressor API. This effectively adds the no-op "none"
compressor and zstd (when available) into the default set.

While we can't yet plug alternate compressors into revlogs, this
command gives us a preview of the performance. On the mozilla-unified
repository:

$ hg perfrevlogchunks -c
! compress w/ none
! wall 0.115159 comb 0.110000 user 0.110000 sys 0.000000 (best of 86)
! compress w/ zlib
! wall 5.681406 comb 5.680000 user 5.680000 sys 0.000000 (best of 3)
! compress w/ zstd
! wall 2.624781 comb 2.620000 user 2.620000 sys 0.000000 (best of 4)

$ hg perfrevlogchunks -m
! compress w/ none
! wall 0.124486 comb 0.120000 user 0.120000 sys 0.000000 (best of 79)
! compress w/ zlib
! wall 10.144701 comb 10.150000 user 10.150000 sys 0.000000 (best of 3)
! compress w/ zstd
! wall 4.383118 comb 4.390000 user 4.390000 sys 0.000000 (best of 3)

Those numbers for zstd look promising. But they aren't the full story.
For that, we'll need to look at decompression times and storage sizes.
Stay tuned...

diff --git a/contrib/perf.py b/contrib/perf.py
--- a/contrib/perf.py
+++ b/contrib/perf.py
@@ -860,9 +860,10 @@ def perfrevlog(ui, repo, file_=None, sta
     fm.end()
 
 @command('perfrevlogchunks', revlogopts + formatteropts +
-         [('s', 'startrev', 0, 'revision to start at')],
+         [('e', 'engines', '', 'compression engines to use'),
+          ('s', 'startrev', 0, 'revision to start at')],
          '-c|-m|FILE')
-def perfrevlogchunks(ui, repo, file_=None, startrev=0, **opts):
+def perfrevlogchunks(ui, repo, file_=None, engines=None, startrev=0, **opts):
     """Benchmark operations on revlog chunks.
 
     Logically, each revlog is a collection of fulltext revisions. However,
@@ -875,6 +876,26 @@ def perfrevlogchunks(ui, repo, file_=Non
     see ``perfrevlog`` and ``perfrevlogrevision``.
     """
     rl = cmdutil.openrevlog(repo, 'perfrevlogchunks', file_, opts)
+
+    # Verify engines argument.
+    if engines:
+        engines = set(e.strip() for e in engines.split(','))
+        for engine in engines:
+            try:
+                util.compressionengines[engine]
+            except KeyError:
+                raise error.Abort('unknown compression engine: %s' % engine)
+    else:
+        engines = []
+        for e in util.compengines:
+            engine = util.compengines[e]
+            try:
+                if engine.available():
+                    engine.revlogcompressor().compress('dummy')
+                    engines.append(e)
+            except NotImplementedError:
+                pass
+
     revs = list(rl.revs(startrev, len(rl) - 1))
 
     def rlfh(rl):
@@ -917,10 +938,17 @@ def perfrevlogchunks(ui, repo, file_=Non
         # Save chunks as a side-effect.
         chunks[0] = rl._chunks(revs, df=fh)
 
-    def docompress():
+    def docompress(compressor):
         rl.clearcaches()
-        for chunk in chunks[0]:
-            rl.compress(chunk)
+
+        try:
+            # Swap in the requested compression engine.
+            oldcompressor = rl._compressor
+            rl._compressor = compressor
+            for chunk in chunks[0]:
+                rl.compress(chunk)
+        finally:
+            rl._compressor = oldcompressor
 
     benches = [
         (lambda: doread(), 'read'),
@@ -929,9 +957,13 @@ def perfrevlogchunks(ui, repo, file_=Non
         (lambda: doreadbatchcachedfh(), 'read batch w/ reused fd'),
         (lambda: dochunk(), 'chunk'),
         (lambda: dochunkbatch(), 'chunk batch'),
-        (lambda: docompress(), 'compress'),
     ]
 
+    for engine in sorted(engines):
+        compressor = util.compengines[engine].revlogcompressor()
+        benches.append((functools.partial(docompress, compressor),
+                        'compress w/ %s' % engine))
+
     for fn, title in benches:
         timer, fm = gettimer(ui, opts)
         timer(fn, title=title)