[PATCH 6 of 6] bundle2: allow compressed bundle
Pierre-Yves David
pierre-yves.david at ens-lyon.org
Wed Sep 23 20:21:25 CDT 2015
# HG changeset patch
# User Pierre-Yves David <pierre-yves.david at fb.com>
# Date 1443038172 25200
# Wed Sep 23 12:56:12 2015 -0700
# Node ID 485eb2a02a2bfb9914ee9534af8f2ff133bf1331
# Parent 62ee35a3fcb068f72588b8fb594c4a7bfd163df7
bundle2: allow compressed bundle
This changesets adds support for a 'compression' parameter in bundle2 stream.
When set, it control the compression algorithm used for the payload part of the
bundle2.
There is currently no usage of this, but in test.
diff --git a/mercurial/bundle2.py b/mercurial/bundle2.py
--- a/mercurial/bundle2.py
+++ b/mercurial/bundle2.py
@@ -477,10 +477,19 @@ class bundle20(object):
def __init__(self, ui, capabilities=()):
self.ui = ui
self._params = []
self._parts = []
self.capabilities = dict(capabilities)
+ self._compressor = util.compressors[None]()
+
+ def setcompression(self, alg):
+ """setup core part compression to <alg>"""
+ if alg is None:
+ return
+ assert not any(n.lower() == 'Compression' for n, v in self._params)
+ self.addparam('Compression', alg)
+ self._compressor = util.compressors[alg]()
@property
def nbparts(self):
"""total number of parts added to the bundler"""
return len(self._parts)
@@ -528,12 +537,14 @@ class bundle20(object):
param = self._paramchunk()
outdebug(self.ui, 'bundle parameter: %s' % param)
yield _pack(_fstreamparamsize, len(param))
if param:
yield param
+ # starting compression
for chunk in self._getcorechunk():
- yield chunk
+ yield self._compressor.compress(chunk)
+ yield self._compressor.flush()
def _paramchunk(self):
"""return a encoded version of all stream parameters"""
blocks = []
for par, value in self._params:
@@ -631,10 +642,11 @@ class unbundle20(unpackermixin):
`iterparts` methods."""
def __init__(self, ui, fp):
"""If header is specified, we do not read it out of the stream."""
self.ui = ui
+ self._decompressor = util.decompressors[None]
super(unbundle20, self).__init__(fp)
@util.propertycache
def params(self):
"""dictionary of stream level parameters"""
@@ -680,10 +692,12 @@ class unbundle20(unpackermixin):
def iterparts(self):
"""yield all parts contained in the stream"""
# make sure param have been loaded
self.params
+ # From there, payload need to be decompressed
+ self._fp = self._decompressor(self._fp)
indebug(self.ui, 'start extraction of bundle2 parts')
headerblock = self._readpartheader()
while headerblock is not None:
part = unbundlepart(self.ui, headerblock, self._fp)
yield part
@@ -717,10 +731,18 @@ def b2streamparamhandler(name):
assert name not in formatmap
b2streamparamsmap[name] = func
return func
return decorator
+ at b2streamparamhandler('compression')
+def processcompression(unbundler, param, value):
+ """read compression parameter and install payload decompression"""
+ if value not in util.decompressors:
+ raise error.BundleUnknownFeatureError(params=(param,),
+ values=(value,))
+ unbundler._decompressor = util.decompressors[value]
+
class bundlepart(object):
"""A bundle2 part contains application level payload
The part `type` is used to route the part to the application level
handler.
diff --git a/tests/test-bundle2-format.t b/tests/test-bundle2-format.t
--- a/tests/test-bundle2-format.t
+++ b/tests/test-bundle2-format.t
@@ -77,11 +77,12 @@ Create an extension to test bundle2 API
> ('', 'parts', False, 'include some arbitrary parts to the bundle'),
> ('', 'reply', False, 'produce a reply bundle'),
> ('', 'pushrace', False, 'includes a check:head part with unknown nodes'),
> ('', 'genraise', False, 'includes a part that raise an exception during generation'),
> ('', 'timeout', False, 'emulate a timeout during bundle generation'),
- > ('r', 'rev', [], 'includes those changeset in the bundle'),],
+ > ('r', 'rev', [], 'includes those changeset in the bundle'),
+ > ('', 'compress', '', 'compress the stream'),],
> '[OUTPUTFILE]')
> def cmdbundle2(ui, repo, path=None, **opts):
> """write a bundle2 container on standard output"""
> bundler = bundle2.bundle20(ui)
> for p in opts['param']:
@@ -89,10 +90,13 @@ Create an extension to test bundle2 API
> try:
> bundler.addparam(*p)
> except ValueError, exc:
> raise util.Abort('%s' % exc)
>
+ > if opts['compress']:
+ > bundler.setcompression(opts['compress'])
+ >
> if opts['reply']:
> capsstring = 'ping-pong\nelephants=babar,celeste\ncity%3D%21=celeste%2Cville'
> bundler.newpart('replycaps', data=capsstring)
>
> if opts['pushrace']:
@@ -881,7 +885,79 @@ And its handling on the other size raise
$ cat ../genfailed.hg2 | hg unbundle2
0 unread bytes
abort: unexpected error: Someone set up us the bomb!
[255]
+Test compression
+================
+
+Simple case where it just work: GZ
+----------------------------------
+
+ $ hg bundle2 --compress GZ --rev '8+7+5+4' ../rev.hg2.bz
+ $ cat ../rev.hg2.bz
+ HG20\x00\x00\x00\x0eCompression=GZx\x9c\x95\x94}h\x95U\x1c\xc7\x9f;1\xe8\xce\xfa\xc3e\xbe\xa0\xa4\xb4R\xb9)\xe7\xf5y\xce\x89\xfac\xed^w\x8b\x9c\xc3?*\x1ch\xcfy\x9b\xddj\xae\xb0(t\xb8\xe5\x96[\xbb\x86a\xa3\x15n:q\xc8j\xe8\xa5\xda\x95d("\xcei\xcd\x06Y4(+Q*X\xc3\x17V*\x9a\x9dg\xdc\xc65\x9e\xc4\x1d\xf8\x9e\x87\xf3\x9c\xf3;\xbf\x0f\xbf\x97\xe38\xce\xf4B\xb9\xd6\xaf\xae\xd2U\xaf\xae{\xad\xc6\xc9\x8d\xbb\x8a\xec\xb4\x07\xed\x7f\xfd\xed\xd3S\xbeN\xf4\x0e\xafYRs\xeaP\xd7\x96\x9e\xba\xd4\x9a\x1f\x87\x9b\x9f\x1d\xe8zjy\xe9\xcb\x7f\xcf\xeb\xfe~\xd3\x82\xce/68!#\xcc6\xb7\xb58\x90\xab\xa1!\x92xZ (esc)
+ \x8a\xb11 (esc)
+ H\xa6)\x92J2\xe6\x1b\xe1J\x85\xb9F at F\xedac\xb5\xd6\xaa \x1e\xac^\xb0 (esc)
+ \xae\x8a\xc4\x03\xc6\xd6\xf9\xa3{\xeb\xfbN\xde\x7f\xe4\x97U_\x15v\x96\xd2]\xbf\x9d?8\x18)L\x0f\xb7]n\x9b\xb3\xaa~\xc6\xd5\x15[\xf7|R\xf1|s\x18c\x98m>#QZj.\x19r\x8d\xcb \x07\x14x\x823\xe9b\x86}\x0c\x00\x17\x88S\x86=u\x0bc\xe2\x16\xc6\x84\x9dv\x8fvz\xcb\xde\xfc\xa8\xa3\xf0F\xd3\xa5\xf6\xc7\x96\xb6\x9f`;W\xae(\xce\xb2\x8d\xe9\xf4>ofS\xdd\xe5k\xadg\xbe\xf9r\xee_\x8da<a\xb6\xf9\x8c\xd8\xa5\x82cE=\xa3\x0ca\x90h$(\x87P\xb9\xc2\x97\xc6 \x01\x11\x80\x84\x10\x98\xcf\xe8\xe4\x13\x96\x05Q,8\xf3\xc4\xec\xeaC\xe7\x96^j\xc8\xbe\x11\xdd2x\xa2\xfa\xdd\x8f\xb3a\x84aQ\x0c\xb3\xcd'dBk\xc2\xb4\x92\x1e\x86\x8c\x12h$\x00\x10\xdb\x7fP\x00\xc6\x91\xe7\xfaL""\xcc\xbf\x84\x81 (esc)
+ \x92\xc1\xaa*\xc7\x1bI\xe6\xeek\xa9~\xe0\xe9\xb2\x91^|sh\xe0\xfc#?4\xed\xcf\x0e\xf2\xb3\xd3L\xd7\xaeY3o\x8c=\xb8c!+\xe8=\xe0o\x9d:\xb7\xf9\xdc$*\xb2>\xa7X\xdc\x91\xd8@\xe9#\x8e\x88\x84\xae\x0f\xb9\x00.\xb5t6\xf3 at S@4\x15\xc0\xd7\x12\x8d\xe7\xbbe\xf9\xc8\xef\x03\x0f\xff\xf9\xfe\xb6\x8a\r (no-eol) (esc)
+ m\xfd\xecQp\xf7\xa7\xad\x9bk\x9d\xdat{SC\xd1Cc\xfd\x19\xf9\xcag\x95\xe5\xef\xc4\xe6l\x9eD\xe1\xc5\xacz\x82o\xc2\xe1\xd2\xb5-\x81)\xf0] lo\x10\xae\x88\xcf%\x05\xd0\x93\x06x\x80`C-\x10\x1bGq+\xb7\x7f\xbb\xe9\xa7\xe4}g{\xdf\x9b\xf7b\xcf\xcd\xd8\xf4H\xbcdQWC\xff\xea\x8b\x0b\xaetdS\x07\x86\xfaf<^\xf7\xe1\xaf\xa7\xc2\x90\xff\xa7\xbe\x9e\xc9)\xb6\xccAH\x18i\x94\x8b|\x04}\x8c\x98\xa7\x95PD\xd9\xd0 \xc8\x140\x14Q\xadl\x16\x03\x94\x0fZF\x93\x7f\x1c\x87\x8d%\xd7\x9d\xa2\xd1\x92L\xf3\xc2T\xba\xf8p\x18\xca$ (esc)
+ )\x96Cq\xf2\x93\x95t\x18\xb5e\xc4\xb8\xf6l\\4 \x1e\xd5\x0c!\xc0\xb1\x90\x9e\x12@\xb9\x18\xfaZ\x00A\xa29\xd3\xa9\xc1s!\x8e^<\xb9\xb8\xf8HjvF\xa7\x1a\xb6\xdd[Q^\x19\x1dY\x12\xc62\x89\x02\x9a\xc0\x8fO\xb8 (esc)
+ \xba^\xecX7D\xa3/\xdd3\xed\xc9\xd3\xdd\xc7"\x1b/\xd4\x94\x8e\x95?w\xa7\xaen\xf32\x8d\xbbJL\xb8 (esc)
+ ZC4:\xb3:\xd6w\xff\\\xb6\xfa\xad\xf9\xdb\xfbj3\xdf\xc1}\x99\xcf\xef\xd4\xd5m\xdaw|;\x19\xfd\xaf\xc5?\xf1`\xc3\x17 (no-eol) (esc)
+ $ hg debugbundle ../rev.hg2.bz
+ Stream params: {'Compression': 'GZ'}
+ changegroup -- '{}'
+ 32af7686d403cf45b5d95f2d70cebea587ac806a
+ 9520eea781bcca16c1e15acc0ba14335a0e8e5ba
+ eea13746799a9e0bfd88f29d3c2e9dc9389f524f
+ 02de42196ebee42ef284b6780a87cdc96e8eaab6
+ $ hg unbundle ../rev.hg2.bz
+ adding changesets
+ adding manifests
+ adding file changes
+ added 0 changesets with 0 changes to 3 files
+Simple case where it just work: BZ
+----------------------------------
+
+ $ hg bundle2 --compress BZ --rev '8+7+5+4' ../rev.hg2.bz
+ $ cat ../rev.hg2.bz
+ HG20\x00\x00\x00\x0eCompression=BZBZh91AY&SY\xa3K\x18=\x00\x00\x1a\x7f\xff\xff\xbf_\xf6\xef\xef\x7f\xf6?\xf7\xd1\xd9\xff\xff\xf7n\xff\xffn\xf7\xf6\xbd\xdf\xb5\xab\xff\xcfg\xf6\xe7{\xf7\xc0\x02\xd73\x82\x8bQ\x04\xa5S\xd5='\xa0\x99\x18M\r (no-eol) (esc)
+ 4\x00\xd1\xa1\xe8\x80\xc8z\x87\xa9\xa3Cj=F\x86&\x804=@\xc8\xc9\xb54\xf4\x8fH\x0fQ\xea44\xfdM\xaa\x19\x03@\x0c\x08\xda\x86C\xd4\xf5\x0fB\x1e\xa0\xf3T3T\xd3\x13M\x03 at 2\x00\x002\x03&\x80\r (no-eol) (esc)
+ \x00\r (no-eol) (esc)
+ \x00h\xc8\xc8\x03 20\x98\x8c\x80\x00\x00\x03M\x00\xc8\x00\x00\r (no-eol) (esc)
+ \x00\x00"\x99\xa14\xc2d\xa6\xd54\x1a\x00\x00\x06\x86\x83M\x07\xa8\xd1\xa0h\x01\xa0\x00\x00\x00\x00\r (no-eol) (esc)
+ \x06\x80\x00\x00\x00\r (no-eol) (esc)
+ \x00\x03@\x00\x00\x04\xa4\xa1M\xa9\x89\x89\xb4\x9a2\x0cCF\x86\x87\xa9\x8dA\x9a\x98F\x9a\r (no-eol) (esc)
+ 12\x1a4\r (no-eol) (esc)
+ \x0c\x8d\xa2\x0c\x98M\x06\x8c@\xc2`\x8d\r (no-eol) (esc)
+ \x0c \xc9\x89\xfa\xa0\xd0\xd3!\xa1\xea4\xd3h\x9e\xa6\xd1t\x053\xcbf\x96\x93(d@\x91"\xacU\x9b\xea@{8\x94\xe2\xf8\x06\x00\xcb(\x02\x00M\xab@$\x10C\x18\xcfd\xb4\x06\x83\x0c4l\xb4\xa3\xd4 (esc)
+
+ \xe4\xa8\\N#\xc0\xc9z1\x97\x87wzd\x88\x80\x8e`\x97 \x93\x0f\x8e\xeb\xc4b\xa4D\xa3R \xb2\x99\xa9.\xe1\xd7)JT\xacDz\xbb\xcc\x04=\xe0\xaa\xbdj3^\x9b\xa2W6\xfa\xcbE\xbbm>\xc1\xd9\xd9\xf5\x83i\x8a\xd0\xe0\xe2\xe7\xae\x90U$\xda?\xabx\xc0L\xb4V\xa3\x9e\xa4\xaf\x9cet\x86\xecm\xdcb\xdc3\xca\xc8P\xdd\x9d\x98\x8e\x9eY \xf3\xf0B\x91J \xf5u\x8d=\xa5\xa5\x15\xcb\x8d\x10c\xb0\xc2.\xb2\x81\xf7\xc1v\x0eSl\x0eFs\xb5\xaeg\xf9L\x0bEk\xa82*/\xa2T\xa4D\x05 \xa18\xd1\xa4\xc6 \xa8+\x08\x99\xa4\x14\xae\x8d\xa3\xe3\xaa4'\xd8D\xca\xc3]!\x8b\x1a\x1e\x97)q+ JJUU\x94Xe\xb2\xbc\xf3\xa5\x90&6vgzQ\x98\xd6\x8aJ\x99P\xb5\x99\x8f\x94!\x17\xa9\x8b\xf3\xadL3\xd4.@\xc8\x0c;\x90S9\xdbH\x024\x83H\xd6\xb3\x99\x13\xd2Xe\x8eq\xac\xa9\x06\x95\xf2\xc4\x8e\xb4\x08k\xd3\x0c\xae\xd9\x90VqC\xa7\xa2b\x16>Pc\xd3W<-\x9f\x0f4\x05\x08\xd8\xa6KY1Tf:E\x0c\x8a\xc7\x90:\xf0j\x83\x1b\xf5\xca\xfb\x80+P\x06\xfbQ~\xa6\xa4\xd4\x81D\x82!T\x00[\x1a0\x83b\xa3\x18\xb6$\x19\x1eE\xdfM\\\xdb\xa6\xaf[\xac\x90\xfa>\xed\xf9\xecL\xba6\xee\xd8` \xa7\xc7;\xcb\xd1\x90C}'\x16P]\xad\xf4\x14\x07\x0b\x90\\\xcck\x8d?\xa6\x88\xf447\xa8\xcf\x14c6\x19\xf7>(\xde\x99\xe8\x16\xa4\x9d\r (no-eol) (esc)
+ @\xa1\xa7$R\x14\xa6rbYZ\xca-\xe5Q\x90x\x88\xd9\xc6\xc7!\xd0\xf7F\xb2\x04FDN \x9c\x12\xb1\x03N%\xe0\xa9\x0cX[\x1d<\x93 \x01Q\xde\xa9\x1ci#2F\x14\xb4\x90\xdb\x17\x98\x98P\x03\x90)\xaa@\xb0\x13\xd8C\xd2_\xc5\x9d\xeb\xf3\xf2\xadA\xe8z\xa9\xed\xa1X\x84\xa6B\xbf\xd6\xfc$\x82\xc1 2&J\x15\xa6\x1d)\x7f~\xf4=\x07\xbcb\x9a[\xecD=r\x1dA\x8b\\\x80\xde\x0eb\x9a.\xf8\x83\x00\xd5\x07\xa0\x9c\xc6t\x98\x11\xb2^\xa98\x02\x03\xee\xfd\x86\\\xf4\x86\xb3\xae\xda\x05\x94\x01\xc5\xc6\xea\x18\xe6\xba*\xba\xb3\x04\\\x96\x89rc[\x10\x11\xf6g4\x98\xcb\xe4\xc0N\xfa\xe6\x99\x19nP\xe8&\x8d\x0c\x17\xe0\xbe\xef\xe1\x8e\x02o2\x82\xdc&\xf8\xa1\x08\xf3\x8a\r (no-eol) (esc)
+ \xf3\xc4u\x00Hs\xb8\xbe;\r (no-eol) (esc)
+ \x7f\xd0\xfd\xc7x\x96\xec\xe0\x03\x80hM\x8dC\x8c\xd7hX\xf9P\xf0\x18\xcb!X\x1b`\xcd\x1f\x846.\x16\x1f (esc)
+ \xf7N\x8f\xeb\xdf\x01-\xc2y\x0b\xf7$\xea\r (no-eol) (esc)
+ \xe8Y\x86Qn\x1c0\xa3\xad/\xee\x8c\x90\xc8\x84\xd5\xe84\xc1\x95\xb2\xc9\xf6M\x87\x1c}\x19\xd6AXVz\xe0l\xba\x10\xc7\xe8396\x96\xe7\xd2\xf9Y\x9a\x08\x95H8\xe7\x0b\xb7 (esc)
+ $g\xc49\x8bC\x88W\x9c\x01\xf5a\xb5\xe1'A~\xaf\x83\xfe.\xe4\x8ap\xa1!F\x960z (no-eol) (esc)
+ $ hg debugbundle ../rev.hg2.bz
+ Stream params: {'Compression': 'BZ'}
+ changegroup -- '{}'
+ 32af7686d403cf45b5d95f2d70cebea587ac806a
+ 9520eea781bcca16c1e15acc0ba14335a0e8e5ba
+ eea13746799a9e0bfd88f29d3c2e9dc9389f524f
+ 02de42196ebee42ef284b6780a87cdc96e8eaab6
+ $ hg unbundle ../rev.hg2.bz
+ adding changesets
+ adding manifests
+ adding file changes
+ added 0 changesets with 0 changes to 3 files
+
+unknown compression while unbundling
+-----------------------------
+
+ $ hg bundle2 --param Compression=FooBarUnknown --rev '8+7+5+4' ../rev.hg2.bz
+ $ cat ../rev.hg2.bz | hg statbundle2
+ abort: unknown parameters: Stream Parameter - Compression='FooBarUnknown'
+ [255]
$ cd ..
More information about the Mercurial-devel
mailing list