[PATCH 1 of 4 RFC] revlog: basic support for zlib streams across chunks
Wagner Bruna
wagner.bruna+mercurial at gmail.com
Mon Feb 8 17:20:48 CST 2010
# HG changeset patch
# User Wagner Bruna <wbruna at softwareexpress.com.br>
# Date 1265661125 7200
# Node ID 6b70e182505a7a863b2931ee9b7f96e1a0d089cc
# Parent f163775e36e08c89802734cfa0f0e50606203126
revlog: basic support for zlib streams across chunks
diff --git a/mercurial/revlog.py b/mercurial/revlog.py
--- a/mercurial/revlog.py
+++ b/mercurial/revlog.py
@@ -68,7 +68,7 @@
s.update(text)
return s.digest()
-def compress(text):
+def compress(text, compressobj=None):
""" generate a possibly-compressed representation of text """
if not text:
return ("", text)
@@ -76,6 +76,19 @@
bin = None
if l < 44:
pass
+ elif compressobj:
+ z = compressobj
+ # zlib makes an internal copy, thus doubling memory usage for
+ # large files, so lets do this in pieces
+ p = []
+ pos = 0
+ while pos < l:
+ pos2 = pos + 2**20
+ p.append(z.compress(text[pos:pos2]))
+ pos = pos2
+ p.append(z.flush(zlib.Z_SYNC_FLUSH))
+ bin = "".join(p)
+ return ('z', bin)
elif l > 1000000:
# zlib makes an internal copy, thus doubling memory usage for
# large files, so lets do this in pieces
@@ -97,7 +110,7 @@
return ('u', text)
return ("", bin)
-def decompress(bin):
+def decompress(bin, decompressobj=None):
""" decompress the given input """
if not bin:
return bin
@@ -106,6 +119,8 @@
return bin
if t == 'x':
return _decompress(bin)
+ if t == 'z':
+ return decompressobj.decompress(bin[1:])
if t == 'u':
return bin[1:]
raise RevlogError(_("unknown compression type %r") % t)
More information about the Mercurial-devel
mailing list