[PATCH 1 of 2] Taking my changes to 0.6c to improve performance,
and merging them in with
Eric Hopper
hopper at omnifarious.org
Sun Sep 4 10:16:26 CDT 2005
the tip of the main repository.
2 files changed, 71 insertions(+), 16 deletions(-)
mercurial/localrepo.py | 16 ----------
mercurial/util.py | 71 +++++++++++++++++++++++++++++++++++++++++++++++-
# HG changeset patch
# User Eric Hopper <hopper at omnifarious.org>
# Node ID 765d6f9a31b9b0ec6ab65920f52221a5db647729
# Parent a33a7a543803c7383a6918b19797bf9fd6b83e8e
# Parent 4940e615ac82d55fc9605cbe913cb85307814e3f
Taking my changes to 0.6c to improve performance, and merging them in with
the tip of the main repository.
diff -r a33a7a543803 -r 765d6f9a31b9 mercurial/localrepo.py
--- a/mercurial/localrepo.py Sun Aug 28 06:45:27 2005
+++ b/mercurial/localrepo.py Sun Sep 4 06:33:56 2005
@@ -886,21 +886,7 @@
return remote.addchangegroup(cg)
def changegroup(self, basenodes):
- class genread:
- def __init__(self, generator):
- self.g = generator
- self.buf = ""
- def fillbuf(self):
- self.buf += "".join(self.g)
-
- def read(self, l):
- while l > len(self.buf):
- try:
- self.buf += self.g.next()
- except StopIteration:
- break
- d, self.buf = self.buf[:l], self.buf[l:]
- return d
+ genread = util.chunkbuffer
def gengroup():
nodes = self.newer(basenodes)
diff -r a33a7a543803 -r 765d6f9a31b9 mercurial/util.py
--- a/mercurial/util.py Sun Aug 28 06:45:27 2005
+++ b/mercurial/util.py Sun Sep 4 06:33:56 2005
@@ -12,7 +12,7 @@
import os, errno
from demandload import *
-demandload(globals(), "re")
+demandload(globals(), "re cStringIO")
def binary(s):
"""return true if a string is binary data using diff's heuristic"""
@@ -354,3 +354,72 @@
val = os.WSTOPSIG(code)
return "stopped by signal %d" % val, val
raise ValueError("invalid exit code")
+
+class chunkbuffer(object):
+ """Allow arbitrary sized chunks of data to be efficiently read from an
+iterator over chunks of arbitrary size."""
+ def __init__(self, in_iter, targetsize = 2**16):
+ """in_iter is the iterator that's iterating over the input chunks.
+targetsize is how big a buffer to try to maintain."""
+ self.in_iter = iter(in_iter)
+ self.buf = ''
+ targetsize = int(targetsize)
+ if (targetsize <= 0):
+ raise ValueError("targetsize must be greater than 0, was %d" % targetsize)
+ self.targetsize = int(targetsize)
+ self.iterempty = False
+ def fillbuf(self):
+ """x.fillbuf()
+
+Ignore the target size, and just read every chunk from the iterator
+until it's empty."""
+ if not self.iterempty:
+ collector = cStringIO.StringIO()
+ collector.write(self.buf)
+ reduce(lambda wrt, ch: (wrt(ch) and None) or wrt,
+ self.in_iter, collector.write)
+ self.buf = collector.getvalue()
+ collector.close()
+ collector = None
+ self.iterempty = True
+
+ def read(self, l):
+ """x.read(l) -> str
+
+Read l bytes of data from the iterator of chunks of data. Returns less
+than l bytes if the iterator runs dry."""
+ if (l > len(self.buf)) and not self.iterempty:
+ # Clamp to a multiple of self.targetsize
+ targetsize = self.targetsize * ((l // self.targetsize) + 1)
+ collector = cStringIO.StringIO()
+ collector.write(self.buf)
+ collected = len(self.buf)
+ for chunk in self.in_iter:
+ collector.write(chunk)
+ collected += len(chunk)
+ if collected >= targetsize:
+ break
+ if collected < targetsize:
+ self.iterempty = True
+ self.buf = collector.getvalue()
+ collector.close()
+ collector = None
+ s = self.buf[:l]
+ self.buf = buffer(self.buf, l)
+ return s
+ def __repr__(self):
+ return "<%s.%s targetsize = %u buffered = %u bytes>" % \
+ (self.__class__.__module__, self.__class__.__name__,
+ self.targetsize, len(self.buf))
+
+def filechunkiter(f, size = 65536):
+ """filechunkiter(file[, size]) -> generator
+
+Create a generator that produces all the data in the file size (default
+65536) bytes at a time. Chunks may be less than size bytes if the
+chunk is the last chunk in the file, or the file is a socket or some
+other type of file that sometimes reads less data than is requested."""
+ s = f.read(size)
+ while len(s) >= 0:
+ yield s
+ s = f.read(size)
More information about the Mercurial
mailing list