[PATCH 2 of 3] move store walking from streamclone to filelog
Adrian Buehlmann
adrian at cadifra.com
Thu Jul 17 09:23:07 CDT 2008
# HG changeset patch
# User Adrian Buehlmann <adrian at cadifra.com>
# Date 1216299442 -7200
# Node ID d29d8f832d09a8570a25a68c964baf8572454b75
# Parent 64d51f7facf9804d3b004f4eae7844ae7a1937ca
move store walking from streamclone to filelog
* add method localrepo.localrepository.storefiles
* add function filelog.store
* add class filelog._store
streamclone now uses repo.storefiles to get the filenames
of the store
diff --git a/mercurial/filelog.py b/mercurial/filelog.py
--- a/mercurial/filelog.py
+++ b/mercurial/filelog.py
@@ -7,6 +7,7 @@
from node import bin, nullid
from revlog import revlog
+import os, stat, osutil, util
class filelog(revlog):
def __init__(self, opener, path):
@@ -114,3 +115,59 @@
def o(path, *args, **kw):
return openerfn(fn(path), *args, **kw)
return o
+
+def _dirwalk(path, recurse, strip_count, decodefn):
+ '''yields (*.d or *.i filename, size)'''
+ for e, kind, st in osutil.listdir(path, stat=True):
+ pe = os.path.join(path, e)
+ if kind == stat.S_IFDIR:
+ if recurse:
+ for x in _dirwalk(pe, True, strip_count, decodefn):
+ yield x
+ else:
+ if kind != stat.S_IFREG or len(e) < 2:
+ continue
+ sfx = e[-2:]
+ if sfx in ('.d', '.i'):
+ f = decodefn(util.pconvert(pe[strip_count:]))
+ yield (f, st.st_size)
+
+class _store:
+ def __init__(self, encodefn, decodefn, spath, createmode):
+ self.spath = spath
+ self.encodefn = encodefn
+ self.__decodefn = decodefn
+ so = util.opener(self.spath)
+ so.createmode = createmode
+ self.opener = encodedopener(so, self.encodefn)
+
+ def _walkfiledata(self):
+ strip_count = len(self.spath) + len(os.sep)
+ dpath = os.path.join(self.spath, 'data')
+ for x in _dirwalk(dpath, True, strip_count, self.__decodefn):
+ yield x
+
+ def walk(self):
+ '''yields: direncoded filename, size of file'''
+ # yield file data first
+ for x in self._walkfiledata():
+ yield x
+ # yield manifest before changelog
+ strip_count = len(self.spath) + len(os.sep)
+ meta = util.sort(_dirwalk(self.spath, False, strip_count, lambda x: x))
+ meta.reverse()
+ for x in meta:
+ yield x
+
+def encodefn(requirements):
+ if 'store' not in requirements:
+ return lambda x: x
+ else:
+ return encodefilename
+
+def store(requirements, spath, createmode):
+ efn = encodefn(requirements)
+ if 'store' not in requirements:
+ return _store(efn, lambda x: x, spath, createmode)
+ else:
+ return _store(efn, decodefilename, spath, createmode)
diff --git a/mercurial/localrepo.py b/mercurial/localrepo.py
--- a/mercurial/localrepo.py
+++ b/mercurial/localrepo.py
@@ -60,14 +60,9 @@
if r not in self.supported:
raise repo.RepoError(_("requirement '%s' not supported") % r)
- # setup store
if "store" in requirements:
- self.encodefn = filelog.encodefilename
- self.decodefn = filelog.decodefilename
self.spath = os.path.join(self.path, "store")
else:
- self.encodefn = lambda x: x
- self.decodefn = lambda x: x
self.spath = self.path
try:
@@ -81,9 +76,10 @@
self._createmode = mode
self.opener.createmode = mode
- sopener = util.opener(self.spath)
- sopener.createmode = mode
- self.sopener = filelog.encodedopener(sopener, self.encodefn)
+
+ self.store = filelog.store(requirements, self.spath, mode)
+ self.sopener = self.store.opener
+ self.encodefn = self.store.encodefn
self.ui = ui.ui(parentui=parentui)
try:
@@ -2087,6 +2083,29 @@
return self.stream_in(remote)
return self.pull(remote, heads)
+ def storefiles(self):
+ '''get all *.i and *.d files in the store
+
+ Returns (ok, list of (filename, size), total_bytes)'''
+
+ # get consistent snapshot of repo, lock during scan
+ repolock = None
+ try:
+ entries = []
+ total_bytes = 0
+ try:
+ repolock = self.lock()
+ except (lock.LockHeld, lock.LockUnavailable), inst:
+ self.ui.warn('locking the repository failed: %s\n' % (inst,))
+ return False, entries, total_bytes
+ self.ui.debug('scanning\n')
+ for name, size in self.store.walk():
+ entries.append((name, size))
+ total_bytes += size
+ return True, entries, total_bytes
+ finally:
+ del repolock
+
# used to avoid circular references so destructors work
def aftertrans(files):
renamefiles = [tuple(t) for t in files]
diff --git a/mercurial/statichttprepo.py b/mercurial/statichttprepo.py
--- a/mercurial/statichttprepo.py
+++ b/mercurial/statichttprepo.py
@@ -55,13 +55,10 @@
# setup store
if "store" in requirements:
- self.encodefn = filelog.encodefilename
- self.decodefn = filelog.decodefilename
self.spath = self.path + "/store"
else:
- self.encodefn = lambda x: x
- self.decodefn = lambda x: x
self.spath = self.path
+ self.encodefn = filelog.encodefn(requirements)
self.sopener = filelog.encodedopener(opener(self.spath), self.encodefn)
self.manifest = manifest.manifest(self.sopener)
@@ -77,6 +74,9 @@
def local(self):
return False
+ def storefiles(self, getsizes):
+ return (False, [], 0)
+
def instance(ui, path, create):
if create:
raise util.Abort(_('cannot create new static-http repository'))
diff --git a/mercurial/streamclone.py b/mercurial/streamclone.py
--- a/mercurial/streamclone.py
+++ b/mercurial/streamclone.py
@@ -5,39 +5,11 @@
# This software may be used and distributed according to the terms
# of the GNU General Public License, incorporated herein by reference.
-import os, osutil, stat, util, lock
+import util
# if server supports streaming clone, it advertises "stream"
# capability with value that is version+flags of repo it is serving.
# client only streams if it can read that repo format.
-
-def walkrepo(root):
- '''iterate over metadata files in repository.
- walk in natural (sorted) order.
- yields 2-tuples: name of .d or .i file, size of file.'''
-
- strip_count = len(root) + len(os.sep)
- def walk(path, recurse):
- for e, kind, st in osutil.listdir(path, stat=True):
- pe = os.path.join(path, e)
- if kind == stat.S_IFDIR:
- if recurse:
- for x in walk(pe, True):
- yield x
- else:
- if kind != stat.S_IFREG or len(e) < 2:
- continue
- sfx = e[-2:]
- if sfx in ('.d', '.i'):
- yield pe[strip_count:], st.st_size
- # write file data first
- for x in walk(os.path.join(root, 'data'), True):
- yield x
- # write manifest before changelog
- meta = util.sort(walk(root, False))
- meta.reverse()
- for x in meta:
- yield x
# stream file format is simple.
#
@@ -59,27 +31,11 @@
fileobj.write('1\n')
return
- # get consistent snapshot of repo. lock during scan so lock not
- # needed while we stream, and commits can happen.
- repolock = None
- try:
- try:
- repolock = repo.lock()
- except (lock.LockHeld, lock.LockUnavailable), inst:
- repo.ui.warn('locking the repository failed: %s\n' % (inst,))
- fileobj.write('2\n')
- return
-
- fileobj.write('0\n')
- repo.ui.debug('scanning\n')
- entries = []
- total_bytes = 0
- for name, size in walkrepo(repo.spath):
- name = repo.decodefn(util.pconvert(name))
- entries.append((name, size))
- total_bytes += size
- finally:
- del repolock
+ scanok, entries, total_bytes = repo.storefiles()
+ if not scanok:
+ fileobj.write('2\n')
+ return
+ fileobj.write('0\n')
repo.ui.debug('%d files, %d bytes to transfer\n' %
(len(entries), total_bytes))
More information about the Mercurial-devel
mailing list