[PATCH 2 of 3] move store walking from streamclone.py into store.py

Adrian Buehlmann adrian at cadifra.com
Mon Jul 21 14:43:58 CDT 2008


# HG changeset patch
# User Adrian Buehlmann <adrian at cadifra.com>
# Date 1216645162 -7200
# Node ID 00e5d2425f873233e06ed3c6631211ce49935c84
# Parent  a7f1babd458d9cb5b6db1b6d326624ab355b4268
move store walking from streamclone.py into store.py

diff --git a/mercurial/localrepo.py b/mercurial/localrepo.py
--- a/mercurial/localrepo.py
+++ b/mercurial/localrepo.py
@@ -60,14 +60,9 @@
             if r not in self.supported:
                 raise repo.RepoError(_("requirement '%s' not supported") % r)
 
-        # setup store
         if "store" in requirements:
-            self.encodefn = store.encodefilename
-            self.decodefn = store.decodefilename
             self.spath = os.path.join(self.path, "store")
         else:
-            self.encodefn = lambda x: x
-            self.decodefn = lambda x: x
             self.spath = self.path
 
         try:
@@ -81,9 +76,10 @@
 
         self._createmode = mode
         self.opener.createmode = mode
-        sopener = util.opener(self.spath)
-        sopener.createmode = mode
-        self.sopener = store.encodedopener(sopener, self.encodefn)
+
+        self.store = store.store(requirements, self.spath, mode)
+        self.sopener = self.store.opener
+        self.encodefn = self.store.encodefn
 
         self.ui = ui.ui(parentui=parentui)
         try:
@@ -2087,6 +2083,29 @@
             return self.stream_in(remote)
         return self.pull(remote, heads)
 
+    def storefiles(self):
+        '''get all *.i and *.d files in the store
+
+        Returns (ok, list of (filename, size), total_bytes)'''
+
+        lock_ = None
+        try:
+            # get consistent snapshot of repo, lock during scan
+            entries = []
+            total_bytes = 0
+            try:
+                lock_ = self.lock()
+            except (lock.LockHeld, lock.LockUnavailable), inst:
+                self.ui.warn('locking the repository failed: %s\n' % (inst,))
+                return False, entries, total_bytes
+            self.ui.debug('scanning\n')
+            for name, size in self.store.walk():
+                entries.append((name, size))
+                total_bytes += size
+            return True, entries, total_bytes
+        finally:
+            del lock_
+
 # used to avoid circular references so destructors work
 def aftertrans(files):
     renamefiles = [tuple(t) for t in files]
diff --git a/mercurial/statichttprepo.py b/mercurial/statichttprepo.py
--- a/mercurial/statichttprepo.py
+++ b/mercurial/statichttprepo.py
@@ -55,14 +55,13 @@
 
         # setup store
         if "store" in requirements:
-            self.encodefn = store.encodefilename
-            self.decodefn = store.decodefilename
             self.spath = self.path + "/store"
         else:
-            self.encodefn = lambda x: x
-            self.decodefn = lambda x: x
             self.spath = self.path
-        self.sopener = store.encodedopener(opener(self.spath), self.encodefn)
+        self.encodefn = store.encodefn(requirements)
+        so = opener(self.spath)
+        self.sopener = lambda path, *args, **kw: so(
+            self.encodefn(path), *args, **kw)
 
         self.manifest = manifest.manifest(self.sopener)
         self.changelog = changelog.changelog(self.sopener)
diff --git a/mercurial/store.py b/mercurial/store.py
--- a/mercurial/store.py
+++ b/mercurial/store.py
@@ -4,6 +4,8 @@
 #
 # This software may be used and distributed according to the terms
 # of the GNU General Public License, incorporated herein by reference.
+
+import os, stat, osutil, util
 
 def _buildencodefun():
     e = '_'
@@ -33,7 +35,79 @@
 
 encodefilename, decodefilename = _buildencodefun()
 
-def encodedopener(openerfn, fn):
-    def o(path, *args, **kw):
-        return openerfn(fn(path), *args, **kw)
-    return o
+def _dirwalk(path, recurse):
+    '''yields (filename, size)'''
+    for e, kind, st in osutil.listdir(path, stat=True):
+        pe = os.path.join(path, e)
+        if kind == stat.S_IFDIR:
+            if recurse:
+                for x in _dirwalk(pe, True):
+                    yield x
+        elif kind == stat.S_IFREG:
+            yield pe, st.st_size
+
+class _store:
+    '''abstract base class for repository store'''
+    def __init__(self, spath):
+        self.spath = spath
+
+    def _revlogfiles(self, relpath='', recurse=False):
+        '''yields (filename, size)'''
+        if relpath:
+            path = os.path.join(self.spath, relpath)
+        else:
+            path = self.spath
+        striplen = len(self.spath) + len(os.sep)
+        filetypes = ('.d', '.i')
+        for f, size in _dirwalk(path, recurse):
+            if (len(f) > 2) and f[-2:] in filetypes:
+                yield util.pconvert(f[striplen:]), size
+
+    def _datafiles(self):
+        pass
+
+    def walk(self):
+        '''yields (direncoded filename, size)'''
+        # yield data files first
+        for x in self._datafiles():
+            yield x
+        # yield manifest before changelog
+        meta = util.sort(self._revlogfiles())
+        meta.reverse()
+        for x in meta:
+            yield x
+
+class directstore(_store):
+    def __init__(self, spath, createmode):
+        _store.__init__(self, spath)
+        self.encodefn = lambda x: x
+        self.opener = util.opener(self.spath)
+        self.opener.createmode = createmode
+
+    def _datafiles(self):
+        for x in self._revlogfiles('data', True):
+            yield x
+
+class encodedstore(_store):
+    def __init__(self, spath, createmode):
+        _store.__init__(self, spath)
+        self.encodefn = encodefilename
+        op = util.opener(self.spath)
+        op.createmode = createmode
+        self.opener = lambda f, *args, **kw: op(self.encodefn(f), *args, **kw)
+
+    def _datafiles(self):
+        for f, size in self._revlogfiles('data', True):
+            yield decodefilename(f), size
+
+def encodefn(requirements):
+    if 'store' not in requirements:
+        return lambda x: x
+    else:
+        return encodefilename
+
+def store(requirements, spath, createmode):
+    if 'store' not in requirements:
+        return directstore(spath, createmode)
+    else:
+        return encodedstore(spath, createmode)
diff --git a/mercurial/streamclone.py b/mercurial/streamclone.py
--- a/mercurial/streamclone.py
+++ b/mercurial/streamclone.py
@@ -5,39 +5,11 @@
 # This software may be used and distributed according to the terms
 # of the GNU General Public License, incorporated herein by reference.
 
-import os, osutil, stat, util, lock
+import util
 
 # if server supports streaming clone, it advertises "stream"
 # capability with value that is version+flags of repo it is serving.
 # client only streams if it can read that repo format.
-
-def walkrepo(root):
-    '''iterate over metadata files in repository.
-    walk in natural (sorted) order.
-    yields 2-tuples: name of .d or .i file, size of file.'''
-
-    strip_count = len(root) + len(os.sep)
-    def walk(path, recurse):
-        for e, kind, st in osutil.listdir(path, stat=True):
-            pe = os.path.join(path, e)
-            if kind == stat.S_IFDIR:
-                if recurse:
-                    for x in walk(pe, True):
-                        yield x
-            else:
-                if kind != stat.S_IFREG or len(e) < 2:
-                    continue
-                sfx = e[-2:]
-                if sfx in ('.d', '.i'):
-                    yield pe[strip_count:], st.st_size
-    # write file data first
-    for x in walk(os.path.join(root, 'data'), True):
-        yield x
-    # write manifest before changelog
-    meta = util.sort(walk(root, False))
-    meta.reverse()
-    for x in meta:
-        yield x
 
 # stream file format is simple.
 #
@@ -59,27 +31,11 @@
         fileobj.write('1\n')
         return
 
-    # get consistent snapshot of repo. lock during scan so lock not
-    # needed while we stream, and commits can happen.
-    repolock = None
-    try:
-        try:
-            repolock = repo.lock()
-        except (lock.LockHeld, lock.LockUnavailable), inst:
-            repo.ui.warn('locking the repository failed: %s\n' % (inst,))
-            fileobj.write('2\n')
-            return
-
-        fileobj.write('0\n')
-        repo.ui.debug('scanning\n')
-        entries = []
-        total_bytes = 0
-        for name, size in walkrepo(repo.spath):
-            name = repo.decodefn(util.pconvert(name))
-            entries.append((name, size))
-            total_bytes += size
-    finally:
-        del repolock
+    scanok, entries, total_bytes = repo.storefiles()
+    if not scanok:
+        fileobj.write('2\n')
+        return
+    fileobj.write('0\n')
 
     repo.ui.debug('%d files, %d bytes to transfer\n' %
                   (len(entries), total_bytes))


More information about the Mercurial-devel mailing list