[PATCH RFC] forest extension
Robin Farine
robin.farine at terminus.org
Sat Dec 9 07:18:53 CST 2006
I would like the forest extension to be reviewed and, if you think
it is worth it, ultimately included in Mercurial mainline.
The first part of this submission describes the extension, it is a copy
of the extension doc string. The second part is a git diff against
today's mercurial crew.
The full history of this extension is currently available at
<URL:http://www.terminus.org/hg/hgforest>.
Thanks,
Robin
-----------------------------------------------------------------------
Operations on trees with nested Mercurial repositories.
This extension provides commands that apply to a composite tree called
a forest. Some commands simply wrap standard Mercurial commands, such
as 'clone' or 'status', and others involve a snapshot file.
A snapshot file represents the state of a forest at a given time. It
has the format of a ConfigParser file and lists the trees in a forest,
each tree with the following attributes:
root path relative to the top-level tree
revision the revision the working directory is based on
paths a list of (alias, location) pairs
The 'fsnap' command generates or updates such a file based on a forest
in the file system. Other commands use this information to populate a
forest or to pull/push changes.
Configuration
This extension recognizes the following item in the forest
configuration section:
walkhg = (0|1)
Whether repositories under a .hg directory should be skipped (0) or
not (1). The default value is 0. Some commands accept the --walkhg
command-line option to override the behavior selected by this item.
-----------------------------------------------------------------------
diff --git a/hgext/forest.py b/hgext/forest.py
new file mode 100644
--- /dev/null
+++ b/hgext/forest.py
@@ -0,0 +1,403 @@
+# Forest, an extension to work on a set of nested Mercurial trees.
+#
+# Copyright (C) 2006 by Robin Farine <robin.farine at terminus.org>
+#
+# This software may be used and distributed according to the terms
+# of the GNU General Public License, incorporated herein by reference.
+
+"""Operations on trees with nested Mercurial repositories.
+
+This extension provides commands that apply to a composite tree called
+a forest. Some commands simply wrap standard Mercurial commands, such
+as 'clone' or 'status', and others involve a snapshot file.
+
+A snapshot file represents the state of a forest at a given time. It
+has the format of a ConfigParser file and lists the trees in a forest,
+each tree with the following attributes:
+
+ root path relative to the top-level tree
+ revision the revision the working directory is based on
+ paths a list of (alias, location) pairs
+
+The 'fsnap' command generates or updates such a file based on a forest
+in the file system. Other commands use this information to populate a
+forest or to pull/push changes.
+
+
+Configuration
+
+This extension recognizes the following item in the forest
+configuration section:
+
+walkhg = (0|1)
+
+ Whether repositories under a .hg directory should be skipped (0) or
+ not (1). The default value is 0. Some commands accept the --walkhg
+ command-line option to override the behavior selected by this item.
+"""
+
+import ConfigParser
+import os
+
+from mercurial import commands, hg, node, util
+from mercurial.i18n import gettext as _
+from mercurial.repo import RepoError
+
+cmdtable = None
+
+commands.norepo += " fclone fseed"
+
+
+def cmd_options(ui, cmd, remove=None):
+ aliases, spec = commands.findcmd(ui, cmd)
+ res = list(spec[1])
+ if remove is not None:
+ res = [opt for opt in res if opt[0] not in remove]
+ return res
+
+
+def enumerate_repos(ui, top='', **opts):
+ """Generate a lexicographically sorted list of repository roots."""
+
+ walkhg = opts['walkhg']
+ if walkhg == 2:
+ walkhg = ui.configbool('forest', 'walkhg', 0)
+ dirs = ['.']
+ while dirs:
+ root = dirs.pop()
+ entries = os.listdir(os.path.join(top, root))
+ entries.sort()
+ entries.reverse()
+ for e in entries:
+ if e == 'data' and os.path.split(root)[1] == '.hg':
+ continue
+ path = os.path.join(root, e)
+ if not os.path.isdir(os.path.join(top, path)):
+ continue
+ if e == '.hg':
+ yield util.normpath(root)
+ if not walkhg:
+ continue
+ dirs.append(path)
+
+
+def mq_patches_applied(rootpath):
+ rootpath = os.path.join(rootpath, ".hg")
+ entries = os.listdir(rootpath)
+ for e in entries:
+ path = os.path.join(rootpath, e)
+ if e == "data" or not os.path.isdir(path):
+ continue
+ series = os.path.join(path, "series")
+ status = os.path.join(path, "status")
+ if os.path.isfile(series):
+ s = os.stat(status)
+ if s.st_size > 0:
+ return True
+ return False
+
+def repository(ui, root):
+ while os.path.islink(root):
+ path = os.readlink(root)
+ if not os.path.isabs(path):
+ path = os.path.join(os.path.dirname(root), path)
+ root = path
+ return hg.repository(ui, root)
+
+
+class ForestSnapshot(object):
+
+ class Tree(object):
+
+ __slots__ = ('root', 'rev', 'paths')
+
+ def __init__(self, root, rev, paths={}):
+ self.root = root
+ self.rev = rev
+ self.paths = paths
+
+ def info(self, pathalias):
+ return self.root, self.rev, self.paths.get(pathalias, None)
+
+ def update(self, rev, paths):
+ self.rev = rev
+ for name, path in paths.items():
+ if self.paths.has_key(name):
+ self.paths[name] = path
+
+ def write(self, ui, section):
+ ui.write("root = %s\n" % self.root)
+ ui.write("revision = %s\n" % self.rev)
+ ui.write("\n[%s]\n" % (section + ".paths"))
+ for name, path in self.paths.items():
+ ui.write("%s = %s\n" % (name, path))
+
+
+ __slots__ = ('rootmap', 'trees')
+
+ def __init__(self, snapfile=None):
+ self.rootmap = {}
+ self.trees = []
+ if snapfile is not None:
+ cfg = ConfigParser.RawConfigParser()
+ cfg.read([snapfile])
+ index = 0
+ while True:
+ index += 1
+ section = "tree" + str(index)
+ if not cfg.has_section(section):
+ break
+ root = cfg.get(section, 'root')
+ tree = ForestSnapshot.Tree(root, cfg.get(section, 'revision'),
+ dict(cfg.items(section + '.paths')))
+ self.rootmap[root] = tree
+ self.trees.append(tree)
+
+ def __call__(self, ui, toprepo, func, pathalias):
+ """Apply a function to trees matching a snapshot entry.
+
+ Call func(repo, rev, path) for each repo in toprepo and its
+ nested repositories where repo matches a snapshot entry.
+ """
+
+ repo = None
+ for t in self.trees:
+ root, rev, path = t.info(pathalias)
+ ui.write("[%s]\n" % root)
+ if path is None:
+ ui.write(_("skipped, no path alias '%s' defined\n\n")
+ % pathalias)
+ continue
+ if repo is None:
+ repo = toprepo
+ else:
+ try:
+ repo = repository(ui, root)
+ except RepoError:
+ ui.write(_("skipped, no valid repo found\n\n"))
+ continue
+ if mq_patches_applied(repo.root):
+ ui.write(_("skipped, mq patches applied\n\n"))
+ continue
+ func(repo, path, rev)
+ ui.write("\n")
+
+
+ def update(self, ui, repo, **opts):
+ """Update a snapshot by scanning a forest.
+
+ If the ForestSnapshot instance to update was initialized from
+ a snapshot file, this regenerates the list of trees with their
+ current revisions but does not add any path alias to updated
+ tree entries. Newly created tree entries get all the path aliases
+ from the corresponding repository.
+ """
+
+ rootmap = {}
+ self.trees = []
+ for root in enumerate_repos(ui, **opts):
+ if mq_patches_applied(root):
+ raise util.Abort(_("'%s' has mq patches applied") % root)
+ if root != '.':
+ repo = repository(ui, root)
+ if opts['tip']:
+ rev = 'tip'
+ else:
+ rev = node.hex(repo.dirstate.parents()[0])
+ paths = dict(repo.ui.configitems('paths'))
+ if self.rootmap.has_key(root):
+ tree = self.rootmap[root]
+ tree.update(rev, paths)
+ else:
+ tree = ForestSnapshot.Tree(root, rev, paths)
+ rootmap[root] = tree
+ self.trees.append(tree)
+ self.rootmap = rootmap
+
+ def write(self, ui):
+ index = 1
+ for t in self.trees:
+ section = 'tree' + str(index)
+ ui.write("[%s]\n" % section)
+ t.write(ui, section)
+ ui.write("\n")
+ index += 1
+
+
+def clone(ui, source, dest, **opts):
+ """Clone a local forest."""
+ source = os.path.normpath(source)
+ dest = os.path.normpath(dest)
+ opts['rev'] = []
+ roots = []
+ for root in enumerate_repos(ui, source, **opts):
+ if root == '.':
+ srcpath = source
+ destpath = dest
+ else:
+ subdir = util.localpath(root)
+ srcpath = os.path.join(source, subdir)
+ destpath = os.path.join(dest, subdir)
+ if mq_patches_applied(srcpath):
+ raise util.Abort(_("'%s' has mq patches applied") % root)
+ roots.append((root, srcpath, destpath))
+ for root in roots:
+ destpfx = os.path.dirname(root[2])
+ if destpfx and not os.path.exists(destpfx):
+ os.makedirs(destpfx)
+ ui.write("[%s]\n" % root[0])
+ commands.clone(ui, root[1], root[2], **opts)
+ ui.write("\n")
+
+
+def pull(ui, toprepo, snapfile, pathalias, **opts):
+ """Pull changes from remote repositories to a local forest.
+
+ Iterate over the entries in the snapshot file and, for each entry
+ matching an actual tree in the forest and with a location
+ associated with 'pathalias', pull changes from this location to
+ the tree.
+
+ Skip entries that do not match or trees for which there is no entry.
+ """
+
+ opts['force'] = None
+ opts['rev'] = []
+
+ def doit(repo, path, *unused):
+ commands.pull(repo.ui, repo, path, **opts)
+
+ snapshot = ForestSnapshot(snapfile)
+ snapshot(ui, toprepo, doit, pathalias)
+
+
+def push(ui, toprepo, snapfile, pathalias, **opts):
+ """Push changes in a local forest to remote destinations.
+
+ Iterate over the entries in the snapshot file and, for each entry
+ matching an actual tree in the forest and with a location
+ associated with 'pathalias', push changes from this tree to the
+ location.
+
+ Skip entries that do not match or trees for which there is no entry.
+ """
+
+ opts['force'] = None
+ opts['rev'] = []
+
+ def doit(repo, path, *unused):
+ commands.push(repo.ui, repo, path, **opts)
+
+ snapshot = ForestSnapshot(snapfile)
+ snapshot(ui, toprepo, doit, pathalias)
+
+
+def seed(ui, snapshot, pathalias='default', **opts):
+ """Populate a forest according to a snapshot file."""
+
+ cfg = ConfigParser.RawConfigParser()
+ cfg.read(snapshot)
+ pfx = opts['root']
+ if pfx:
+ index = 0
+ else:
+ index = 1
+ while True:
+ index += 1
+ section = 'tree' + str(index)
+ if not cfg.has_section(section):
+ break
+ root = cfg.get(section, 'root')
+ ui.write("[%s]\n" % root)
+ dest = os.path.normpath(os.path.join(pfx, util.localpath(root)))
+ psect = section + '.paths'
+ if not cfg.has_option(psect, pathalias):
+ ui.write(_("skipped, no path alias '%s' defined\n\n") % pathalias)
+ continue
+ source = cfg.get(psect, pathalias)
+ if os.path.exists(dest):
+ ui.write(_("skipped, destination '%s' already exists\n\n") % dest)
+ continue
+ destpfx = os.path.dirname(dest)
+ if destpfx and not os.path.exists(destpfx):
+ os.makedirs(destpfx)
+ # 'clone -r rev' not implemented for all remote repos, clone
+ # everything and then use 'update' if necessary
+ opts['rev'] = []
+ commands.clone(ui, source, dest, **opts)
+ if not opts['tip']:
+ rev = cfg.get(section, 'revision')
+ if rev and rev != 'tip' and rev != node.nullid:
+ repo = repository(ui, dest)
+ commands.update(repo.ui, repo, node=rev)
+ ui.write("\n")
+
+
+
+def snapshot(ui, repo, snapfile=None, **opts):
+ """Generate a new or updated forest snapshot and display it."""
+
+ snapshot = ForestSnapshot(snapfile)
+ snapshot.update(ui, repo, **opts)
+ snapshot.write(ui)
+
+
+def status(ui, repo, *pats, **opts):
+ """Display the status of a forest of working directories."""
+
+ for root in enumerate_repos(ui, **opts):
+ mqflag = ""
+ if mq_patches_applied(root):
+ mqflag = " *mq*"
+ ui.write("[%s]%s\n" % (root, mqflag))
+ repo = repository(ui, root)
+ commands.status(repo.ui, repo, *pats, **opts)
+ ui.write("\n")
+
+
+def trees(ui, *unused, **opts):
+ """List the roots of the repositories."""
+
+ for root in enumerate_repos(ui, '', **opts):
+ ui.write(root + '\n')
+
+
+def uisetup(ui):
+ global cmdtable
+ walkhgopt = ('', 'walkhg', 2,
+ _("whether to walk (1) repositories under '.hg' or not (0)"))
+ cmdtable = {
+ "fclone" :
+ (clone,
+ [walkhgopt] + cmd_options(ui, 'clone', remove=('r',)),
+ _('hg fclone [OPTIONS] SOURCE DESTINATION')),
+ "fpull" :
+ (pull,
+ cmd_options(ui, 'pull', remove=('f', 'r')),
+ _('hg fpull [OPTIONS] SNAPSHOT-FILE PATH-ALIAS')),
+ "fpush" :
+ (push,
+ cmd_options(ui, 'push', remove=('f', 'r')),
+ _('hg fpush [OPTIONS] SNAPSHOT-FILE PATH-ALIAS')),
+ "fseed" :
+ (seed,
+ [('t', 'tip', None,
+ _("use tip instead of revisions stored in the snapshot file")),
+ ('', 'root', '',
+ _("create root as well as children under <root>")),
+ walkhgopt] + cmd_options(ui, 'clone', remove=('r',)),
+ _('hg fseed [OPTIONS] SNAPSHOT-FILE [PATH-ALIAS]')),
+ "fsnap" :
+ (snapshot,
+ [('t', 'tip', None,
+ _("record tip instead of actual child revisions")),
+ walkhgopt],
+ 'hg fsnap [OPTIONS] [SNAPSHOT-FILE]'),
+ "fstatus" :
+ (status,
+ [walkhgopt] + cmd_options(ui, 'status'),
+ _('hg fstatus [OPTIONS]')),
+ "ftrees" :
+ (trees, [walkhgopt],
+ 'hg ftrees [OPTIONS]'),
+ }
diff --git a/tests/test-forest b/tests/test-forest
new file mode 100755
--- /dev/null
+++ b/tests/test-forest
@@ -0,0 +1,104 @@
+#! /bin/sh
+
+HGRCPATH=$HGTMP/.hgrc
+export HGRCPATH
+echo "[extensions]" >> $HGRCPATH
+echo "mq=" >> $HGRCPATH
+echo "forest=" >> $HGRCPATH
+
+echo "# setup initial forest"
+hg init toplevel
+echo "f" > toplevel/f
+mkdir toplevel/d
+echo "d/f" > toplevel/d/f
+mkdir toplevel/d/d
+echo "d/d/f" > toplevel/d/d/f
+hg init toplevel/d/d/t
+echo "d/d/t/f" > toplevel/d/d/t/f
+hg init toplevel/t
+echo "t/f" > toplevel/t/f
+hg init toplevel/t/t
+echo "t/t/f" > toplevel/t/t/f
+hg commit --cwd toplevel -A -m "start" -d "0 0"
+hg commit --cwd toplevel/d/d/t -A -m "start" -d "0 0"
+hg commit --cwd toplevel/t -A -m "start" -d "0 0"
+hg commit --cwd toplevel/t/t -A -m "start" -d "0 0"
+
+echo "# ftrees"
+hg ftrees --cwd toplevel
+
+echo "# fstatus"
+echo "x" >> toplevel/d/d/t/f
+echo "new" >> toplevel/t/t/f2
+hg fstatus --cwd toplevel
+hg revert --cwd toplevel/d/d/t --no-backup f
+rm -f toplevel/t/t/f2
+hg fstatus --cwd toplevel
+
+echo "# fclone"
+hg fclone toplevel topcopy
+hg fsnap --cwd topcopy > top-snap
+
+echo "# fsnap"
+hg fsnap --cwd toplevel > top-snap1
+echo "x" >> toplevel/t/t/f
+hg commit --cwd toplevel/t/t -m "new line" -d "0 0"
+echo "f2" > toplevel/d/d/f2
+hg commit --cwd toplevel/d/d -A -m "new file" -d "0 0"
+hg fsnap --cwd toplevel > top-snap2
+diff -u top-snap1 top-snap2 | \
+ sed -e 's/--- top-snap1.*$/--- top-snap1/' \
+ -e 's/+++ top-snap2.*$/+++ top-snap2/'
+
+echo "# fseed"
+hg clone toplevel newtop
+hg fseed --cwd newtop ../top-snap default
+rm -rf newtop
+hg fseed --root newtop top-snap default >/dev/null
+hg --cwd newtop fsnap | sed "s@$HGTMP at HGTMP@g"
+rm -rf newtop
+
+echo "# fpull"
+hg fpull --cwd topcopy -u ../top-snap default | sed "s@$HGTMP at HGTMP@g"
+
+echo "# fpush"
+echo "t/t/f" > topcopy/t/t/f
+hg commit --cwd topcopy/t/t -m "delete new line" -d "0 0"
+hg remove --cwd topcopy d/d/f2
+hg commit --cwd topcopy/d/d -m "remove new file" -d "0 0"
+hg fpush --cwd topcopy ../top-snap default | sed "s@$HGTMP at HGTMP@g"
+
+# create an mq patch in topcopy/t
+hg qinit --cwd topcopy/t
+hg qnew --cwd topcopy/t mq-patch
+echo "zzz" > topcopy/t/z
+hg add --cwd topcopy/t z
+hg qrefresh --cwd topcopy/t
+
+echo "# fstatus + mq"
+hg fstatus --cwd topcopy
+
+echo "# fclone + mq"
+hg fclone topcopy newtop
+rm -rf newtop
+
+echo "# fsnap + mq"
+hg fsnap --cwd topcopy ../top-snap1
+
+echo "# fpull + mq"
+hg fpull --cwd topcopy -u ../top-snap default | sed "s@$HGTMP at HGTMP@g"
+
+echo "# fpush + mq"
+hg fpush --cwd topcopy ../top-snap default | sed "s@$HGTMP at HGTMP@g"
+
+echo "# walk **/.hg"
+hg init walkhg
+hg init walkhg/.hg/h
+hg init walkhg/a
+hg init walkhg/a/.hg/h
+hg ftrees --cwd walkhg
+hg ftrees --cwd walkhg --walkhg=1
+echo "[forest]" >> walkhg/.hg/hgrc
+echo "walkhg = 1" >> walkhg/.hg/hgrc
+hg ftrees --cwd walkhg
+hg ftrees --cwd walkhg --walkhg=0
diff --git a/tests/test-forest.out b/tests/test-forest.out
new file mode 100644
--- /dev/null
+++ b/tests/test-forest.out
@@ -0,0 +1,227 @@
+# setup initial forest
+adding d/d/f
+adding d/f
+adding f
+adding f
+adding f
+adding f
+# ftrees
+.
+d/d/t
+t
+t/t
+# fstatus
+[.]
+
+[d/d/t]
+M f
+
+[t]
+
+[t/t]
+? f2
+
+[.]
+
+[d/d/t]
+
+[t]
+
+[t/t]
+
+# fclone
+[.]
+3 files updated, 0 files merged, 0 files removed, 0 files unresolved
+
+[d/d/t]
+1 files updated, 0 files merged, 0 files removed, 0 files unresolved
+
+[t]
+1 files updated, 0 files merged, 0 files removed, 0 files unresolved
+
+[t/t]
+1 files updated, 0 files merged, 0 files removed, 0 files unresolved
+
+# fsnap
+adding d/d/f2
+--- top-snap1
++++ top-snap2
+@@ -1,6 +1,6 @@
+ [tree1]
+ root = .
+-revision = fccf42f55033a9715e9e990fcc1749e3d0d19d39
++revision = bc7d06dbb331e93b327d848dc724e61cd2dc2d66
+
+ [tree1.paths]
+
+@@ -18,7 +18,7 @@
+
+ [tree4]
+ root = t/t
+-revision = 5d60830890a20c050332e222b8307bbb70940a3f
++revision = e7ef7301b2ddca4eca0c4e80fe0cc8c943d05645
+
+ [tree4.paths]
+
+# fseed
+4 files updated, 0 files merged, 0 files removed, 0 files unresolved
+[d/d/t]
+1 files updated, 0 files merged, 0 files removed, 0 files unresolved
+0 files updated, 0 files merged, 0 files removed, 0 files unresolved
+
+[t]
+1 files updated, 0 files merged, 0 files removed, 0 files unresolved
+0 files updated, 0 files merged, 0 files removed, 0 files unresolved
+
+[t/t]
+1 files updated, 0 files merged, 0 files removed, 0 files unresolved
+1 files updated, 0 files merged, 0 files removed, 0 files unresolved
+
+[tree1]
+root = .
+revision = fccf42f55033a9715e9e990fcc1749e3d0d19d39
+
+[tree1.paths]
+default = HGTMP/test-forest/toplevel
+
+[tree2]
+root = d/d/t
+revision = 11d08ba64b676ed2f87a16089f3a0e5060c7bc36
+
+[tree2.paths]
+default = HGTMP/test-forest/toplevel/d/d/t
+
+[tree3]
+root = t
+revision = 37c7c7838b045dddb0718588a6318f002f0bed0a
+
+[tree3.paths]
+default = HGTMP/test-forest/toplevel/t
+
+[tree4]
+root = t/t
+revision = 5d60830890a20c050332e222b8307bbb70940a3f
+
+[tree4.paths]
+default = HGTMP/test-forest/toplevel/t/t
+
+# fpull
+[.]
+pulling from HGTMP/test-forest/toplevel
+searching for changes
+adding changesets
+adding manifests
+adding file changes
+added 1 changesets with 1 changes to 1 files
+1 files updated, 0 files merged, 0 files removed, 0 files unresolved
+
+[d/d/t]
+pulling from HGTMP/test-forest/toplevel/d/d/t
+searching for changes
+no changes found
+
+[t]
+pulling from HGTMP/test-forest/toplevel/t
+searching for changes
+no changes found
+
+[t/t]
+pulling from HGTMP/test-forest/toplevel/t/t
+searching for changes
+adding changesets
+adding manifests
+adding file changes
+added 1 changesets with 1 changes to 1 files
+1 files updated, 0 files merged, 0 files removed, 0 files unresolved
+
+# fpush
+[.]
+pushing to HGTMP/test-forest/toplevel
+searching for changes
+adding changesets
+adding manifests
+adding file changes
+added 1 changesets with 0 changes to 0 files
+
+[d/d/t]
+pushing to HGTMP/test-forest/toplevel/d/d/t
+searching for changes
+no changes found
+
+[t]
+pushing to HGTMP/test-forest/toplevel/t
+searching for changes
+no changes found
+
+[t/t]
+pushing to HGTMP/test-forest/toplevel/t/t
+searching for changes
+adding changesets
+adding manifests
+adding file changes
+added 1 changesets with 1 changes to 1 files
+
+# fstatus + mq
+[.]
+
+[d/d/t]
+
+[t] *mq*
+
+[t/t]
+
+# fclone + mq
+abort: 't' has mq patches applied
+# fsnap + mq
+abort: 't' has mq patches applied
+# fpull + mq
+[.]
+pulling from HGTMP/test-forest/toplevel
+searching for changes
+no changes found
+
+[d/d/t]
+pulling from HGTMP/test-forest/toplevel/d/d/t
+searching for changes
+no changes found
+
+[t]
+skipped, mq patches applied
+
+[t/t]
+pulling from HGTMP/test-forest/toplevel/t/t
+searching for changes
+no changes found
+
+# fpush + mq
+[.]
+pushing to HGTMP/test-forest/toplevel
+searching for changes
+no changes found
+
+[d/d/t]
+pushing to HGTMP/test-forest/toplevel/d/d/t
+searching for changes
+no changes found
+
+[t]
+skipped, mq patches applied
+
+[t/t]
+pushing to HGTMP/test-forest/toplevel/t/t
+searching for changes
+no changes found
+
+# walk **/.hg
+.
+a
+.
+.hg/h
+a
+a/.hg/h
+.
+.hg/h
+a
+a/.hg/h
+.
+a
More information about the Mercurial-devel
mailing list