D1856: wireproto: server-side support for pullbundles
joerg.sonnenberger (Joerg Sonnenberger)
phabricator at mercurial-scm.org
Fri Jan 12 17:32:19 UTC 2018
joerg.sonnenberger created this revision.
Herald added a subscriber: mercurial-devel.
Herald added a reviewer: hg-reviewers.
REVISION SUMMARY
Pullbundles are similar to clonebundles, but served as normal inline
bundle streams. They are almost transparent to the client -- the only
visible effect is that the client might get less changes than what it
asked for, i.e. not all requested head revisions are provided. The
missing client-side logic is to retry a pull in that case.
REPOSITORY
rHG Mercurial
REVISION DETAIL
https://phab.mercurial-scm.org/D1856
AFFECTED FILES
mercurial/configitems.py
mercurial/help/config.txt
mercurial/wireproto.py
tests/test-pull-r.t
CHANGE DETAILS
diff --git a/tests/test-pull-r.t b/tests/test-pull-r.t
--- a/tests/test-pull-r.t
+++ b/tests/test-pull-r.t
@@ -145,3 +145,59 @@
$ cd ..
$ killdaemons.py
+
+Test pullbundle functionality
+
+ $ cd repo
+ $ cat <<EOF > .hg/hgrc
+ > [server]
+ > pullbundle = True
+ > EOF
+ $ hg bundle --base null -r 0 .hg/0.hg
+ 1 changesets found
+ $ hg bundle --base 0 -r 1 .hg/1.hg
+ 1 changesets found
+ $ hg bundle --base 1 -r 2 .hg/2.hg
+ 1 changesets found
+ $ cat <<EOF > .hg/pullbundles.manifest
+ > 2.hg heads=effea6de0384e684f44435651cb7bd70b8735bd4 base=bbd179dfa0a71671c253b3ae0aa1513b60d199fa
+ > 1.hg heads=ed1b79f46b9a29f5a6efa59cf12fcfca43bead5a base=bbd179dfa0a71671c253b3ae0aa1513b60d199fa
+ > 0.hg heads=bbd179dfa0a71671c253b3ae0aa1513b60d199fa
+ > EOF
+ $ hg serve --debug -p $HGPORT2 --pid-file=../repo.pid > ../repo-server.txt 2>&1 &
+ $ while ! grep listening ../repo-server.txt > /dev/null; do sleep 1; done
+ $ cat ../repo.pid >> $DAEMON_PIDS
+ $ cd ..
+ $ hg clone -r 0 http://localhost:$HGPORT2/ repo.pullbundle
+ adding changesets
+ adding manifests
+ adding file changes
+ added 1 changesets with 1 changes to 1 files
+ new changesets bbd179dfa0a7
+ updating to branch default
+ 1 files updated, 0 files merged, 0 files removed, 0 files unresolved
+ $ cd repo.pullbundle
+ $ hg pull -r 1
+ pulling from http://localhost:$HGPORT2/
+ searching for changes
+ adding changesets
+ adding manifests
+ adding file changes
+ added 1 changesets with 1 changes to 1 files
+ new changesets ed1b79f46b9a
+ (run 'hg update' to get a working copy)
+ $ hg pull -r 2
+ pulling from http://localhost:$HGPORT2/
+ searching for changes
+ adding changesets
+ adding manifests
+ adding file changes
+ added 1 changesets with 1 changes to 1 files (+1 heads)
+ new changesets effea6de0384
+ (run 'hg heads' to see heads, 'hg merge' to merge)
+ $ cd ..
+ $ killdaemons.py
+ $ grep 'sending pullbundle ' repo-server.txt
+ sending pullbundle "0.hg"
+ sending pullbundle "1.hg"
+ sending pullbundle "2.hg"
diff --git a/mercurial/wireproto.py b/mercurial/wireproto.py
--- a/mercurial/wireproto.py
+++ b/mercurial/wireproto.py
@@ -831,6 +831,65 @@
opts = options('debugwireargs', ['three', 'four'], others)
return repo.debugwireargs(one, two, **pycompat.strkwargs(opts))
+def find_pullbundle(repo, opts, clheads, heads, common):
+ def decodehexstring(s):
+ return set([h.decode('hex') for h in s.split(':')])
+
+ def hasdescendant(ancestor, revs):
+ # Quick path for exact match
+ if ancestor in revs:
+ return True
+ for rev in revs:
+ if repo.changelog.isancestor(ancestor, rev):
+ return True
+ return False
+ def hasancestor(descendant, revs):
+ if descendant in revs:
+ return True
+ for rev in revs:
+ if repo.changelog.isancestor(rev, descendant):
+ return True
+ return False
+
+ manifest = repo.vfs.tryread('pullbundles.manifest')
+ res = exchange.parseclonebundlesmanifest(repo, manifest)
+ res = exchange.filterclonebundleentries(repo, res)
+ for entry in res:
+ if 'heads' in entry:
+ try:
+ bundle_heads = decodehexstring(entry['heads'])
+ except TypeError:
+ # Bad heads entry
+ continue
+ if len(bundle_heads) > len(heads):
+ # Client wants less heads than the bundle contains
+ continue
+ if bundle_heads.issubset(common):
+ continue # Nothing new
+ if all(hasdescendant(rev, common) for rev in bundle_heads):
+ continue # Still nothing new
+ if any(not hasdescendant(rev, heads) for rev in bundle_heads):
+ continue
+ if 'bases' in entry:
+ try:
+ bundle_bases = decodehexstring(entry['bases'])
+ except TypeError:
+ # Bad bases entry
+ continue
+ if len(bundle_bases) < len(common):
+ # Client is missing a revision the bundle requires
+ continue
+ if all(hasdescendant(rev, common) for rev in bundle_bases):
+ continue
+ path = entry['URL']
+ repo.ui.debug('sending pullbundle "%s"\n' % path)
+ try:
+ return repo.vfs.open(path)
+ except IOError:
+ repo.ui.debug('pullbundle "%s" not accessible\n' % path)
+ continue
+ return None
+
@wireprotocommand('getbundle', '*')
def getbundle(repo, proto, others):
opts = options('getbundle', gboptsmap.keys(), others)
@@ -861,12 +920,20 @@
hint=bundle2requiredhint)
try:
+ clheads = set(repo.changelog.heads())
+ heads = set(opts.get('heads', set()))
+ common = set(opts.get('common', set()))
+ common.discard(nullid)
+
+ if repo.ui.configbool('server', 'pullbundle'):
+ # Check if a pre-built bundle covers this request.
+ bundle = find_pullbundle(repo, opts, clheads, heads, common)
+ if bundle:
+ return streamres(gen=util.filechunkiter(bundle),
+ prefer_uncompressed=True)
+
if repo.ui.configbool('server', 'disablefullbundle'):
# Check to see if this is a full clone.
- clheads = set(repo.changelog.heads())
- heads = set(opts.get('heads', set()))
- common = set(opts.get('common', set()))
- common.discard(nullid)
if not common and clheads == heads:
raise error.Abort(
_('server has pull-based clones disabled'),
diff --git a/mercurial/help/config.txt b/mercurial/help/config.txt
--- a/mercurial/help/config.txt
+++ b/mercurial/help/config.txt
@@ -1772,6 +1772,14 @@
are highly recommended. Partial clones will still be allowed.
(default: False)
+``pullbundle``
+ When set, the server will check pullbundle.manifest for bundles
+ covering the requested heads and common nodes. The first matching
+ entry will be streamed to the client.
+
+ For HTTP transport, the stream will still use zlib compression
+ for older clients.
+
``concurrent-push-mode``
Level of allowed race condition between two pushing clients.
diff --git a/mercurial/configitems.py b/mercurial/configitems.py
--- a/mercurial/configitems.py
+++ b/mercurial/configitems.py
@@ -877,6 +877,9 @@
coreconfigitem('server', 'disablefullbundle',
default=False,
)
+coreconfigitem('server', 'pullbundle',
+ default=False,
+)
coreconfigitem('server', 'maxhttpheaderlen',
default=1024,
)
To: joerg.sonnenberger, #hg-reviewers
Cc: mercurial-devel
More information about the Mercurial-devel
mailing list