D1856: wireproto: server-side support for pullbundles

[joerg.sonnenberger (Joerg Sonnenberger)]

joerg.sonnenberger updated this revision to Diff 4811.

REPOSITORY
  rHG Mercurial

CHANGES SINCE LAST UPDATE
  https://phab.mercurial-scm.org/D1856?vs=4810&id=4811

REVISION DETAIL
  https://phab.mercurial-scm.org/D1856

AFFECTED FILES
  mercurial/configitems.py
  mercurial/help/config.txt
  mercurial/wireproto.py
  tests/test-pull-r.t

CHANGE DETAILS

diff --git a/tests/test-pull-r.t b/tests/test-pull-r.t
--- a/tests/test-pull-r.t
+++ b/tests/test-pull-r.t
@@ -145,3 +145,59 @@
 
   $ cd ..
   $ killdaemons.py
+
+Test pullbundle functionality
+
+  $ cd repo
+  $ cat <<EOF > .hg/hgrc
+  > [server]
+  > pullbundle = True
+  > EOF
+  $ hg bundle --base null -r 0 .hg/0.hg
+  1 changesets found
+  $ hg bundle --base 0 -r 1 .hg/1.hg
+  1 changesets found
+  $ hg bundle --base 1 -r 2 .hg/2.hg
+  1 changesets found
+  $ cat <<EOF > .hg/pullbundles.manifest
+  > 2.hg heads=effea6de0384e684f44435651cb7bd70b8735bd4 bases=bbd179dfa0a71671c253b3ae0aa1513b60d199fa
+  > 1.hg heads=ed1b79f46b9a29f5a6efa59cf12fcfca43bead5a bases=bbd179dfa0a71671c253b3ae0aa1513b60d199fa
+  > 0.hg heads=bbd179dfa0a71671c253b3ae0aa1513b60d199fa
+  > EOF
+  $ hg serve --debug -p $HGPORT2 --pid-file=../repo.pid > ../repo-server.txt 2>&1 &
+  $ while ! grep listening ../repo-server.txt > /dev/null; do sleep 1; done
+  $ cat ../repo.pid >> $DAEMON_PIDS
+  $ cd ..
+  $ hg clone -r 0 http://localhost:$HGPORT2/ repo.pullbundle
+  adding changesets
+  adding manifests
+  adding file changes
+  added 1 changesets with 1 changes to 1 files
+  new changesets bbd179dfa0a7
+  updating to branch default
+  1 files updated, 0 files merged, 0 files removed, 0 files unresolved
+  $ cd repo.pullbundle
+  $ hg pull -r 1
+  pulling from http://localhost:$HGPORT2/
+  searching for changes
+  adding changesets
+  adding manifests
+  adding file changes
+  added 1 changesets with 1 changes to 1 files
+  new changesets ed1b79f46b9a
+  (run 'hg update' to get a working copy)
+  $ hg pull -r 2
+  pulling from http://localhost:$HGPORT2/
+  searching for changes
+  adding changesets
+  adding manifests
+  adding file changes
+  added 1 changesets with 1 changes to 1 files (+1 heads)
+  new changesets effea6de0384
+  (run 'hg heads' to see heads, 'hg merge' to merge)
+  $ cd ..
+  $ killdaemons.py
+  $ grep 'sending pullbundle ' repo-server.txt
+  sending pullbundle "0.hg"
+  sending pullbundle "1.hg"
+  sending pullbundle "2.hg"
diff --git a/mercurial/wireproto.py b/mercurial/wireproto.py
--- a/mercurial/wireproto.py
+++ b/mercurial/wireproto.py
@@ -831,6 +831,65 @@
     opts = options('debugwireargs', ['three', 'four'], others)
     return repo.debugwireargs(one, two, **pycompat.strkwargs(opts))
 
+def find_pullbundle(repo, opts, clheads, heads, common):
+    def decodehexstring(s):
+        return set([h.decode('hex') for h in s.split(':')])
+
+    def hasdescendant(ancestor, revs):
+        # Quick path for exact match
+        if ancestor in revs:
+            return True
+        for rev in revs:
+            if repo.changelog.isancestor(ancestor, rev):
+                return True
+        return False
+    def hasancestor(descendant, revs):
+        if descendant in revs:
+            return True
+        for rev in revs:
+            if repo.changelog.isancestor(rev, descendant):
+                return True
+        return False
+
+    manifest = repo.vfs.tryread('pullbundles.manifest')
+    res = exchange.parseclonebundlesmanifest(repo, manifest)
+    res = exchange.filterclonebundleentries(repo, res)
+    for entry in res:
+        if 'heads' in entry:
+            try:
+                bundle_heads = decodehexstring(entry['heads'])
+            except TypeError:
+                # Bad heads entry
+                continue
+            if len(bundle_heads) > len(heads):
+                # Client wants less heads than the bundle contains
+                continue
+            if bundle_heads.issubset(common):
+                continue # Nothing new
+            if all(hasdescendant(rev, common) for rev in bundle_heads):
+                continue # Still nothing new
+            if any(not hasdescendant(rev, heads) for rev in bundle_heads):
+                continue
+        if 'bases' in entry:
+            try:
+                bundle_bases = decodehexstring(entry['bases'])
+            except TypeError:
+                # Bad bases entry
+                continue
+            if len(bundle_bases) > len(common):
+                # Client is missing a revision the bundle requires
+                continue
+            if not all(hasdescendant(rev, common) for rev in bundle_bases):
+                continue
+        path = entry['URL']
+        repo.ui.debug('sending pullbundle "%s"\n' % path)
+        try:
+            return repo.vfs.open(path)
+        except IOError:
+            repo.ui.debug('pullbundle "%s" not accessible\n' % path)
+            continue
+    return None
+
 @wireprotocommand('getbundle', '*')
 def getbundle(repo, proto, others):
     opts = options('getbundle', gboptsmap.keys(), others)
@@ -861,12 +920,20 @@
                               hint=bundle2requiredhint)
 
     try:
+        clheads = set(repo.changelog.heads())
+        heads = set(opts.get('heads', set()))
+        common = set(opts.get('common', set()))
+        common.discard(nullid)
+
+        if repo.ui.configbool('server', 'pullbundle'):
+            # Check if a pre-built bundle covers this request.
+            bundle = find_pullbundle(repo, opts, clheads, heads, common)
+            if bundle:
+                return streamres(gen=util.filechunkiter(bundle),
+                                 prefer_uncompressed=True)
+
         if repo.ui.configbool('server', 'disablefullbundle'):
             # Check to see if this is a full clone.
-            clheads = set(repo.changelog.heads())
-            heads = set(opts.get('heads', set()))
-            common = set(opts.get('common', set()))
-            common.discard(nullid)
             if not common and clheads == heads:
                 raise error.Abort(
                     _('server has pull-based clones disabled'),
diff --git a/mercurial/help/config.txt b/mercurial/help/config.txt
--- a/mercurial/help/config.txt
+++ b/mercurial/help/config.txt
@@ -1772,6 +1772,14 @@
     are highly recommended. Partial clones will still be allowed.
     (default: False)
 
+``pullbundle``
+    When set, the server will check pullbundle.manifest for bundles
+    covering the requested heads and common nodes. The first matching
+    entry will be streamed to the client.
+
+    For HTTP transport, the stream will still use zlib compression
+    for older clients.
+
 ``concurrent-push-mode``
     Level of allowed race condition between two pushing clients.
 
diff --git a/mercurial/configitems.py b/mercurial/configitems.py
--- a/mercurial/configitems.py
+++ b/mercurial/configitems.py
@@ -877,6 +877,9 @@
 coreconfigitem('server', 'disablefullbundle',
     default=False,
 )
+coreconfigitem('server', 'pullbundle',
+    default=False,
+)
 coreconfigitem('server', 'maxhttpheaderlen',
     default=1024,
 )



To: joerg.sonnenberger, #hg-reviewers
Cc: mercurial-devel