[PATCH RFC] bundle2: add the ability to fetch part content from another URL
Boris Feld
boris.feld at octobus.net
Thu Oct 18 17:37:31 UTC 2018
# HG changeset patch
# User Boris Feld <boris.feld at octobus.net>
# Date 1539880502 -7200
# Thu Oct 18 18:35:02 2018 +0200
# Node ID 9b6b8f0b3d821f17db216ba346934b8ffb6de1d5
# Parent 2c0aa02ecd5a05ae76b6345962ee3a0ef773bd8a
# EXP-Topic bundle2-remote
# Available At https://bitbucket.org/octobus/mercurial-devel/
# hg pull https://bitbucket.org/octobus/mercurial-devel/ -r 9b6b8f0b3d82
bundle2: add the ability to fetch part content from another URL
This is a first changeset adding the ability to retrieve a part content from
another location. The part headers and its parameters still need to be set by
the initial server. This is especially helpful to save server CPU and
bandwidth when the part content is large and expensive to compute.
Such feature is very useful when combined with the stable-range slicing that
being experimented in a third party extension. However, it needs client-side
support in Core to be properly leveraged. It is inspired by Gregory Szorc
generic redirect support in protocol v2.
To be fully usable, this feature would need a couple more straightforward
changes:
* digests and size checking,
* compression support,
* protocol restriction (maybe),
* ability to disable the feature server side.
* advertised as a bundle2 capability
The current changeset is provided as an RFC.
diff --git a/mercurial/bundle2.py b/mercurial/bundle2.py
--- a/mercurial/bundle2.py
+++ b/mercurial/bundle2.py
@@ -1292,12 +1292,14 @@ class unbundlepart(unpackermixin):
def _initparams(self, mandatoryparams, advisoryparams):
"""internal function to setup all logic related parameters"""
# make it read only to prevent people touching it by mistake.
- self.mandatoryparams = tuple(mandatoryparams)
- self.advisoryparams = tuple(advisoryparams)
+ self.mandatoryparams = tuple(p for p in mandatoryparams
+ if not p[0].startswith('remote-content'))
+ self.advisoryparams = tuple(p for p in advisoryparams
+ if not p[0].startswith('remote-content'))
# user friendly UI
- self.params = util.sortdict(self.mandatoryparams)
- self.params.update(self.advisoryparams)
- self.mandatorykeys = frozenset(p[0] for p in mandatoryparams)
+ self.params = util.sortdict(mandatoryparams)
+ self.params.update(advisoryparams)
+ self.mandatorykeys = frozenset(p[0] for p in self.mandatoryparams)
def _readheader(self):
"""read the header and setup the object"""
@@ -1330,10 +1332,25 @@ class unbundlepart(unpackermixin):
advparams.append((self._fromheader(key), self._fromheader(value)))
self._initparams(manparams, advparams)
## part payload
- self._payloadstream = util.chunkbuffer(self._payloadchunks())
+ self._payloadstream = self._payloadstream()
# we read the data, tell it
self._initialized = True
+
+ def _payloadstream(self):
+ """filelike object reading the part payload
+
+ The payload might be fetch remotely."""
+
+ payload = util.chunkbuffer(self._payloadchunks())
+ remotecontent = self.params.pop('remote-content', None)
+ if remotecontent is None:
+ return payload
+ else:
+ fullurl = payload.read()
+ data = url.open(self.ui, fullurl)
+ return data
+
def _payloadchunks(self):
"""Generator of decoded chunks in the payload."""
return decodepayloadchunks(self.ui, self._fp)
diff --git a/tests/test-bundle2-remote-changegroup.t b/tests/test-bundle2-remote-changegroup.t
--- a/tests/test-bundle2-remote-changegroup.t
+++ b/tests/test-bundle2-remote-changegroup.t
@@ -74,6 +74,21 @@ Create an extension to test bundle2 remo
> part = newpart(b'remote-changegroup')
> for k, v in eval(args).items():
> part.addparam(pycompat.sysbytes(k), pycompat.bytestr(v))
+ > elif verb == b'native-remote-changegroup':
+ > fullurl, filepath, revs = args.split()
+ > nodes = [c.node() for c in repo.set(revs)]
+ > outgoing = discovery.outgoing(repo, missingroots=nodes, missingheads=nodes)
+ > cgversion = b'02'
+ > cgstream = changegroup.makestream(repo, outgoing, cgversion, source,
+ > bundlecaps=bundlecaps)
+ > with open(filepath, 'wb') as cachefile:
+ > for chunk in cgstream:
+ > cachefile.write(chunk)
+ > part = newpart(b'changegroup', fullurl)
+ > part.addparam(b'remote-content', b'1')
+ > part.addparam(b'version', cgversion)
+ > part.addparam(b'nbchanges', pycompat.bytestr(len(outgoing.missing)))
+ > part.addparam(b'targetphase', b'1')
> elif verb == b'changegroup':
> _common, heads = args.split()
> common.extend(repo[r].node() for r in repo.revs(_common))
@@ -179,6 +194,48 @@ Test a pull with an remote-changegroup
$ rm -rf clone
+Test a pull with an changegroup remotely fetch by bundle2's own feature
+
+ $ cat > repo/.hg/bundle2maker << EOF
+ > native-remote-changegroup http://localhost:$HGPORT/bundle.b2part bundle.b2part 5:7
+ > EOF
+ $ hg clone orig clone -r 3 -r 4
+ adding changesets
+ adding manifests
+ adding file changes
+ added 5 changesets with 5 changes to 5 files (+1 heads)
+ new changesets cd010b8cd998:9520eea781bc
+ updating to branch default
+ 4 files updated, 0 files merged, 0 files removed, 0 files unresolved
+ $ hg pull -R clone ssh://user@dummy/repo --traceback
+ pulling from ssh://user@dummy/repo
+ searching for changes
+ remote: changegroup
+ adding changesets
+ adding manifests
+ adding file changes
+ added 3 changesets with 2 changes to 2 files (+1 heads)
+ new changesets 24b6387c8c8c:02de42196ebe
+ (run 'hg heads .' to see heads, 'hg merge' to merge)
+ $ hg -R clone log -G
+ o 7:02de42196ebe public Nicolas Dumazet <nicdumz.commits at gmail.com> H
+ |
+ | o 6:eea13746799a public Nicolas Dumazet <nicdumz.commits at gmail.com> G
+ |/|
+ o | 5:24b6387c8c8c public Nicolas Dumazet <nicdumz.commits at gmail.com> F
+ | |
+ | o 4:9520eea781bc public Nicolas Dumazet <nicdumz.commits at gmail.com> E
+ |/
+ | @ 3:32af7686d403 public Nicolas Dumazet <nicdumz.commits at gmail.com> D
+ | |
+ | o 2:5fddd98957c8 public Nicolas Dumazet <nicdumz.commits at gmail.com> C
+ | |
+ | o 1:42ccdea3bb16 public Nicolas Dumazet <nicdumz.commits at gmail.com> B
+ |/
+ o 0:cd010b8cd998 public Nicolas Dumazet <nicdumz.commits at gmail.com> A
+
+ $ rm -rf clone
+
Test a pull with an remote-changegroup and a following changegroup
$ hg bundle -R repo --type v1 --base 2 -r '3:4' bundle2.hg
More information about the Mercurial-devel
mailing list