D4982: exchangev2: use filesdata
indygreg (Gregory Szorc)
phabricator at mercurial-scm.org
Fri Oct 12 11:17:54 EDT 2018
This revision was automatically updated to reflect the committed changes.
Closed by commit rHGb843356d4ae1: exchangev2: use filesdata (authored by indygreg, committed by ).
REPOSITORY
rHG Mercurial
CHANGES SINCE LAST UPDATE
https://phab.mercurial-scm.org/D4982?vs=11848&id=11889
REVISION DETAIL
https://phab.mercurial-scm.org/D4982
AFFECTED FILES
mercurial/exchangev2.py
tests/test-wireproto-exchangev2.t
CHANGE DETAILS
diff --git a/tests/test-wireproto-exchangev2.t b/tests/test-wireproto-exchangev2.t
--- a/tests/test-wireproto-exchangev2.t
+++ b/tests/test-wireproto-exchangev2.t
@@ -101,40 +101,30 @@
received frame(size=11; request=1; stream=2; streamflags=encoded; type=command-response; flags=continuation)
received frame(size=992; request=1; stream=2; streamflags=encoded; type=command-response; flags=continuation)
received frame(size=0; request=1; stream=2; streamflags=; type=command-response; flags=eos)
- sending 2 commands
- sending command filedata: {
+ sending 1 commands
+ sending command filesdata: {
'fields': set([
'parents',
'revision'
]),
'haveparents': True,
- 'nodes': [
- '+N\xb0s\x19\xbf\xa0w\xa4\n/\x04\x916Y\xae\xf0\xdaB\xda',
- '\x9a8\x12)\x97\xb3\xac\x97\xbe*\x9a\xa2\xe5V\x83\x83A\xfd\xf2\xcc',
- '\xc2\xa2\x05\xc8\xb2\xad\xe2J\xf2`b\xe5<\xd5\xbc8\x01\xd6`\xda'
- ],
- 'path': 'a'
- }
- sending command filedata: {
- 'fields': set([
- 'parents',
- 'revision'
- ]),
- 'haveparents': True,
- 'nodes': [
- '\x81\x9e%\x8d1\xa5\xe1`f)\xf3e\xbb\x90*\x1b!\xeeB\x16',
- '\xb1zk\xd3g=\x9a\xb8\xce\xd5\x81\xa2\t\xf6/=\xa5\xccEx',
- '\xc5\xb1\xf9\xd3n\x1c\xc18\xbf\xb6\xef\xb3\xde\xb7]\x8c\xcad\x94\xc3'
- ],
- 'path': 'b'
+ 'revisions': [
+ {
+ 'nodes': [
+ '3\x90\xef\x85\x00s\xfb\xc2\xf0\xdf\xff"D4,\x8e\x92)\x01:',
+ 'D2\xd86&\xe8\xa9\x86U\xf0b\xec\x1f*C\xb0\x7f\x7f\xbb\xb0',
+ '\xcd%4vk\xec\xe18\xc7\xc1\xaf\xdch%0/\x0fb\xd8\x1f',
+ '\xe9j\xe2\x0fA\x88H{\x9a\xe4\xef9A\xc2|\x81\x141F\xe5',
+ '\xca\xa2\xa4eE\x1d\xd1\xfa\xcd\xa0\xf5\xb1#\x12\xc3UXA\x88\xa1'
+ ],
+ 'type': 'changesetexplicit'
+ }
+ ]
}
received frame(size=9; request=1; stream=2; streamflags=stream-begin; type=stream-settings; flags=eos)
received frame(size=11; request=1; stream=2; streamflags=encoded; type=command-response; flags=continuation)
- received frame(size=431; request=1; stream=2; streamflags=encoded; type=command-response; flags=continuation)
+ received frame(size=901; request=1; stream=2; streamflags=encoded; type=command-response; flags=continuation)
received frame(size=0; request=1; stream=2; streamflags=; type=command-response; flags=eos)
- received frame(size=11; request=3; stream=2; streamflags=encoded; type=command-response; flags=continuation)
- received frame(size=431; request=3; stream=2; streamflags=encoded; type=command-response; flags=continuation)
- received frame(size=0; request=3; stream=2; streamflags=; type=command-response; flags=eos)
updating the branch cache
new changesets 3390ef850073:caa2a465451d (3 drafts)
(sent 5 HTTP requests and * bytes; received * bytes in responses) (glob)
@@ -239,37 +229,27 @@
received frame(size=11; request=1; stream=2; streamflags=encoded; type=command-response; flags=continuation)
received frame(size=404; request=1; stream=2; streamflags=encoded; type=command-response; flags=continuation)
received frame(size=0; request=1; stream=2; streamflags=; type=command-response; flags=eos)
- sending 2 commands
- sending command filedata: {
+ sending 1 commands
+ sending command filesdata: {
'fields': set([
'parents',
'revision'
]),
'haveparents': True,
- 'nodes': [
- '+N\xb0s\x19\xbf\xa0w\xa4\n/\x04\x916Y\xae\xf0\xdaB\xda',
- '\x9a8\x12)\x97\xb3\xac\x97\xbe*\x9a\xa2\xe5V\x83\x83A\xfd\xf2\xcc'
- ],
- 'path': 'a'
- }
- sending command filedata: {
- 'fields': set([
- 'parents',
- 'revision'
- ]),
- 'haveparents': True,
- 'nodes': [
- '\x81\x9e%\x8d1\xa5\xe1`f)\xf3e\xbb\x90*\x1b!\xeeB\x16'
- ],
- 'path': 'b'
+ 'revisions': [
+ {
+ 'nodes': [
+ '3\x90\xef\x85\x00s\xfb\xc2\xf0\xdf\xff"D4,\x8e\x92)\x01:',
+ 'D2\xd86&\xe8\xa9\x86U\xf0b\xec\x1f*C\xb0\x7f\x7f\xbb\xb0'
+ ],
+ 'type': 'changesetexplicit'
+ }
+ ]
}
received frame(size=9; request=1; stream=2; streamflags=stream-begin; type=stream-settings; flags=eos)
received frame(size=11; request=1; stream=2; streamflags=encoded; type=command-response; flags=continuation)
- received frame(size=277; request=1; stream=2; streamflags=encoded; type=command-response; flags=continuation)
+ received frame(size=439; request=1; stream=2; streamflags=encoded; type=command-response; flags=continuation)
received frame(size=0; request=1; stream=2; streamflags=; type=command-response; flags=eos)
- received frame(size=11; request=3; stream=2; streamflags=encoded; type=command-response; flags=continuation)
- received frame(size=123; request=3; stream=2; streamflags=encoded; type=command-response; flags=continuation)
- received frame(size=0; request=3; stream=2; streamflags=; type=command-response; flags=eos)
updating the branch cache
new changesets 3390ef850073:4432d83626e8
(sent 6 HTTP requests and * bytes; received * bytes in responses) (glob)
@@ -357,39 +337,28 @@
received frame(size=11; request=1; stream=2; streamflags=encoded; type=command-response; flags=continuation)
received frame(size=601; request=1; stream=2; streamflags=encoded; type=command-response; flags=continuation)
received frame(size=0; request=1; stream=2; streamflags=; type=command-response; flags=eos)
- sending 2 commands
- sending command filedata: {
+ sending 1 commands
+ sending command filesdata: {
'fields': set([
'parents',
'revision'
]),
'haveparents': True,
- 'nodes': [
- '+N\xb0s\x19\xbf\xa0w\xa4\n/\x04\x916Y\xae\xf0\xdaB\xda',
- '\xc2\xa2\x05\xc8\xb2\xad\xe2J\xf2`b\xe5<\xd5\xbc8\x01\xd6`\xda'
- ],
- 'path': 'a'
- }
- sending command filedata: {
- 'fields': set([
- 'parents',
- 'revision'
- ]),
- 'haveparents': True,
- 'nodes': [
- '\x81\x9e%\x8d1\xa5\xe1`f)\xf3e\xbb\x90*\x1b!\xeeB\x16',
- '\xb1zk\xd3g=\x9a\xb8\xce\xd5\x81\xa2\t\xf6/=\xa5\xccEx',
- '\xc5\xb1\xf9\xd3n\x1c\xc18\xbf\xb6\xef\xb3\xde\xb7]\x8c\xcad\x94\xc3'
- ],
- 'path': 'b'
+ 'revisions': [
+ {
+ 'nodes': [
+ '\xcd%4vk\xec\xe18\xc7\xc1\xaf\xdch%0/\x0fb\xd8\x1f',
+ '\xe9j\xe2\x0fA\x88H{\x9a\xe4\xef9A\xc2|\x81\x141F\xe5',
+ '\xca\xa2\xa4eE\x1d\xd1\xfa\xcd\xa0\xf5\xb1#\x12\xc3UXA\x88\xa1'
+ ],
+ 'type': 'changesetexplicit'
+ }
+ ]
}
received frame(size=9; request=1; stream=2; streamflags=stream-begin; type=stream-settings; flags=eos)
received frame(size=11; request=1; stream=2; streamflags=encoded; type=command-response; flags=continuation)
- received frame(size=277; request=1; stream=2; streamflags=encoded; type=command-response; flags=continuation)
+ received frame(size=527; request=1; stream=2; streamflags=encoded; type=command-response; flags=continuation)
received frame(size=0; request=1; stream=2; streamflags=; type=command-response; flags=eos)
- received frame(size=11; request=3; stream=2; streamflags=encoded; type=command-response; flags=continuation)
- received frame(size=431; request=3; stream=2; streamflags=encoded; type=command-response; flags=continuation)
- received frame(size=0; request=3; stream=2; streamflags=; type=command-response; flags=eos)
updating the branch cache
new changesets cd2534766bec:caa2a465451d (3 drafts)
(run 'hg update' to get a working copy)
@@ -557,40 +526,30 @@
received frame(size=11; request=1; stream=2; streamflags=encoded; type=command-response; flags=continuation)
received frame(size=992; request=1; stream=2; streamflags=encoded; type=command-response; flags=continuation)
received frame(size=0; request=1; stream=2; streamflags=; type=command-response; flags=eos)
- sending 2 commands
- sending command filedata: {
+ sending 1 commands
+ sending command filesdata: {
'fields': set([
'parents',
'revision'
]),
'haveparents': True,
- 'nodes': [
- '+N\xb0s\x19\xbf\xa0w\xa4\n/\x04\x916Y\xae\xf0\xdaB\xda',
- '\x9a8\x12)\x97\xb3\xac\x97\xbe*\x9a\xa2\xe5V\x83\x83A\xfd\xf2\xcc',
- '\xc2\xa2\x05\xc8\xb2\xad\xe2J\xf2`b\xe5<\xd5\xbc8\x01\xd6`\xda'
- ],
- 'path': 'a'
- }
- sending command filedata: {
- 'fields': set([
- 'parents',
- 'revision'
- ]),
- 'haveparents': True,
- 'nodes': [
- '\x81\x9e%\x8d1\xa5\xe1`f)\xf3e\xbb\x90*\x1b!\xeeB\x16',
- '\xb1zk\xd3g=\x9a\xb8\xce\xd5\x81\xa2\t\xf6/=\xa5\xccEx',
- '\xc5\xb1\xf9\xd3n\x1c\xc18\xbf\xb6\xef\xb3\xde\xb7]\x8c\xcad\x94\xc3'
- ],
- 'path': 'b'
+ 'revisions': [
+ {
+ 'nodes': [
+ '3\x90\xef\x85\x00s\xfb\xc2\xf0\xdf\xff"D4,\x8e\x92)\x01:',
+ 'D2\xd86&\xe8\xa9\x86U\xf0b\xec\x1f*C\xb0\x7f\x7f\xbb\xb0',
+ '\xcd%4vk\xec\xe18\xc7\xc1\xaf\xdch%0/\x0fb\xd8\x1f',
+ '\xe9j\xe2\x0fA\x88H{\x9a\xe4\xef9A\xc2|\x81\x141F\xe5',
+ '\xca\xa2\xa4eE\x1d\xd1\xfa\xcd\xa0\xf5\xb1#\x12\xc3UXA\x88\xa1'
+ ],
+ 'type': 'changesetexplicit'
+ }
+ ]
}
received frame(size=9; request=1; stream=2; streamflags=stream-begin; type=stream-settings; flags=eos)
received frame(size=11; request=1; stream=2; streamflags=encoded; type=command-response; flags=continuation)
- received frame(size=431; request=1; stream=2; streamflags=encoded; type=command-response; flags=continuation)
+ received frame(size=901; request=1; stream=2; streamflags=encoded; type=command-response; flags=continuation)
received frame(size=0; request=1; stream=2; streamflags=; type=command-response; flags=eos)
- received frame(size=11; request=3; stream=2; streamflags=encoded; type=command-response; flags=continuation)
- received frame(size=431; request=3; stream=2; streamflags=encoded; type=command-response; flags=continuation)
- received frame(size=0; request=3; stream=2; streamflags=; type=command-response; flags=eos)
updating the branch cache
new changesets 3390ef850073:caa2a465451d (1 drafts)
(sent 5 HTTP requests and * bytes; received * bytes in responses) (glob)
diff --git a/mercurial/exchangev2.py b/mercurial/exchangev2.py
--- a/mercurial/exchangev2.py
+++ b/mercurial/exchangev2.py
@@ -64,7 +64,8 @@
# Find all file nodes referenced by added manifests and fetch those
# revisions.
fnodes = _derivefilesfrommanifests(repo, manres['added'])
- _fetchfiles(repo, tr, remote, fnodes, manres['linkrevs'])
+ _fetchfilesfromcsets(repo, tr, remote, fnodes, csetres['added'],
+ manres['linkrevs'])
def _pullchangesetdiscovery(repo, remote, heads, abortwhenunrelated=True):
"""Determine which changesets need to be pulled."""
@@ -346,6 +347,7 @@
return fnodes
def _fetchfiles(repo, tr, remote, fnodes, linkrevs):
+ """Fetch file data from explicit file revisions."""
def iterrevisions(objs, progress):
for filerevision in objs:
node = filerevision[b'node']
@@ -418,3 +420,84 @@
iterrevisions(objs, progress),
locallinkrevs[path].__getitem__,
weakref.proxy(tr))
+
+def _fetchfilesfromcsets(repo, tr, remote, fnodes, csets, manlinkrevs):
+ """Fetch file data from explicit changeset revisions."""
+
+ def iterrevisions(objs, remaining, progress):
+ while remaining:
+ filerevision = next(objs)
+
+ node = filerevision[b'node']
+
+ extrafields = {}
+
+ for field, size in filerevision.get(b'fieldsfollowing', []):
+ extrafields[field] = next(objs)
+
+ if b'delta' in extrafields:
+ basenode = filerevision[b'deltabasenode']
+ delta = extrafields[b'delta']
+ elif b'revision' in extrafields:
+ basenode = nullid
+ revision = extrafields[b'revision']
+ delta = mdiff.trivialdiffheader(len(revision)) + revision
+ else:
+ continue
+
+ yield (
+ node,
+ filerevision[b'parents'][0],
+ filerevision[b'parents'][1],
+ node,
+ basenode,
+ delta,
+ # Flags not yet supported.
+ 0,
+ )
+
+ progress.increment()
+ remaining -= 1
+
+ progress = repo.ui.makeprogress(
+ _('files'), unit=_('chunks'),
+ total=sum(len(v) for v in fnodes.itervalues()))
+
+ commandmeta = remote.apidescriptor[b'commands'][b'filesdata']
+ batchsize = commandmeta.get(b'recommendedbatchsize', 50000)
+
+ for i in pycompat.xrange(0, len(csets), batchsize):
+ batch = [x for x in csets[i:i + batchsize]]
+ if not batch:
+ continue
+
+ with remote.commandexecutor() as e:
+ args = {
+ b'revisions': [{
+ b'type': b'changesetexplicit',
+ b'nodes': batch,
+ }],
+ b'fields': {b'parents', b'revision'},
+ b'haveparents': True,
+ }
+
+ objs = e.callcommand(b'filesdata', args).result()
+
+ # First object is an overall header.
+ overall = next(objs)
+
+ # We have overall['totalpaths'] segments.
+ for i in pycompat.xrange(overall[b'totalpaths']):
+ header = next(objs)
+
+ path = header[b'path']
+ store = repo.file(path)
+
+ linkrevs = {
+ fnode: manlinkrevs[mnode]
+ for fnode, mnode in fnodes[path].iteritems()}
+
+ store.addgroup(iterrevisions(objs, header[b'totalitems'],
+ progress),
+ linkrevs.__getitem__,
+ weakref.proxy(tr))
To: indygreg, #hg-reviewers
Cc: mercurial-devel
More information about the Mercurial-devel
mailing list