[PATCH 3 of 4 v2] changegroup3: add empty chunk separating directories and files

[Martin von Zweigbergk]

# HG changeset patch
# User Martin von Zweigbergk <martinvonz at google.com>
# Date 1452553831 28800
#      Mon Jan 11 15:10:31 2016 -0800
# Node ID 12f845942d02e226575a9185cda3b2ede378fe46
# Parent  7d6bd9f7dfefe6a55f6569a5b977a1fad6ac4021
changegroup3: add empty chunk separating directories and files

Remotefilelog overrides changegroup._addchangegroupfiles(), assuming
it is about files, which seems like a natural assumption. However, in
changegroup3, directory manifests are sent in the files section of the
changegroup. These naturally make remotefilelog unhappy.

The fact that the directories are not separated from the files
(although they do come before the files) also makes server.validate
harder to implement. Since we read one chunk at a time from the steam,
once we have found a file (non-directory) entry in the stream, we
would have to push the read data back into the stream, or otherwise
refactor the code. It will be easier if we add an empty chunk after
all directory manifests.

This change adds that empty chunk, although we don't yet take
advantage of it on the reading side. We will soon move the tree
manifest stuff out of _addchangegroupfiles() and into
_unpackmanifests().

diff --git a/mercurial/bundlerepo.py b/mercurial/bundlerepo.py
--- a/mercurial/bundlerepo.py
+++ b/mercurial/bundlerepo.py
@@ -329,6 +329,10 @@
         # consume the header if it exists
         self.bundle.manifestheader()
         m = bundlemanifest(self.svfs, self.bundle, self.changelog.rev)
+        # XXX: hack to work with changegroup3, but we still don't handle
+        # tree manifests correctly
+        if self.bundle.version == "03":
+            self.bundle.filelogheader()
         self.filestart = self.bundle.tell()
         return m
 
diff --git a/mercurial/changegroup.py b/mercurial/changegroup.py
--- a/mercurial/changegroup.py
+++ b/mercurial/changegroup.py
@@ -506,8 +506,8 @@
     """Unpacker for cg3 streams.
 
     cg3 streams add support for exchanging treemanifests and revlog
-    flags, so the only changes from cg2 are the delta header and
-    version number.
+    flags. It adds the revlog flags to the delta header and an empty chunk
+    separating manifests and files.
     """
     deltaheader = _CHANGEGROUPV3_DELTA_HEADER
     deltaheadersize = struct.calcsize(deltaheader)
@@ -909,6 +909,7 @@
             yield self.fileheader(name)
             for chunk in self.group(nodes, dirlog(name), nodes.get):
                 yield chunk
+        yield self.close()
 
     def builddeltaheader(self, node, p1n, p2n, basenode, linknode, flags):
         return struct.pack(
@@ -917,7 +918,7 @@
 _packermap = {'01': (cg1packer, cg1unpacker),
              # cg2 adds support for exchanging generaldelta
              '02': (cg2packer, cg2unpacker),
-             # cg3 adds support for exchanging treemanifests
+             # cg3 adds support for exchanging revlog flags and treemanifests
              '03': (cg3packer, cg3unpacker),
 }
 
@@ -1054,9 +1055,13 @@
 def _addchangegroupfiles(repo, source, revmap, trp, pr, needfiles):
     revisions = 0
     files = 0
+    submfsdone = False
     while True:
         chunkdata = source.filelogheader()
         if not chunkdata:
+            if source.version == "03" and not submfsdone:
+                submfsdone = True
+                continue
             break
         f = chunkdata["filename"]
         repo.ui.debug("adding %s revisions\n" % f)