[PATCH 3 of 3 STABLE] pushkey: splits obsolete marker exchange into multiple keys

[pierre-yves.david at logilab.fr]

# HG changeset patch
# User Pierre-Yves David <pierre-yves.david at logilab.fr>
# Date 1343406776 -7200
# Branch stable
# Node ID d076b76c67cab06c9bab022a8d5268a875d04a91
# Parent  179fab052e985e418e30ca4e8980120753f27e65
pushkey: splits obsolete marker exchange into multiple keys

Obsolete markers are now exchanged in smaller pieces that fit in a http header.

This changes is done to avoid 400 bad request error when exchanging obsolete
puskey over http.

The last key pushed is always hold by the "dump0" key to ensure an easy place to
hook for people who need it.

diff --git a/mercurial/localrepo.py b/mercurial/localrepo.py
--- a/mercurial/localrepo.py
+++ b/mercurial/localrepo.py
@@ -1783,15 +1783,17 @@ class localrepository(object):
                 # should be seen as public
                 phases.advanceboundary(self, phases.public, subset)
 
             self.ui.debug('fetching remote obsolete markers')
             remoteobs = remote.listkeys('obsolete')
-            if 'dump' in remoteobs:
+            if 'dump0' in remoteobs:
                 if tr is None:
                     tr = self.transaction(trname)
-                data = base85.b85decode(remoteobs['dump'])
-                self.obsstore.mergemarkers(tr, data)
+                for key in sorted(remoteobs, reverse=True):
+                    if key.startswith('dump'):
+                        data = base85.b85decode(remoteobs[key])
+                        self.obsstore.mergemarkers(tr, data)
             if tr is not None:
                 tr.close()
         finally:
             if tr is not None:
                 tr.release()
@@ -1953,14 +1955,19 @@ class localrepository(object):
                             self.ui.warn(_('updating %s to public failed!\n')
                                             % newremotehead)
                 self.ui.debug('try to push obsolete markers to remote\n')
                 if (self.obsstore and
                     'obsolete' in remote.listkeys('namespaces')):
-                    data = self.listkeys('obsolete')['dump']
-                    r = remote.pushkey('obsolete', 'dump', '', data)
-                    if not r:
-                        self.ui.warn(_('failed to push obsolete markers!\n'))
+                    rslts = []
+                    remotedata = self.listkeys('obsolete')
+                    for key in sorted(remotedata, reverse=True):
+                        # reverse sort to ensure we end with dump0
+                        data = remotedata[key]
+                        rslts.append(remote.pushkey('obsolete', key, '', data))
+                    if [r for r in rslts if not r]:
+                        msg = _('failed to push some obsolete markers!\n')
+                        self.ui.warn(msg)
             finally:
                 if lock is not None:
                     lock.release()
         finally:
             locallock.release()
diff --git a/mercurial/obsolete.py b/mercurial/obsolete.py
--- a/mercurial/obsolete.py
+++ b/mercurial/obsolete.py
@@ -232,28 +232,49 @@ def _encodemarkers(markers, addheader=Fa
     # Kept separate from flushmarkers(), it will be reused for
     # markers exchange.
     if addheader:
         yield _pack('>B', _fmversion)
     for marker in markers:
-        pre, sucs, flags, metadata = marker
-        nbsuc = len(sucs)
-        format = _fmfixed + (_fmnode * nbsuc)
-        data = [nbsuc, len(metadata), flags, pre]
-        data.extend(sucs)
-        yield _pack(format, *data)
-        yield metadata
+        yield _encodeonemarker(marker)
+
+
+def _encodeonemarker(marker):
+    pre, sucs, flags, metadata = marker
+    nbsuc = len(sucs)
+    format = _fmfixed + (_fmnode * nbsuc)
+    data = [nbsuc, len(metadata), flags, pre]
+    data.extend(sucs)
+    return _pack(format, *data) + metadata
+
+# arbitrary picked to fit into 8K limit from HTTP server
+# you have to take in account:
+# - the version header
+# - the base85 encoding
+_maxpayload = 5300
 
 def listmarkers(repo):
     """List markers over pushkey"""
     if not repo.obsstore:
         return {}
-    markers = _encodemarkers(repo.obsstore, True)
-    return {'dump': base85.b85encode(''.join(markers))}
+    keys = {}
+    parts = []
+    currentlen = _maxpayload * 2  # ensure we create a new part
+    for marker in  repo.obsstore:
+        nextdata = _encodeonemarker(marker)
+        if (len(nextdata) + currentlen > _maxpayload):
+            currentpart = []
+            currentlen = 0
+            parts.append(currentpart)
+        currentpart.append(nextdata)
+    for idx, part in enumerate(reversed(parts)):
+        data = ''.join([_pack('>B', _fmversion)] + part)
+        keys['dump%i' % idx] = base85.b85encode(data)
+    return keys
 
 def pushmarker(repo, key, old, new):
     """Push markers over pushkey"""
-    if key != 'dump':
+    if not key.startswith('dump'):
         repo.ui.warn(_('unknown key: %r') % key)
         return 0
     if old:
         repo.ui.warn(_('unexpected old value') % key)
         return 0