[PATCH rebased-on-peer] obsolete: exchange obsolete markers using dedicated wireprotocol commands
Pierre-Yves David
pierre-yves.david at ens-lyon.org
Mon Jul 16 18:13:59 CDT 2012
# HG changeset patch
# User Pierre-Yves David <pierre-yves.david at logilab.fr>
# Date 1342480369 -7200
# Node ID d91a49381c617151de72b02d0542f51687384e48
# Parent ac872a134978ef84a155563f0a7cc2e297b707e8
obsolete: exchange obsolete markers using dedicated wireprotocol commands
This changeset drop pushkeybased exchange in favor of dedicated wireprotocol
commands. Pushkey is not designed to exchange high amount of data. For example
the http implementation of pushkey use http header and just can't handle
- `readmarkers` and `writemarkers` are made public function and used directly,
- pushkey implementation are dropped,
- peers gains an `addobsoletemarkers` and `getobsoletemarkers`
methods dedicated to send and receive obsolete markers.
- two new commands with the same names are added to the wire protocol. None of
them use stream logic because current markers reader is not able to use stream
anyway yet.
Error management is minimal and need later improvement.
The protocol is still to exchange everything. A discovery method may be added
in the future for smarter exchanges. `getobsoletemarkers` will then take an
argument to not return all marker.
diff --git a/mercurial/localrepo.py b/mercurial/localrepo.py
--- a/mercurial/localrepo.py
+++ b/mercurial/localrepo.py
@@ -8,7 +8,7 @@
from i18n import _
import peer, changegroup, subrepo, discovery, pushkey, obsolete
import changelog, dirstate, filelog, manifest, context, bookmarks, phases
-import lock, transaction, store, encoding, base85
+import lock, transaction, store, encoding
import scmutil, util, extensions, hook, error, revset
import match as matchmod
import merge as mergemod
@@ -23,7 +23,8 @@
def join(self, obj, fname):
return obj.sjoin(fname)
-MODERNCAPS = set(('lookup', 'branchmap', 'pushkey', 'known', 'getbundle'))
+MODERNCAPS = set(('lookup', 'branchmap', 'pushkey', 'known', 'getbundle',
+ 'dumpobsolete'))
LEGACYCAPS = MODERNCAPS.union(set(['changegroupsubset']))
class localpeer(peer.peerrepository):
@@ -86,6 +87,23 @@
'''used to test argument passing over the wire'''
return "%s %s %s %s %s" % (one, two, three, four, five)
+ def addobsoletemarkers(self, markers):
+ lock = self._repo.lock()
+ try:
+ tr = self._repo.transaction('pushed-obsolete-marker')
+ try:
+ new = self._repo.obsstore.add(tr, markers)
+ tr.close()
+ return new
+ finally:
+ tr.release()
+ finally:
+ lock.release()
+
+ def getobsoletemarkers(self):
+ return iter(self._repo.obsstore)
+
+
class locallegacypeer(localpeer):
'''peer extension which implements legacy methods too; used for tests with
restricted capabilities'''
@@ -1765,19 +1783,18 @@
# should be seen as public
phases.advanceboundary(self, phases.public, subset)
- remoteobs = remote.listkeys('obsolete')
- if 'dump' in remoteobs:
- if tr is None:
- tr = self.transaction(trname)
- data = base85.b85decode(remoteobs['dump'])
- self.obsstore.mergemarkers(tr, data)
+ if remote.capable('dumpobsolete'):
+ markers = list(remote.getobsoletemarkers())
+ if markers:
+ if tr is None:
+ tr = self.transaction(trname)
+ self.obsstore.add(tr, markers)
if tr is not None:
tr.close()
finally:
if tr is not None:
tr.release()
lock.release()
-
return result
def checkpush(self, force, revs):
@@ -1933,11 +1950,9 @@
if not r:
self.ui.warn(_('updating %s to public failed!\n')
% newremotehead)
- if ('obsolete' in remote.listkeys('namespaces')
- and self.obsstore):
- data = self.listkeys('obsolete')['dump']
- r = remote.pushkey('obsolete', 'dump', '', data)
- if not r:
+ if remote.capable('dumpobsolete') and self.obsstore:
+ oret = remote.addobsoletemarkers(self.obsstore)
+ if oret == -1:
self.ui.warn(_('failed to push obsolete markers!\n'))
finally:
if lock is not None:
diff --git a/mercurial/obsolete.py b/mercurial/obsolete.py
--- a/mercurial/obsolete.py
+++ b/mercurial/obsolete.py
@@ -52,7 +52,7 @@
cannot contain '\0'.
"""
import os, struct
-from mercurial import util, base85
+from mercurial import util
from i18n import _
_pack = struct.pack
@@ -67,7 +67,7 @@
_fmfsize = struct.calcsize(_fmfixed)
_fnodesize = struct.calcsize(_fmnode)
-def _readmarkers(data):
+def readmarkers(data):
"""Read and enumerate markers from raw data"""
off = 0
diskversion = _unpack('>B', data[off:off + 1])[0]
@@ -164,7 +164,7 @@
self.sopener = sopener
data = sopener.tryread('obsstore')
if data:
- self._load(_readmarkers(data))
+ self._load(readmarkers(data))
def __iter__(self):
return iter(self._all)
@@ -207,7 +207,7 @@
offset = f.tell()
transaction.add('obsstore', offset)
# offset == 0: new file - add the version header
- for bytes in _encodemarkers(new, offset == 0):
+ for bytes in encodemarkers(new, offset == 0):
f.write(bytes)
finally:
# XXX: f.close() == filecache invalidation == obsstore rebuilt.
@@ -216,10 +216,6 @@
self._load(new)
return len(new)
- def mergemarkers(self, transation, data):
- markers = _readmarkers(data)
- self.add(transation, markers)
-
def _load(self, markers):
for mark in markers:
self._all.append(mark)
@@ -228,7 +224,7 @@
for suc in sucs:
self.successors.setdefault(suc, set()).add(mark)
-def _encodemarkers(markers, addheader=False):
+def encodemarkers(markers, addheader=False):
# Kept separate from flushmarkers(), it will be reused for
# markers exchange.
if addheader:
@@ -242,34 +238,6 @@
yield _pack(format, *data)
yield metadata
-def listmarkers(repo):
- """List markers over pushkey"""
- if not repo.obsstore:
- return {}
- markers = _encodemarkers(repo.obsstore, True)
- return {'dump': base85.b85encode(''.join(markers))}
-
-def pushmarker(repo, key, old, new):
- """Push markers over pushkey"""
- if key != 'dump':
- repo.ui.warn(_('unknown key: %r') % key)
- return 0
- if old:
- repo.ui.warn(_('unexpected old value') % key)
- return 0
- data = base85.b85decode(new)
- lock = repo.lock()
- try:
- tr = repo.transaction('pushkey: obsolete markers')
- try:
- repo.obsstore.mergemarkers(tr, data)
- tr.close()
- return 1
- finally:
- tr.release()
- finally:
- lock.release()
-
def allmarkers(repo):
"""all obsolete markers known in a repository"""
for markerdata in repo.obsstore:
diff --git a/mercurial/pushkey.py b/mercurial/pushkey.py
--- a/mercurial/pushkey.py
+++ b/mercurial/pushkey.py
@@ -5,7 +5,7 @@
# This software may be used and distributed according to the terms of the
# GNU General Public License version 2 or any later version.
-import bookmarks, phases, obsolete
+import bookmarks, phases
def _nslist(repo):
n = {}
@@ -16,7 +16,6 @@
_namespaces = {"namespaces": (lambda *x: False, _nslist),
"bookmarks": (bookmarks.pushbookmark, bookmarks.listbookmarks),
"phases": (phases.pushphase, phases.listphases),
- "obsolete": (obsolete.pushmarker, obsolete.listmarkers),
}
def register(namespace, pushkey, listkeys):
diff --git a/mercurial/wireproto.py b/mercurial/wireproto.py
--- a/mercurial/wireproto.py
+++ b/mercurial/wireproto.py
@@ -10,7 +10,7 @@
from node import bin, hex
import changegroup as changegroupmod
import peer, error, encoding, util, store
-import phases
+import phases, obsolete
# abstract batching support
@@ -290,6 +290,23 @@
f = self._callstream("getbundle", **opts)
return changegroupmod.unbundle10(self._decompress(f), 'UN')
+ def addobsoletemarkers(self, markers):
+ self.requirecap('dumpobsolete', _('exchange obsolete marker'))
+ markersstream = obsolete.encodemarkers(markers, True)
+ data = ''.join(markersstream)
+ #data = '-' * 1665485
+ r = self._call("addobsoletemarkers", data=data)
+ ret, output = r.split('\n', 1)
+ return int(ret)
+
+ def getobsoletemarkers(self):
+ data = self._call("getobsoletemarkers")
+ markers = ()
+ if data:
+ markers = list(obsolete.readmarkers(data))
+ return markers
+
+
def unbundle(self, cg, heads, source):
'''Send cg (a readable file-like object representing the
changegroup to push, typically a chunkbuffer object) to the
@@ -414,7 +431,7 @@
def capabilities(repo, proto):
caps = ('lookup changegroupsubset branchmap pushkey known getbundle '
- 'unbundlehash batch').split()
+ 'unbundlehash batch dumpobsolete').split()
if _allowstream(repo.ui):
if repo.ui.configbool('server', 'preferuncompressed', False):
caps.append('stream-preferred')
@@ -596,6 +613,43 @@
fp.close()
os.unlink(tempname)
+
+def getobsoletemarkers(repo, proto):
+ markersstream = ''
+ if repo.obsstore:
+ markersstream = ''.join(obsolete.encodemarkers(repo.obsstore, True))
+ return markersstream
+
+
+def addobsoletemarkers(repo, proto):
+ fd, tempname = tempfile.mkstemp(prefix='hg-obsolete-markers-')
+ fp = os.fdopen(fd, 'wb+')
+ r = -1
+ proto.redirect()
+ try:
+ proto.getfile(fp)
+ fp.seek(0)
+ data = fp.read()
+ markers = list(obsolete.readmarkers(data))
+ if markers:
+ lock = repo.lock()
+ try:
+ tr = repo.transaction('babar')
+ try:
+ new = repo.obsstore.add(tr, markers)
+ tr.close()
+ finally:
+ tr.release()
+ finally:
+ lock.release()
+ r = new
+ else:
+ r = 0
+ finally:
+ fp.close()
+ os.unlink(tempname)
+ return pushres(r)
+
commands = {
'batch': (batch, 'cmds *'),
'between': (between, 'pairs'),
@@ -606,6 +660,8 @@
'changegroupsubset': (changegroupsubset, 'bases heads'),
'debugwireargs': (debugwireargs, 'one two *'),
'getbundle': (getbundle, '*'),
+ 'getobsoletemarkers': (getobsoletemarkers, ''),
+ 'addobsoletemarkers': (addobsoletemarkers, ''),
'heads': (heads, ''),
'hello': (hello, ''),
'known': (known, 'nodes *'),
diff --git a/tests/test-bookmarks-pushpull.t b/tests/test-bookmarks-pushpull.t
--- a/tests/test-bookmarks-pushpull.t
+++ b/tests/test-bookmarks-pushpull.t
@@ -40,7 +40,6 @@
bookmarks
phases
namespaces
- obsolete
$ hg debugpushkey ../a bookmarks
Y 4e3505fd95835d721066b76e75dbb8cc554d7f77
X 4e3505fd95835d721066b76e75dbb8cc554d7f77
@@ -215,7 +214,6 @@
bookmarks
phases
namespaces
- obsolete
$ hg debugpushkey http://localhost:$HGPORT/ bookmarks
Y 4efff6d98829d9c824c621afd6e3f01865f5439f
foobar 9b140be1080824d768c5a4691a564088eede71f9
diff --git a/tests/test-hgweb-commands.t b/tests/test-hgweb-commands.t
--- a/tests/test-hgweb-commands.t
+++ b/tests/test-hgweb-commands.t
@@ -1102,7 +1102,7 @@
$ "$TESTDIR/get-with-headers.py" 127.0.0.1:$HGPORT '?cmd=capabilities'; echo
200 Script output follows
- lookup changegroupsubset branchmap pushkey known getbundle unbundlehash batch unbundle=HG10GZ,HG10BZ,HG10UN httpheader=1024
+ lookup changegroupsubset branchmap pushkey known getbundle unbundlehash batch dumpobsolete unbundle=HG10GZ,HG10BZ,HG10UN httpheader=1024
heads
@@ -1279,7 +1279,7 @@
$ "$TESTDIR/get-with-headers.py" 127.0.0.1:$HGPORT '?cmd=capabilities'; echo
200 Script output follows
- lookup changegroupsubset branchmap pushkey known getbundle unbundlehash batch stream-preferred stream unbundle=HG10GZ,HG10BZ,HG10UN httpheader=1024
+ lookup changegroupsubset branchmap pushkey known getbundle unbundlehash batch dumpobsolete stream-preferred stream unbundle=HG10GZ,HG10BZ,HG10UN httpheader=1024
heads
diff --git a/tests/test-hook.t b/tests/test-hook.t
--- a/tests/test-hook.t
+++ b/tests/test-hook.t
@@ -198,7 +198,6 @@
listkeys hook: HG_NAMESPACE=bookmarks HG_VALUES={'bar': '0000000000000000000000000000000000000000', 'foo': '0000000000000000000000000000000000000000'}
no changes found
listkeys hook: HG_NAMESPACE=phases HG_VALUES={'cb9a9f314b8b07ba71012fcdbc544b5a4d82ff5b': '1', 'publishing': 'True'}
- listkeys hook: HG_NAMESPACE=obsolete HG_VALUES={}
listkeys hook: HG_NAMESPACE=bookmarks HG_VALUES={'bar': '0000000000000000000000000000000000000000', 'foo': '0000000000000000000000000000000000000000'}
adding remote bookmark bar
importing bookmark bar
@@ -214,7 +213,6 @@
searching for changes
no changes found
listkeys hook: HG_NAMESPACE=phases HG_VALUES={'cb9a9f314b8b07ba71012fcdbc544b5a4d82ff5b': '1', 'publishing': 'True'}
- listkeys hook: HG_NAMESPACE=namespaces HG_VALUES={'bookmarks': '', 'namespaces': '', 'obsolete': '', 'phases': ''}
listkeys hook: HG_NAMESPACE=bookmarks HG_VALUES={'bar': '0000000000000000000000000000000000000000', 'foo': '0000000000000000000000000000000000000000'}
listkeys hook: HG_NAMESPACE=bookmarks HG_VALUES={'bar': '0000000000000000000000000000000000000000', 'foo': '0000000000000000000000000000000000000000'}
exporting bookmark baz
diff --git a/tests/test-http-proxy.t b/tests/test-http-proxy.t
--- a/tests/test-http-proxy.t
+++ b/tests/test-http-proxy.t
@@ -105,24 +105,24 @@
* - - [*] "GET http://localhost:$HGPORT/?cmd=batch HTTP/1.1" - - x-hgarg-1:cmds=heads+%3Bknown+nodes%3D (glob)
* - - [*] "GET http://localhost:$HGPORT/?cmd=getbundle HTTP/1.1" - - x-hgarg-1:common=0000000000000000000000000000000000000000&heads=83180e7845de420a1bb46896fd5fe05294f8d629 (glob)
* - - [*] "GET http://localhost:$HGPORT/?cmd=listkeys HTTP/1.1" - - x-hgarg-1:namespace=phases (glob)
- * - - [*] "GET http://localhost:$HGPORT/?cmd=listkeys HTTP/1.1" - - x-hgarg-1:namespace=obsolete (glob)
+ * - - [*] "GET http://localhost:$HGPORT/?cmd=getobsoletemarkers HTTP/1.1" - - (glob)
* - - [*] "GET http://localhost:$HGPORT/?cmd=listkeys HTTP/1.1" - - x-hgarg-1:namespace=bookmarks (glob)
* - - [*] "GET http://localhost:$HGPORT/?cmd=capabilities HTTP/1.1" - - (glob)
* - - [*] "GET http://localhost:$HGPORT/?cmd=batch HTTP/1.1" - - x-hgarg-1:cmds=heads+%3Bknown+nodes%3D (glob)
* - - [*] "GET http://localhost:$HGPORT/?cmd=getbundle HTTP/1.1" - - x-hgarg-1:common=0000000000000000000000000000000000000000&heads=83180e7845de420a1bb46896fd5fe05294f8d629 (glob)
* - - [*] "GET http://localhost:$HGPORT/?cmd=listkeys HTTP/1.1" - - x-hgarg-1:namespace=phases (glob)
- * - - [*] "GET http://localhost:$HGPORT/?cmd=listkeys HTTP/1.1" - - x-hgarg-1:namespace=obsolete (glob)
+ * - - [*] "GET http://localhost:$HGPORT/?cmd=getobsoletemarkers HTTP/1.1" - - (glob)
* - - [*] "GET http://localhost:$HGPORT/?cmd=listkeys HTTP/1.1" - - x-hgarg-1:namespace=bookmarks (glob)
* - - [*] "GET http://localhost:$HGPORT/?cmd=capabilities HTTP/1.1" - - (glob)
* - - [*] "GET http://localhost:$HGPORT/?cmd=batch HTTP/1.1" - - x-hgarg-1:cmds=heads+%3Bknown+nodes%3D (glob)
* - - [*] "GET http://localhost:$HGPORT/?cmd=getbundle HTTP/1.1" - - x-hgarg-1:common=0000000000000000000000000000000000000000&heads=83180e7845de420a1bb46896fd5fe05294f8d629 (glob)
* - - [*] "GET http://localhost:$HGPORT/?cmd=listkeys HTTP/1.1" - - x-hgarg-1:namespace=phases (glob)
- * - - [*] "GET http://localhost:$HGPORT/?cmd=listkeys HTTP/1.1" - - x-hgarg-1:namespace=obsolete (glob)
+ * - - [*] "GET http://localhost:$HGPORT/?cmd=getobsoletemarkers HTTP/1.1" - - (glob)
* - - [*] "GET http://localhost:$HGPORT/?cmd=listkeys HTTP/1.1" - - x-hgarg-1:namespace=bookmarks (glob)
* - - [*] "GET http://localhost:$HGPORT/?cmd=capabilities HTTP/1.1" - - (glob)
* - - [*] "GET http://localhost:$HGPORT/?cmd=batch HTTP/1.1" - - x-hgarg-1:cmds=heads+%3Bknown+nodes%3D (glob)
* - - [*] "GET http://localhost:$HGPORT/?cmd=getbundle HTTP/1.1" - - x-hgarg-1:common=0000000000000000000000000000000000000000&heads=83180e7845de420a1bb46896fd5fe05294f8d629 (glob)
* - - [*] "GET http://localhost:$HGPORT/?cmd=listkeys HTTP/1.1" - - x-hgarg-1:namespace=phases (glob)
- * - - [*] "GET http://localhost:$HGPORT/?cmd=listkeys HTTP/1.1" - - x-hgarg-1:namespace=obsolete (glob)
+ * - - [*] "GET http://localhost:$HGPORT/?cmd=getobsoletemarkers HTTP/1.1" - - (glob)
* - - [*] "GET http://localhost:$HGPORT/?cmd=listkeys HTTP/1.1" - - x-hgarg-1:namespace=bookmarks (glob)
diff --git a/tests/test-obsolete.t b/tests/test-obsolete.t
--- a/tests/test-obsolete.t
+++ b/tests/test-obsolete.t
@@ -251,9 +251,60 @@
1337133713371337133713371337133713371337 5601fb93a350734d935195fee37f4054c529ff39 0 {'date': '1339 0', 'user': 'test'}
2448244824482448244824482448244824482448 1339133913391339133913391339133913391339 0 {'date': '1339 0', 'user': 'test'}
+
+
+Test exchange over the wire
+
+ $ cat >> $HGRCPATH << EOF
+ > [web]
+ > allow_push=*
+ > push_ssl=False
+ > EOF
+
+pull
+
+ $ hg init ../http-push-target
+ $ hg -R ../http-push-target serve -p $HGPORT -d --pid-file=../push-t.pid -E ../push-t.log
+ $ cat ../push-t.pid >> $DAEMON_PIDS
+ $ hg push -f http://localhost:$HGPORT/ # force because we push unstable
+ pushing to http://localhost:$HGPORT/
+ searching for changes
+ remote: adding changesets
+ remote: adding manifests
+ remote: adding file changes
+ remote: added 6 changesets with 6 changes to 6 files (+3 heads)
+ $ cat ../push-t.log
+ $ cd ..
+ $ hg -R http-push-target debugobsolete
+ 2448244824482448244824482448244824482448 1339133913391339133913391339133913391339 0 {'date': '1339 0', 'user': 'test'}
+ 245bde4270cd1072a27757984f9cda8ba26f08ca cdbce2fbb16313928851e97e0d85413f3f7eb77f 0 {'date': '56 12', 'user': 'test'}
+ cdbce2fbb16313928851e97e0d85413f3f7eb77f ca819180edb99ed25ceafb3e9584ac287e240b00 0 {'date': '1337 0', 'user': 'test'}
+ ca819180edb99ed25ceafb3e9584ac287e240b00 1337133713371337133713371337133713371337 0 {'date': '1338 0', 'user': 'test'}
+ 1337133713371337133713371337133713371337 5601fb93a350734d935195fee37f4054c529ff39 0 {'date': '1339 0', 'user': 'test'}
+ $ hg init http-pull-dest
+ $ hg -R http-pull-dest pull http://localhost:$HGPORT/
+ pulling from http://localhost:$HGPORT/
+ requesting all changes
+ adding changesets
+ adding manifests
+ adding file changes
+ added 6 changesets with 6 changes to 6 files (+3 heads)
+ (run 'hg heads' to see heads, 'hg merge' to merge)
+ $ kill `cat push-t.pid`
+ $ cat push-t.log
+ $ hg -R http-pull-dest debugobsolete
+ 2448244824482448244824482448244824482448 1339133913391339133913391339133913391339 0 {'date': '1339 0', 'user': 'test'}
+ 245bde4270cd1072a27757984f9cda8ba26f08ca cdbce2fbb16313928851e97e0d85413f3f7eb77f 0 {'date': '56 12', 'user': 'test'}
+ cdbce2fbb16313928851e97e0d85413f3f7eb77f ca819180edb99ed25ceafb3e9584ac287e240b00 0 {'date': '1337 0', 'user': 'test'}
+ ca819180edb99ed25ceafb3e9584ac287e240b00 1337133713371337133713371337133713371337 0 {'date': '1338 0', 'user': 'test'}
+ 1337133713371337133713371337133713371337 5601fb93a350734d935195fee37f4054c529ff39 0 {'date': '1339 0', 'user': 'test'}
+
+
detect outgoing obsolete and unstable
---------------------------------------
+ $ cd tmpe/
+
$ hg glog
o changeset: 5:5601fb93a350
| tag: tip
diff --git a/tests/test-ssh.t b/tests/test-ssh.t
--- a/tests/test-ssh.t
+++ b/tests/test-ssh.t
@@ -167,7 +167,6 @@
bookmarks
phases
namespaces
- obsolete
$ hg book foo -r 0
$ hg out -B
comparing with ssh://user@dummy/remote
More information about the Mercurial-devel
mailing list