[PATCH] wireproto: use base64 instead of hex for known/getbundle

Steven Brown stevengbrown at gmail.com
Mon May 2 10:31:34 CDT 2011 

On 2 May 2011 22:37, Peter Arrenbrecht <peter.arrenbrecht at gmail.com> wrote:
> # HG changeset patch
> # User Peter Arrenbrecht <peter.arrenbrecht at gmail.com>
> # Date 1304346315 -7200
> wireproto: use base64 instead of hex for known/getbundle
>
> Saves a couple of bytes per node id being sent across the
> wire. Introduces node.to/fromb64 and wireproto.en/decodelistb64.
>
> diff --git a/mercurial/node.py b/mercurial/node.py
> --- a/mercurial/node.py
> +++ b/mercurial/node.py
> @@ -5,7 +5,7 @@
>  # This software may be used and distributed according to the terms of the
>  # GNU General Public License version 2 or any later version.
>
> -import binascii
> +import binascii, base64
>
>  nullrev = -1
>  nullid = "\0" * 20
> @@ -14,5 +14,8 @@
>  hex = binascii.hexlify
>  bin = binascii.unhexlify
>
> +tob64 = base64.urlsafe_b64encode
> +fromb64 = base64.urlsafe_b64decode
> +
>  def short(node):
>     return hex(node[:6])
> diff --git a/mercurial/wireproto.py b/mercurial/wireproto.py
> --- a/mercurial/wireproto.py
> +++ b/mercurial/wireproto.py
> @@ -7,7 +7,7 @@
>
>  import urllib, tempfile, os, sys
>  from i18n import _
> -from node import bin, hex
> +from node import bin, hex, tob64, fromb64
>  import changegroup as changegroupmod
>  import repo, error, encoding, util, store
>  import pushkey as pushkeymod
> @@ -22,6 +22,14 @@
>  def encodelist(l, sep=' '):
>     return sep.join(map(hex, l))
>
> +def decodelistb64(l, sep=' '):
> +    if l:
> +        return map(fromb64, l.split(sep))
> +    return []
> +
> +def encodelistb64(l, sep=' '):
> +    return sep.join(map(tob64, l))
> +
>  # client side
>
>  class wirerepository(repo.repository):
> @@ -41,7 +49,7 @@
>             self._abort(error.ResponseError(_("unexpected response:"), d))
>
>     def known(self, nodes):
> -        n = encodelist(nodes)
> +        n = encodelistb64(nodes)
>         d = self._call("known", nodes=n)
>         try:
>             return [bool(int(f)) for f in d]
> @@ -127,9 +135,9 @@
>         self.requirecap('getbundle', _('look up remote changes'))
>         opts = {}
>         if heads is not None:
> -            opts['heads'] = encodelist(heads)
> +            opts['heads'] = encodelistb64(heads)
>         if common is not None:
> -            opts['common'] = encodelist(common)
> +            opts['common'] = encodelistb64(common)
>         f = self._callstream("getbundle", **opts)
>         return changegroupmod.unbundle10(self._decompress(f), 'UN')
>
> @@ -255,7 +263,7 @@
>  def getbundle(repo, proto, others):
>     opts = options('getbundle', ['heads', 'common'], others)
>     for k, v in opts.iteritems():
> -        opts[k] = decodelist(v)
> +        opts[k] = decodelistb64(v)
>     cg = repo.getbundle('serve', **opts)
>     return streamres(proto.groupchunks(cg))
>
> @@ -289,7 +297,7 @@
>     return "%s %s\n" % (success, r)
>
>  def known(repo, proto, nodes):
> -    return ''.join(b and "1" or "0" for b in repo.known(decodelist(nodes)))
> +    return ''.join(b and "1" or "0" for b in repo.known(decodelistb64(nodes)))
>
>  def pushkey(repo, proto, namespace, key, old, new):
>     # compatibility with pre-1.8 clients which were accidentally
> diff --git a/tests/test-getbundle.t b/tests/test-getbundle.t
> --- a/tests/test-getbundle.t
> +++ b/tests/test-getbundle.t
> @@ -247,7 +247,7 @@
>   * - - [*] "GET /?cmd=capabilities HTTP/1.1" 200 - (glob)
>   * - - [*] "GET /?cmd=getbundle HTTP/1.1" 200 - (glob)
>   * - - [*] "GET /?cmd=capabilities HTTP/1.1" 200 - (glob)
> -  * - - [*] "GET /?cmd=getbundle HTTP/1.1" 200 - x-hgarg-1:common=c1818a9f5977dd4139a48f93f5425c67d44a9368+ea919464b16e003894c48b6cb68df3cd9411b544&heads=6b57ee934bb2996050540f84cdfc8dcad1e7267d+2114148793524fd045998f71a45b0aaf139f752b (glob)
> +  * - - [*] "GET /?cmd=getbundle HTTP/1.1" 200 - x-hgarg-1:common=wYGKn1l33UE5pI-T9UJcZ9RKk2g%3D+6pGUZLFuADiUxItsto3zzZQRtUQ%3D&heads=a1fuk0uymWBQVA-EzfyNytHnJn0%3D+IRQUh5NST9BFmY9xpFsKrxOfdSs%3D (glob)
>
>   $ cat error.log
>
> diff --git a/tests/test-http-proxy.t b/tests/test-http-proxy.t
> --- a/tests/test-http-proxy.t
> +++ b/tests/test-http-proxy.t
> @@ -103,18 +103,18 @@
>   * - - [*] "GET http://localhost:$HGPORT/?cmd=listkeys HTTP/1.1" - - x-hgarg-1:namespace=bookmarks (glob)
>   * - - [*] "GET http://localhost:$HGPORT/?cmd=capabilities HTTP/1.1" - - (glob)
>   * - - [*] "GET http://localhost:$HGPORT/?cmd=heads HTTP/1.1" - - (glob)
> -  * - - [*] "GET http://localhost:$HGPORT/?cmd=getbundle HTTP/1.1" - - x-hgarg-1:common=0000000000000000000000000000000000000000&heads=83180e7845de420a1bb46896fd5fe05294f8d629 (glob)
> +  * - - [*] "GET http://localhost:$HGPORT/?cmd=getbundle HTTP/1.1" - - x-hgarg-1:common=AAAAAAAAAAAAAAAAAAAAAAAAAAA%3D&heads=gxgOeEXeQgobtGiW_V_gUpT41ik%3D (glob)
>   * - - [*] "GET http://localhost:$HGPORT/?cmd=listkeys HTTP/1.1" - - x-hgarg-1:namespace=bookmarks (glob)
>   * - - [*] "GET http://localhost:$HGPORT/?cmd=capabilities HTTP/1.1" - - (glob)
>   * - - [*] "GET http://localhost:$HGPORT/?cmd=heads HTTP/1.1" - - (glob)
> -  * - - [*] "GET http://localhost:$HGPORT/?cmd=getbundle HTTP/1.1" - - x-hgarg-1:common=0000000000000000000000000000000000000000&heads=83180e7845de420a1bb46896fd5fe05294f8d629 (glob)
> +  * - - [*] "GET http://localhost:$HGPORT/?cmd=getbundle HTTP/1.1" - - x-hgarg-1:common=AAAAAAAAAAAAAAAAAAAAAAAAAAA%3D&heads=gxgOeEXeQgobtGiW_V_gUpT41ik%3D (glob)
>   * - - [*] "GET http://localhost:$HGPORT/?cmd=listkeys HTTP/1.1" - - x-hgarg-1:namespace=bookmarks (glob)
>   * - - [*] "GET http://localhost:$HGPORT/?cmd=capabilities HTTP/1.1" - - (glob)
>   * - - [*] "GET http://localhost:$HGPORT/?cmd=heads HTTP/1.1" - - (glob)
> -  * - - [*] "GET http://localhost:$HGPORT/?cmd=getbundle HTTP/1.1" - - x-hgarg-1:common=0000000000000000000000000000000000000000&heads=83180e7845de420a1bb46896fd5fe05294f8d629 (glob)
> +  * - - [*] "GET http://localhost:$HGPORT/?cmd=getbundle HTTP/1.1" - - x-hgarg-1:common=AAAAAAAAAAAAAAAAAAAAAAAAAAA%3D&heads=gxgOeEXeQgobtGiW_V_gUpT41ik%3D (glob)
>   * - - [*] "GET http://localhost:$HGPORT/?cmd=listkeys HTTP/1.1" - - x-hgarg-1:namespace=bookmarks (glob)
>   * - - [*] "GET http://localhost:$HGPORT/?cmd=capabilities HTTP/1.1" - - (glob)
>   * - - [*] "GET http://localhost:$HGPORT/?cmd=heads HTTP/1.1" - - (glob)
> -  * - - [*] "GET http://localhost:$HGPORT/?cmd=getbundle HTTP/1.1" - - x-hgarg-1:common=0000000000000000000000000000000000000000&heads=83180e7845de420a1bb46896fd5fe05294f8d629 (glob)
> +  * - - [*] "GET http://localhost:$HGPORT/?cmd=getbundle HTTP/1.1" - - x-hgarg-1:common=AAAAAAAAAAAAAAAAAAAAAAAAAAA%3D&heads=gxgOeEXeQgobtGiW_V_gUpT41ik%3D (glob)
>   * - - [*] "GET http://localhost:$HGPORT/?cmd=listkeys HTTP/1.1" - - x-hgarg-1:namespace=bookmarks (glob)
>
>

The node ids each end with '%3D' when sent over HTTP due to the base64
padding. Since the node ids have a fixed size, the padding isn't
needed.

How about this for node.py:

def to64(node):
    return base64.urlsafe_b64encode(node)[:-1]

def from64(node):
    return base64.urlsafe_b64decode(node + '=')

If you prefer, I can make this change in a follow-up patch.

- Steve

More information about the Mercurial-devel mailing list