D2061: sshpeer: initial definition and implementation of new SSH protocol

indygreg (Gregory Szorc) phabricator at mercurial-scm.org
Wed Feb 7 17:41:18 EST 2018


This revision was automatically updated to reflect the committed changes.
Closed by commit rHG48a3a9283f09: sshpeer: initial definition and implementation of new SSH protocol (authored by indygreg, committed by ).

REPOSITORY
  rHG Mercurial

CHANGES SINCE LAST UPDATE
  https://phab.mercurial-scm.org/D2061?vs=5255&id=5301

REVISION DETAIL
  https://phab.mercurial-scm.org/D2061

AFFECTED FILES
  mercurial/configitems.py
  mercurial/help/internals/wireprotocol.txt
  mercurial/sshpeer.py
  mercurial/wireprotoserver.py
  tests/sshprotoext.py
  tests/test-ssh-proto.t

CHANGE DETAILS

diff --git a/tests/test-ssh-proto.t b/tests/test-ssh-proto.t
--- a/tests/test-ssh-proto.t
+++ b/tests/test-ssh-proto.t
@@ -388,3 +388,107 @@
   0
   0
   0
+
+Send an upgrade request to a server that doesn't support that command
+
+  $ hg -R server serve --stdio << EOF
+  > upgrade 2e82ab3f-9ce3-4b4e-8f8c-6fd1c0e9e23a proto=irrelevant1%2Cirrelevant2
+  > hello
+  > between
+  > pairs 81
+  > 0000000000000000000000000000000000000000-0000000000000000000000000000000000000000
+  > EOF
+  0
+  384
+  capabilities: lookup changegroupsubset branchmap pushkey known getbundle unbundlehash batch streamreqs=generaldelta,revlogv1 $USUAL_BUNDLE2_CAPS_SERVER$ unbundle=HG10GZ,HG10BZ,HG10UN
+  1
+  
+
+  $ hg --config experimental.sshpeer.advertise-v2=true --debug debugpeer ssh://user@dummy/server
+  running * "*/tests/dummyssh" 'user at dummy' 'hg -R server serve --stdio' (glob)
+  sending upgrade request: * proto=exp-ssh-v2-0001 (glob)
+  devel-peer-request: hello
+  sending hello command
+  devel-peer-request: between
+  devel-peer-request:   pairs: 81 bytes
+  sending between command
+  remote: 0
+  remote: 384
+  remote: capabilities: lookup changegroupsubset branchmap pushkey known getbundle unbundlehash batch streamreqs=generaldelta,revlogv1 $USUAL_BUNDLE2_CAPS_SERVER$ unbundle=HG10GZ,HG10BZ,HG10UN
+  remote: 1
+  url: ssh://user@dummy/server
+  local: no
+  pushable: yes
+
+Send an upgrade request to a server that supports upgrade
+
+  $ SSHSERVERMODE=upgradev2 hg -R server serve --stdio << EOF
+  > upgrade this-is-some-token proto=exp-ssh-v2-0001
+  > hello
+  > between
+  > pairs 81
+  > 0000000000000000000000000000000000000000-0000000000000000000000000000000000000000
+  > EOF
+  upgraded this-is-some-token exp-ssh-v2-0001
+  383
+  capabilities: lookup changegroupsubset branchmap pushkey known getbundle unbundlehash batch streamreqs=generaldelta,revlogv1 $USUAL_BUNDLE2_CAPS_SERVER$ unbundle=HG10GZ,HG10BZ,HG10UN
+
+  $ SSHSERVERMODE=upgradev2 hg --config experimental.sshpeer.advertise-v2=true --debug debugpeer ssh://user@dummy/server
+  running * "*/tests/dummyssh" 'user at dummy' 'hg -R server serve --stdio' (glob)
+  sending upgrade request: * proto=exp-ssh-v2-0001 (glob)
+  devel-peer-request: hello
+  sending hello command
+  devel-peer-request: between
+  devel-peer-request:   pairs: 81 bytes
+  sending between command
+  protocol upgraded to exp-ssh-v2-0001
+  url: ssh://user@dummy/server
+  local: no
+  pushable: yes
+
+Verify the peer has capabilities
+
+  $ SSHSERVERMODE=upgradev2 hg --config experimental.sshpeer.advertise-v2=true --debug debugcapabilities ssh://user@dummy/server
+  running * "*/tests/dummyssh" 'user at dummy' 'hg -R server serve --stdio' (glob)
+  sending upgrade request: * proto=exp-ssh-v2-0001 (glob)
+  devel-peer-request: hello
+  sending hello command
+  devel-peer-request: between
+  devel-peer-request:   pairs: 81 bytes
+  sending between command
+  protocol upgraded to exp-ssh-v2-0001
+  Main capabilities:
+    batch
+    branchmap
+    $USUAL_BUNDLE2_CAPS_SERVER$
+    changegroupsubset
+    getbundle
+    known
+    lookup
+    pushkey
+    streamreqs=generaldelta,revlogv1
+    unbundle=HG10GZ,HG10BZ,HG10UN
+    unbundlehash
+  Bundle2 capabilities:
+    HG20
+    bookmarks
+    changegroup
+      01
+      02
+    digests
+      md5
+      sha1
+      sha512
+    error
+      abort
+      unsupportedcontent
+      pushraced
+      pushkey
+    hgtagsfnodes
+    listkeys
+    phases
+      heads
+    pushkey
+    remote-changegroup
+      http
+      https
diff --git a/tests/sshprotoext.py b/tests/sshprotoext.py
--- a/tests/sshprotoext.py
+++ b/tests/sshprotoext.py
@@ -53,6 +53,35 @@
 
         super(prehelloserver, self).serve_forever()
 
+class upgradev2server(wireprotoserver.sshserver):
+    """Tests behavior for clients that issue upgrade to version 2."""
+    def serve_forever(self):
+        name = wireprotoserver.SSHV2
+        l = self._fin.readline()
+        assert l.startswith(b'upgrade ')
+        token, caps = l[:-1].split(b' ')[1:]
+        assert caps == b'proto=%s' % name
+
+        # Filter hello and between requests.
+        l = self._fin.readline()
+        assert l == b'hello\n'
+        l = self._fin.readline()
+        assert l == b'between\n'
+        l = self._fin.readline()
+        assert l == 'pairs 81\n'
+        self._fin.read(81)
+
+        # Send the upgrade response.
+        self._fout.write(b'upgraded %s %s\n' % (token, name))
+        servercaps = wireproto.capabilities(self._repo, self)
+        rsp = b'capabilities: %s' % servercaps
+        self._fout.write(b'%d\n' % len(rsp))
+        self._fout.write(rsp)
+        self._fout.write(b'\n')
+        self._fout.flush()
+
+        super(upgradev2server, self).serve_forever()
+
 def performhandshake(orig, ui, stdin, stdout, stderr):
     """Wrapped version of sshpeer._performhandshake to send extra commands."""
     mode = ui.config(b'sshpeer', b'handshake-mode')
@@ -85,6 +114,8 @@
         wireprotoserver.sshserver = bannerserver
     elif servermode == b'no-hello':
         wireprotoserver.sshserver = prehelloserver
+    elif servermode == b'upgradev2':
+        wireprotoserver.sshserver = upgradev2server
     elif servermode:
         raise error.ProgrammingError(b'unknown server mode: %s' % servermode)
 
diff --git a/mercurial/wireprotoserver.py b/mercurial/wireprotoserver.py
--- a/mercurial/wireprotoserver.py
+++ b/mercurial/wireprotoserver.py
@@ -32,6 +32,12 @@
 HGTYPE2 = 'application/mercurial-0.2'
 HGERRTYPE = 'application/hg-error'
 
+# Names of the SSH protocol implementations.
+SSHV1 = 'ssh-v1'
+# This is advertised over the wire. Incremental the counter at the end
+# to reflect BC breakages.
+SSHV2 = 'exp-ssh-v2-0001'
+
 class abstractserverproto(object):
     """abstract class that summarizes the protocol API
 
diff --git a/mercurial/sshpeer.py b/mercurial/sshpeer.py
--- a/mercurial/sshpeer.py
+++ b/mercurial/sshpeer.py
@@ -8,13 +8,15 @@
 from __future__ import absolute_import
 
 import re
+import uuid
 
 from .i18n import _
 from . import (
     error,
     pycompat,
     util,
     wireproto,
+    wireprotoserver,
 )
 
 def _serverquote(s):
@@ -162,15 +164,24 @@
         hint = ui.config('ui', 'ssherrorhint')
         raise error.RepoError(msg, hint=hint)
 
-    # The handshake consists of sending 2 wire protocol commands:
-    # ``hello`` and ``between``.
+    # The handshake consists of sending wire protocol commands in reverse
+    # order of protocol implementation and then sniffing for a response
+    # to one of them.
+    #
+    # Those commands (from oldest to newest) are:
     #
-    # The ``hello`` command (which was introduced in Mercurial 0.9.1)
-    # instructs the server to advertise its capabilities.
+    # ``between``
+    #   Asks for the set of revisions between a pair of revisions. Command
+    #   present in all Mercurial server implementations.
     #
-    # The ``between`` command (which has existed in all Mercurial servers
-    # for as long as SSH support has existed), asks for the set of revisions
-    # between a pair of revisions.
+    # ``hello``
+    #   Instructs the server to advertise its capabilities. Introduced in
+    #   Mercurial 0.9.1.
+    #
+    # ``upgrade``
+    #   Requests upgrade from default transport protocol version 1 to
+    #   a newer version. Introduced in Mercurial 4.6 as an experimental
+    #   feature.
     #
     # The ``between`` command is issued with a request for the null
     # range. If the remote is a Mercurial server, this request will
@@ -186,6 +197,18 @@
     # RFC 822 like lines. Of these, the ``capabilities:`` line contains
     # the capabilities of the server.
     #
+    # The ``upgrade`` command isn't really a command in the traditional
+    # sense of version 1 of the transport because it isn't using the
+    # proper mechanism for formatting insteads: instead, it just encodes
+    # arguments on the line, delimited by spaces.
+    #
+    # The ``upgrade`` line looks like ``upgrade <token> <capabilities>``.
+    # If the server doesn't support protocol upgrades, it will reply to
+    # this line with ``0\n``. Otherwise, it emits an
+    # ``upgraded <token> <protocol>`` line to both stdout and stderr.
+    # Content immediately following this line describes additional
+    # protocol and server state.
+    #
     # In addition to the responses to our command requests, the server
     # may emit "banner" output on stdout. SSH servers are allowed to
     # print messages to stdout on login. Issuing commands on connection
@@ -195,6 +218,14 @@
 
     requestlog = ui.configbool('devel', 'debug.peer-request')
 
+    # Generate a random token to help identify responses to version 2
+    # upgrade request.
+    token = bytes(uuid.uuid4())
+    upgradecaps = [
+        ('proto', wireprotoserver.SSHV2),
+    ]
+    upgradecaps = util.urlreq.urlencode(upgradecaps)
+
     try:
         pairsarg = '%s-%s' % ('0' * 40, '0' * 40)
         handshake = [
@@ -204,6 +235,11 @@
             pairsarg,
         ]
 
+        # Request upgrade to version 2 if configured.
+        if ui.configbool('experimental', 'sshpeer.advertise-v2'):
+            ui.debug('sending upgrade request: %s %s\n' % (token, upgradecaps))
+            handshake.insert(0, 'upgrade %s %s\n' % (token, upgradecaps))
+
         if requestlog:
             ui.debug('devel-peer-request: hello\n')
         ui.debug('sending hello command\n')
@@ -217,12 +253,31 @@
     except IOError:
         badresponse()
 
+    # Assume version 1 of wire protocol by default.
+    protoname = wireprotoserver.SSHV1
+    reupgraded = re.compile(b'^upgraded %s (.*)$' % re.escape(token))
+
     lines = ['', 'dummy']
     max_noise = 500
     while lines[-1] and max_noise:
         try:
             l = stdout.readline()
             _forwardoutput(ui, stderr)
+
+            # Look for reply to protocol upgrade request. It has a token
+            # in it, so there should be no false positives.
+            m = reupgraded.match(l)
+            if m:
+                protoname = m.group(1)
+                ui.debug('protocol upgraded to %s\n' % protoname)
+                # If an upgrade was handled, the ``hello`` and ``between``
+                # requests are ignored. The next output belongs to the
+                # protocol, so stop scanning lines.
+                break
+
+            # Otherwise it could be a banner, ``0\n`` response if server
+            # doesn't support upgrade.
+
             if lines[-1] == '1\n' and l == '\n':
                 break
             if l:
@@ -235,20 +290,39 @@
         badresponse()
 
     caps = set()
-    for l in reversed(lines):
-        # Look for response to ``hello`` command. Scan from the back so
-        # we don't misinterpret banner output as the command reply.
-        if l.startswith('capabilities:'):
-            caps.update(l[:-1].split(':')[1].split())
-            break
 
-    # Error if we couldn't find a response to ``hello``. This could
-    # mean:
+    # For version 1, we should see a ``capabilities`` line in response to the
+    # ``hello`` command.
+    if protoname == wireprotoserver.SSHV1:
+        for l in reversed(lines):
+            # Look for response to ``hello`` command. Scan from the back so
+            # we don't misinterpret banner output as the command reply.
+            if l.startswith('capabilities:'):
+                caps.update(l[:-1].split(':')[1].split())
+                break
+    elif protoname == wireprotoserver.SSHV2:
+        # We see a line with number of bytes to follow and then a value
+        # looking like ``capabilities: *``.
+        line = stdout.readline()
+        try:
+            valuelen = int(line)
+        except ValueError:
+            badresponse()
+
+        capsline = stdout.read(valuelen)
+        if not capsline.startswith('capabilities: '):
+            badresponse()
+
+        caps.update(capsline.split(':')[1].split())
+        # Trailing newline.
+        stdout.read(1)
+
+    # Error if we couldn't find capabilities, this means:
     #
     # 1. Remote isn't a Mercurial server
     # 2. Remote is a <0.9.1 Mercurial server
     # 3. Remote is a future Mercurial server that dropped ``hello``
-    #    support.
+    #    and other attempted handshake mechanisms.
     if not caps:
         badresponse()
 
diff --git a/mercurial/help/internals/wireprotocol.txt b/mercurial/help/internals/wireprotocol.txt
--- a/mercurial/help/internals/wireprotocol.txt
+++ b/mercurial/help/internals/wireprotocol.txt
@@ -218,6 +218,95 @@
 after responses. In other words, the length of the response contains the
 trailing ``\n``.
 
+Clients supporting version 2 of the SSH transport send a line beginning
+with ``upgrade`` before the ``hello`` and ``between`` commands. The line
+(which isn't a well-formed command line because it doesn't consist of a
+single command name) serves to both communicate the client's intent to
+switch to transport version 2 (transports are version 1 by default) as
+well as to advertise the client's transport-level capabilities so the
+server may satisfy that request immediately.
+
+The upgrade line has the form:
+
+    upgrade <token> <transport capabilities>
+
+That is the literal string ``upgrade`` followed by a space, followed by
+a randomly generated string, followed by a space, followed by a string
+denoting the client's transport capabilities.
+
+The token can be anything. However, a random UUID is recommended. (Use
+of version 4 UUIDs is recommended because version 1 UUIDs can leak the
+client's MAC address.)
+
+The transport capabilities string is a URL/percent encoded string
+containing key-value pairs defining the client's transport-level
+capabilities. The following capabilities are defined:
+
+proto
+   A comma-delimited list of transport protocol versions the client
+   supports. e.g. ``ssh-v2``.
+
+If the server does not recognize the ``upgrade`` line, it should issue
+an empty response and continue processing the ``hello`` and ``between``
+commands. Here is an example handshake between a version 2 aware client
+and a non version 2 aware server:
+
+   c: upgrade 2e82ab3f-9ce3-4b4e-8f8c-6fd1c0e9e23a proto=ssh-v2
+   c: hello\n
+   c: between\n
+   c: pairs 81\n
+   c: 0000000000000000000000000000000000000000-0000000000000000000000000000000000000000
+   s: 0\n
+   s: 324\n
+   s: capabilities: lookup changegroupsubset branchmap pushkey known getbundle ...\n
+   s: 1\n
+   s: \n
+
+(The initial ``0\n`` line from the server indicates an empty response to
+the unknown ``upgrade ..`` command/line.)
+
+If the server recognizes the ``upgrade`` line and is willing to satisfy that
+upgrade request, it replies to with a payload of the following form:
+
+   upgraded <token> <transport name>\n
+
+This line is the literal string ``upgraded``, a space, the token that was
+specified by the client in its ``upgrade ...`` request line, a space, and the
+name of the transport protocol that was chosen by the server. The transport
+name MUST match one of the names the client specified in the ``proto`` field
+of its ``upgrade ...`` request line.
+
+If a server issues an ``upgraded`` response, it MUST also read and ignore
+the lines associated with the ``hello`` and ``between`` command requests
+that were issued by the server. It is assumed that the negotiated transport
+will respond with equivalent requested information following the transport
+handshake.
+
+All data following the ``\n`` terminating the ``upgraded`` line is the
+domain of the negotiated transport. It is common for the data immediately
+following to contain additional metadata about the state of the transport and
+the server. However, this isn't strictly speaking part of the transport
+handshake and isn't covered by this section.
+
+Here is an example handshake between a version 2 aware client and a version
+2 aware server:
+
+   c:  upgrade 2e82ab3f-9ce3-4b4e-8f8c-6fd1c0e9e23a proto=ssh-v2
+   c:  hello\n
+   c:  between\n
+   c:  pairs 81\n
+   c:  0000000000000000000000000000000000000000-0000000000000000000000000000000000000000
+   s: upgraded 2e82ab3f-9ce3-4b4e-8f8c-6fd1c0e9e23a ssh-v2\n
+   s: <additional transport specific data>
+
+The client-issued token that is echoed in the response provides a more
+resilient mechanism for differentiating *banner* output from Mercurial
+output. In version 1, properly formatted banner output could get confused
+for Mercurial server output. By submitting a randomly generated token
+that is then present in the response, the client can look for that token
+in response lines and have reasonable certainty that the line did not
+originate from a *banner* message.
+
 SSH Version 1 Transport
 -----------------------
 
@@ -281,6 +370,31 @@
 
 The server terminates if it receives an empty command (a ``\n`` character).
 
+SSH Version 2 Transport
+-----------------------
+
+**Experimental**
+
+Version 2 of the SSH transport behaves identically to version 1 of the SSH
+transport with the exception of handshake semantics. See above for how
+version 2 of the SSH transport is negotiated.
+
+Immediately following the ``upgraded`` line signaling a switch to version
+2 of the SSH protocol, the server automatically sends additional details
+about the capabilities of the remote server. This has the form:
+
+   <integer length of value>\n
+   capabilities: ...\n
+
+e.g.
+
+   s: upgraded 2e82ab3f-9ce3-4b4e-8f8c-6fd1c0e9e23a ssh-v2\n
+   s: 240\n
+   s: capabilities: known getbundle batch ...\n
+
+Following capabilities advertisement, the peers communicate using version
+1 of the SSH transport.
+
 Capabilities
 ============
 
diff --git a/mercurial/configitems.py b/mercurial/configitems.py
--- a/mercurial/configitems.py
+++ b/mercurial/configitems.py
@@ -574,6 +574,9 @@
 coreconfigitem('experimental', 'update.atomic-file',
     default=False,
 )
+coreconfigitem('experimental', 'sshpeer.advertise-v2',
+    default=False,
+)
 coreconfigitem('extensions', '.*',
     default=None,
     generic=True,



To: indygreg, #hg-reviewers, durin42
Cc: durin42, joerg.sonnenberger, mercurial-devel


More information about the Mercurial-devel mailing list