D3433: httppeer: detect redirect to URL without query string (issue5860)
indygreg (Gregory Szorc)
phabricator at mercurial-scm.org
Fri May 4 13:25:24 EDT 2018
This revision was automatically updated to reflect the committed changes.
Closed by commit rHG6169d95dce3b: httppeer: detect redirect to URL without query string (issue5860) (authored by indygreg, committed by ).
REPOSITORY
rHG Mercurial
CHANGES SINCE LAST UPDATE
https://phab.mercurial-scm.org/D3433?vs=8452&id=8453
REVISION DETAIL
https://phab.mercurial-scm.org/D3433
AFFECTED FILES
mercurial/httppeer.py
tests/test-http-protocol.t
CHANGE DETAILS
diff --git a/tests/test-http-protocol.t b/tests/test-http-protocol.t
--- a/tests/test-http-protocol.t
+++ b/tests/test-http-protocol.t
@@ -333,3 +333,394 @@
response: [b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00']
$ killdaemons.py
+
+HTTP client follows HTTP redirect on handshake to new repo
+
+ $ cd $TESTTMP
+
+ $ hg init redirector
+ $ hg init redirected
+ $ cd redirected
+ $ touch foo
+ $ hg -q commit -A -m initial
+ $ cd ..
+
+ $ cat > paths.conf << EOF
+ > [paths]
+ > / = $TESTTMP/*
+ > EOF
+
+ $ cat > redirectext.py << EOF
+ > from mercurial import extensions, wireprotoserver
+ > def wrappedcallhttp(orig, repo, req, res, proto, cmd):
+ > path = req.advertisedurl[len(req.advertisedbaseurl):]
+ > if not path.startswith(b'/redirector'):
+ > return orig(repo, req, res, proto, cmd)
+ > relpath = path[len(b'/redirector'):]
+ > res.status = b'301 Redirect'
+ > newurl = b'%s/redirected%s' % (req.baseurl, relpath)
+ > if not repo.ui.configbool('testing', 'redirectqs', True) and b'?' in newurl:
+ > newurl = newurl[0:newurl.index(b'?')]
+ > res.headers[b'Location'] = newurl
+ > res.headers[b'Content-Type'] = b'text/plain'
+ > res.setbodybytes(b'redirected')
+ > return True
+ >
+ > extensions.wrapfunction(wireprotoserver, '_callhttp', wrappedcallhttp)
+ > EOF
+
+ $ hg --config extensions.redirect=$TESTTMP/redirectext.py \
+ > --config server.compressionengines=zlib \
+ > serve --web-conf paths.conf --pid-file hg.pid -p $HGPORT -d
+ $ cat hg.pid > $DAEMON_PIDS
+
+Verify our HTTP 301 is served properly
+
+ $ hg --verbose debugwireproto --peer raw http://$LOCALIP:$HGPORT << EOF
+ > httprequest GET /redirector?cmd=capabilities
+ > user-agent: test
+ > EOF
+ using raw connection to peer
+ s> GET /redirector?cmd=capabilities HTTP/1.1\r\n
+ s> Accept-Encoding: identity\r\n
+ s> user-agent: test\r\n
+ s> host: $LOCALIP:$HGPORT\r\n (glob)
+ s> \r\n
+ s> makefile('rb', None)
+ s> HTTP/1.1 301 Redirect\r\n
+ s> Server: testing stub value\r\n
+ s> Date: $HTTP_DATE$\r\n
+ s> Location: http://$LOCALIP:$HGPORT/redirected?cmd=capabilities\r\n (glob)
+ s> Content-Type: text/plain\r\n
+ s> Content-Length: 10\r\n
+ s> \r\n
+ s> redirected
+ s> GET /redirected?cmd=capabilities HTTP/1.1\r\n
+ s> Accept-Encoding: identity\r\n
+ s> user-agent: test\r\n
+ s> host: $LOCALIP:$HGPORT\r\n (glob)
+ s> \r\n
+ s> makefile('rb', None)
+ s> HTTP/1.1 200 Script output follows\r\n
+ s> Server: testing stub value\r\n
+ s> Date: $HTTP_DATE$\r\n
+ s> Content-Type: application/mercurial-0.1\r\n
+ s> Content-Length: 453\r\n
+ s> \r\n
+ s> batch branchmap $USUAL_BUNDLE2_CAPS_SERVER$ changegroupsubset compression=$BUNDLE2_COMPRESSIONS$ getbundle httpheader=1024 httpmediatype=0.1rx,0.1tx,0.2tx known lookup pushkey streamreqs=generaldelta,revlogv1 unbundle=HG10GZ,HG10BZ,HG10UN unbundlehash
+
+Test with the HTTP peer
+
+ $ hg --verbose debugwireproto http://$LOCALIP:$HGPORT/redirector << EOF
+ > command heads
+ > EOF
+ s> GET /redirector?cmd=capabilities HTTP/1.1\r\n
+ s> Accept-Encoding: identity\r\n
+ s> accept: application/mercurial-0.1\r\n
+ s> host: $LOCALIP:$HGPORT\r\n (glob)
+ s> user-agent: Mercurial debugwireproto\r\n
+ s> \r\n
+ s> makefile('rb', None)
+ s> HTTP/1.1 301 Redirect\r\n
+ s> Server: testing stub value\r\n
+ s> Date: $HTTP_DATE$\r\n
+ s> Location: http://$LOCALIP:$HGPORT/redirected?cmd=capabilities\r\n (glob)
+ s> Content-Type: text/plain\r\n
+ s> Content-Length: 10\r\n
+ s> \r\n
+ s> redirected
+ s> GET /redirected?cmd=capabilities HTTP/1.1\r\n
+ s> Accept-Encoding: identity\r\n
+ s> accept: application/mercurial-0.1\r\n
+ s> host: $LOCALIP:$HGPORT\r\n (glob)
+ s> user-agent: Mercurial debugwireproto\r\n
+ s> \r\n
+ s> makefile('rb', None)
+ s> HTTP/1.1 200 Script output follows\r\n
+ s> Server: testing stub value\r\n
+ s> Date: $HTTP_DATE$\r\n
+ s> Content-Type: application/mercurial-0.1\r\n
+ s> Content-Length: 453\r\n
+ s> \r\n
+ real URL is http://$LOCALIP:$HGPORT/redirected (glob)
+ s> batch branchmap $USUAL_BUNDLE2_CAPS_SERVER$ changegroupsubset compression=$BUNDLE2_COMPRESSIONS$ getbundle httpheader=1024 httpmediatype=0.1rx,0.1tx,0.2tx known lookup pushkey streamreqs=generaldelta,revlogv1 unbundle=HG10GZ,HG10BZ,HG10UN unbundlehash
+ sending heads command
+ s> GET /redirected?cmd=heads HTTP/1.1\r\n
+ s> Accept-Encoding: identity\r\n
+ s> vary: X-HgProto-1\r\n
+ s> x-hgproto-1: 0.1 0.2 comp=$USUAL_COMPRESSIONS$ partial-pull\r\n
+ s> accept: application/mercurial-0.1\r\n
+ s> host: $LOCALIP:$HGPORT\r\n (glob)
+ s> user-agent: Mercurial debugwireproto\r\n
+ s> \r\n
+ s> makefile('rb', None)
+ s> HTTP/1.1 200 Script output follows\r\n
+ s> Server: testing stub value\r\n
+ s> Date: $HTTP_DATE$\r\n
+ s> Content-Type: application/mercurial-0.1\r\n
+ s> Content-Length: 41\r\n
+ s> \r\n
+ s> 96ee1d7354c4ad7372047672c36a1f561e3a6a4c\n
+ response: [b'\x96\xee\x1dsT\xc4\xadsr\x04vr\xc3j\x1fV\x1e:jL']
+
+ $ killdaemons.py
+
+Now test a variation where we strip the query string from the redirect URL.
+(SCM Manager apparently did this and clients would recover from it)
+
+ $ hg --config extensions.redirect=$TESTTMP/redirectext.py \
+ > --config server.compressionengines=zlib \
+ > --config testing.redirectqs=false \
+ > serve --web-conf paths.conf --pid-file hg.pid -p $HGPORT -d
+ $ cat hg.pid > $DAEMON_PIDS
+
+ $ hg --verbose debugwireproto --peer raw http://$LOCALIP:$HGPORT << EOF
+ > httprequest GET /redirector?cmd=capabilities
+ > user-agent: test
+ > EOF
+ using raw connection to peer
+ s> GET /redirector?cmd=capabilities HTTP/1.1\r\n
+ s> Accept-Encoding: identity\r\n
+ s> user-agent: test\r\n
+ s> host: $LOCALIP:$HGPORT\r\n (glob)
+ s> \r\n
+ s> makefile('rb', None)
+ s> HTTP/1.1 301 Redirect\r\n
+ s> Server: testing stub value\r\n
+ s> Date: $HTTP_DATE$\r\n
+ s> Location: http://$LOCALIP:$HGPORT/redirected\r\n (glob)
+ s> Content-Type: text/plain\r\n
+ s> Content-Length: 10\r\n
+ s> \r\n
+ s> redirected
+ s> GET /redirected HTTP/1.1\r\n
+ s> Accept-Encoding: identity\r\n
+ s> user-agent: test\r\n
+ s> host: $LOCALIP:$HGPORT\r\n (glob)
+ s> \r\n
+ s> makefile('rb', None)
+ s> HTTP/1.1 200 Script output follows\r\n
+ s> Server: testing stub value\r\n
+ s> Date: $HTTP_DATE$\r\n
+ s> ETag: W/"*"\r\n (glob)
+ s> Content-Type: text/html; charset=ascii\r\n
+ s> Transfer-Encoding: chunked\r\n
+ s> \r\n
+ s> 414\r\n
+ s> <!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.1//EN" "http://www.w3.org/TR/xhtml11/DTD/xhtml11.dtd">\n
+ s> <html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en-US">\n
+ s> <head>\n
+ s> <link rel="icon" href="/redirected/static/hgicon.png" type="image/png" />\n
+ s> <meta name="robots" content="index, nofollow" />\n
+ s> <link rel="stylesheet" href="/redirected/static/style-paper.css" type="text/css" />\n
+ s> <script type="text/javascript" src="/redirected/static/mercurial.js"></script>\n
+ s> \n
+ s> <title>redirected: log</title>\n
+ s> <link rel="alternate" type="application/atom+xml"\n
+ s> href="/redirected/atom-log" title="Atom feed for redirected" />\n
+ s> <link rel="alternate" type="application/rss+xml"\n
+ s> href="/redirected/rss-log" title="RSS feed for redirected" />\n
+ s> </head>\n
+ s> <body>\n
+ s> \n
+ s> <div class="container">\n
+ s> <div class="menu">\n
+ s> <div class="logo">\n
+ s> <a href="https://mercurial-scm.org/">\n
+ s> <img src="/redirected/static/hglogo.png" alt="mercurial" /></a>\n
+ s> </div>\n
+ s> <ul>\n
+ s> <li class="active">log</li>\n
+ s> <li><a href="/redirected/graph/tip">graph</a></li>\n
+ s> <li><a href="/redirected/tags">tags</a></li>\n
+ s> <li><a href="
+ s> \r\n
+ s> 810\r\n
+ s> /redirected/bookmarks">bookmarks</a></li>\n
+ s> <li><a href="/redirected/branches">branches</a></li>\n
+ s> </ul>\n
+ s> <ul>\n
+ s> <li><a href="/redirected/rev/tip">changeset</a></li>\n
+ s> <li><a href="/redirected/file/tip">browse</a></li>\n
+ s> </ul>\n
+ s> <ul>\n
+ s> \n
+ s> </ul>\n
+ s> <ul>\n
+ s> <li><a href="/redirected/help">help</a></li>\n
+ s> </ul>\n
+ s> <div class="atom-logo">\n
+ s> <a href="/redirected/atom-log" title="subscribe to atom feed">\n
+ s> <img class="atom-logo" src="/redirected/static/feed-icon-14x14.png" alt="atom feed" />\n
+ s> </a>\n
+ s> </div>\n
+ s> </div>\n
+ s> \n
+ s> <div class="main">\n
+ s> <h2 class="breadcrumb"><a href="/">Mercurial</a> > <a href="/redirected">redirected</a> </h2>\n
+ s> <h3>log</h3>\n
+ s> \n
+ s> \n
+ s> <form class="search" action="/redirected/log">\n
+ s> \n
+ s> <p><input name="rev" id="search1" type="text" size="30" value="" /></p>\n
+ s> <div id="hint">Find changesets by keywords (author, files, the commit message), revision\n
+ s> number or hash, or <a href="/redirected/help/revsets">revset expression</a>.</div>\n
+ s> </form>\n
+ s> \n
+ s> <div class="navigate">\n
+ s> <a href="/redirected/shortlog/tip?revcount=30">less</a>\n
+ s> <a href="/redirected/shortlog/tip?revcount=120">more</a>\n
+ s> | rev 0: <a href="/redirected/shortlog/96ee1d7354c4">(0)</a> <a href="/redirected/shortlog/tip">tip</a> \n
+ s> </div>\n
+ s> \n
+ s> <table class="bigtable">\n
+ s> <thead>\n
+ s> <tr>\n
+ s> <th class="age">age</th>\n
+ s> <th class="author">author</th>\n
+ s> <th class="description">description</th>\n
+ s> </tr>\n
+ s> </thead>\n
+ s> <tbody class="stripes2">\n
+ s> <tr>\n
+ s> <td class="age">Thu, 01 Jan 1970 00:00:00 +0000</td>\n
+ s> <td class="author">test</td>\n
+ s> <td class="description">\n
+ s> <a href="/redirected/rev/96ee1d7354c4">initial</a>\n
+ s> <span class="phase">draft</span> <span class="branchhead">default</span> <span class="tag">tip</span> \n
+ s> </td>\n
+ s> </tr>\n
+ s> \n
+ s> </tbody>\n
+ s> </table>\n
+ s> \n
+ s> <div class="navigate">\n
+ s> <a href="/redirected/shortlog/tip?revcount=30">less</a>\n
+ s> <a href="/redirected/shortlog/tip?revcount=120">more</a>\n
+ s> | rev 0: <a href="/redirected/shortlog/96ee1d7354c4">(0)</a> <a href="/redirected/shortlog/tip">tip</a> \n
+ s> </div>\n
+ s> \n
+ s> <script type="text/javascript">\n
+ s> ajaxScrollInit(\n
+ s> \'/redirected/shortlog/%next%\',\n
+ s> \'\', <!-- NEXTHASH\n
+ s> function (htmlText) {
+ s> \r\n
+ s> 14a\r\n
+ s> \n
+ s> var m = htmlText.match(/\'(\\w+)\', <!-- NEXTHASH/);\n
+ s> return m ? m[1] : null;\n
+ s> },\n
+ s> \'.bigtable > tbody\',\n
+ s> \'<tr class="%class%">\\\n
+ s> <td colspan="3" style="text-align: center;">%text%</td>\\\n
+ s> </tr>\'\n
+ s> );\n
+ s> </script>\n
+ s> \n
+ s> </div>\n
+ s> </div>\n
+ s> \n
+ s> \n
+ s> \n
+ s> </body>\n
+ s> </html>\n
+ s> \n
+ s> \r\n
+ s> 0\r\n
+ s> \r\n
+
+ $ hg --verbose debugwireproto http://$LOCALIP:$HGPORT/redirector << EOF
+ > command heads
+ > EOF
+ s> GET /redirector?cmd=capabilities HTTP/1.1\r\n
+ s> Accept-Encoding: identity\r\n
+ s> accept: application/mercurial-0.1\r\n
+ s> host: $LOCALIP:$HGPORT\r\n (glob)
+ s> user-agent: Mercurial debugwireproto\r\n
+ s> \r\n
+ s> makefile('rb', None)
+ s> HTTP/1.1 301 Redirect\r\n
+ s> Server: testing stub value\r\n
+ s> Date: $HTTP_DATE$\r\n
+ s> Location: http://$LOCALIP:$HGPORT/redirected\r\n (glob)
+ s> Content-Type: text/plain\r\n
+ s> Content-Length: 10\r\n
+ s> \r\n
+ s> redirected
+ s> GET /redirected HTTP/1.1\r\n
+ s> Accept-Encoding: identity\r\n
+ s> accept: application/mercurial-0.1\r\n
+ s> host: $LOCALIP:$HGPORT\r\n (glob)
+ s> user-agent: Mercurial debugwireproto\r\n
+ s> \r\n
+ s> makefile('rb', None)
+ s> HTTP/1.1 200 Script output follows\r\n
+ s> Server: testing stub value\r\n
+ s> Date: $HTTP_DATE$\r\n
+ s> ETag: W/"*"\r\n (glob)
+ s> Content-Type: text/html; charset=ascii\r\n
+ s> Transfer-Encoding: chunked\r\n
+ s> \r\n
+ real URL is http://$LOCALIP:$HGPORT/redirected (glob)
+ s> 414\r\n
+ s> <!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.1//EN" "http://www.w3.org/TR/xhtml11/DTD/xhtml11.dtd">\n
+ s> <html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en-US">\n
+ s> <head>\n
+ s> <link rel="icon" href="/redirected/static/hgicon.png" type="image/png" />\n
+ s> <meta name="robots" content="index, nofollow" />\n
+ s> <link rel="stylesheet" href="/redirected/static/style-paper.css" type="text/css" />\n
+ s> <script type="text/javascript" src="/redirected/static/mercurial.js"></script>\n
+ s> \n
+ s> <title>redirected: log</title>\n
+ s> <link rel="alternate" type="application/atom+xml"\n
+ s> href="/redirected/atom-log" title="Atom feed for redirected" />\n
+ s> <link rel="alternate" type="application/rss+xml"\n
+ s> href="/redirected/rss-log" title="RSS feed for redirected" />\n
+ s> </head>\n
+ s> <body>\n
+ s> \n
+ s> <div class="container">\n
+ s> <div class="menu">\n
+ s> <div class="logo">\n
+ s> <a href="https://mercurial-scm.org/">\n
+ s> <img src="/redirected/static/hglogo.png" alt="mercurial" /></a>\n
+ s> </div>\n
+ s> <ul>\n
+ s> <li class="active">log</li>\n
+ s> <li><a href="/redirected/graph/tip">graph</a></li>\n
+ s> <li><a href="/redirected/tags">tags</a
+ s> GET /redirected?cmd=capabilities HTTP/1.1\r\n
+ s> Accept-Encoding: identity\r\n
+ s> accept: application/mercurial-0.1\r\n
+ s> host: $LOCALIP:$HGPORT\r\n (glob)
+ s> user-agent: Mercurial debugwireproto\r\n
+ s> \r\n
+ s> makefile('rb', None)
+ s> HTTP/1.1 200 Script output follows\r\n
+ s> Server: testing stub value\r\n
+ s> Date: $HTTP_DATE$\r\n
+ s> Content-Type: application/mercurial-0.1\r\n
+ s> Content-Length: 453\r\n
+ s> \r\n
+ real URL is http://$LOCALIP:$HGPORT/redirected (glob)
+ s> batch branchmap $USUAL_BUNDLE2_CAPS_SERVER$ changegroupsubset compression=$BUNDLE2_COMPRESSIONS$ getbundle httpheader=1024 httpmediatype=0.1rx,0.1tx,0.2tx known lookup pushkey streamreqs=generaldelta,revlogv1 unbundle=HG10GZ,HG10BZ,HG10UN unbundlehash
+ sending heads command
+ s> GET /redirected?cmd=heads HTTP/1.1\r\n
+ s> Accept-Encoding: identity\r\n
+ s> vary: X-HgProto-1\r\n
+ s> x-hgproto-1: 0.1 0.2 comp=$USUAL_COMPRESSIONS$ partial-pull\r\n
+ s> accept: application/mercurial-0.1\r\n
+ s> host: $LOCALIP:$HGPORT\r\n (glob)
+ s> user-agent: Mercurial debugwireproto\r\n
+ s> \r\n
+ s> makefile('rb', None)
+ s> HTTP/1.1 200 Script output follows\r\n
+ s> Server: testing stub value\r\n
+ s> Date: $HTTP_DATE$\r\n
+ s> Content-Type: application/mercurial-0.1\r\n
+ s> Content-Length: 41\r\n
+ s> \r\n
+ s> 96ee1d7354c4ad7372047672c36a1f561e3a6a4c\n
+ response: [b'\x96\xee\x1dsT\xc4\xadsr\x04vr\xc3j\x1fV\x1e:jL']
diff --git a/mercurial/httppeer.py b/mercurial/httppeer.py
--- a/mercurial/httppeer.py
+++ b/mercurial/httppeer.py
@@ -328,13 +328,24 @@
return res
+class RedirectedRepoError(error.RepoError):
+ def __init__(self, msg, respurl):
+ super(RedirectedRepoError, self).__init__(msg)
+ self.respurl = respurl
+
def parsev1commandresponse(ui, baseurl, requrl, qs, resp, compressible,
allowcbor=False):
# record the url we got redirected to
+ redirected = False
respurl = pycompat.bytesurl(resp.geturl())
if respurl.endswith(qs):
respurl = respurl[:-len(qs)]
+ qsdropped = False
+ else:
+ qsdropped = True
+
if baseurl.rstrip('/') != respurl.rstrip('/'):
+ redirected = True
if not ui.quiet:
ui.warn(_('real URL is %s\n') % respurl)
@@ -351,10 +362,16 @@
# application/hg-changegroup. We don't support such old servers.
if not proto.startswith('application/mercurial-'):
ui.debug("requested URL: '%s'\n" % util.hidepassword(requrl))
- raise error.RepoError(
- _("'%s' does not appear to be an hg repository:\n"
- "---%%<--- (%s)\n%s\n---%%<---\n")
- % (safeurl, proto or 'no content-type', resp.read(1024)))
+ msg = _("'%s' does not appear to be an hg repository:\n"
+ "---%%<--- (%s)\n%s\n---%%<---\n") % (
+ safeurl, proto or 'no content-type', resp.read(1024))
+
+ # Some servers may strip the query string from the redirect. We
+ # raise a special error type so callers can react to this specially.
+ if redirected and qsdropped:
+ raise RedirectedRepoError(msg, respurl)
+ else:
+ raise error.RepoError(msg)
try:
subtype = proto.split('-', 1)[1]
@@ -434,8 +451,6 @@
# End of ipeercommands interface.
- # look up capabilities only when needed
-
def _callstream(self, cmd, _compressible=False, **args):
args = pycompat.byteskwargs(args)
@@ -853,12 +868,32 @@
req, requrl, qs = makev1commandrequest(ui, requestbuilder, caps,
capable, url, 'capabilities',
args)
-
resp = sendrequest(ui, opener, req)
- respurl, ct, resp = parsev1commandresponse(ui, url, requrl, qs, resp,
- compressible=False,
- allowcbor=advertisev2)
+ # The server may redirect us to the repo root, stripping the
+ # ?cmd=capabilities query string from the URL. The server would likely
+ # return HTML in this case and ``parsev1commandresponse()`` would raise.
+ # We catch this special case and re-issue the capabilities request against
+ # the new URL.
+ #
+ # We should ideally not do this, as a redirect that drops the query
+ # string from the URL is arguably a server bug. (Garbage in, garbage out).
+ # However, Mercurial clients for several years appeared to handle this
+ # issue without behavior degradation. And according to issue 5860, it may
+ # be a longstanding bug in some server implementations. So we allow a
+ # redirect that drops the query string to "just work."
+ try:
+ respurl, ct, resp = parsev1commandresponse(ui, url, requrl, qs, resp,
+ compressible=False,
+ allowcbor=advertisev2)
+ except RedirectedRepoError as e:
+ req, requrl, qs = makev1commandrequest(ui, requestbuilder, caps,
+ capable, e.respurl,
+ 'capabilities', args)
+ resp = sendrequest(ui, opener, req)
+ respurl, ct, resp = parsev1commandresponse(ui, url, requrl, qs, resp,
+ compressible=False,
+ allowcbor=advertisev2)
try:
rawdata = resp.read()
To: indygreg, #hg-reviewers
Cc: martinvonz, mharbison72, mercurial-devel
More information about the Mercurial-devel
mailing list