D4777: wireprotov2: server support for sending content redirects
indygreg (Gregory Szorc)
phabricator at mercurial-scm.org
Wed Oct 3 15:30:50 UTC 2018
This revision was automatically updated to reflect the committed changes.
Closed by commit rHGb099e6032f38: wireprotov2: server support for sending content redirects (authored by indygreg, committed by ).
REPOSITORY
rHG Mercurial
CHANGES SINCE LAST UPDATE
https://phab.mercurial-scm.org/D4777?vs=11448&id=11628
REVISION DETAIL
https://phab.mercurial-scm.org/D4777
AFFECTED FILES
mercurial/wireprotoframing.py
mercurial/wireprototypes.py
mercurial/wireprotov2server.py
tests/test-http-api-httpv2.t
tests/test-wireproto-content-redirects.t
tests/test-wireproto-serverreactor.py
tests/wireprotosimplecache.py
CHANGE DETAILS
diff --git a/tests/wireprotosimplecache.py b/tests/wireprotosimplecache.py
--- a/tests/wireprotosimplecache.py
+++ b/tests/wireprotosimplecache.py
@@ -12,6 +12,7 @@
registrar,
repository,
util,
+ wireprotoserver,
wireprototypes,
wireprotov2server,
)
@@ -25,18 +26,59 @@
configtable = {}
configitem = registrar.configitem(configtable)
+configitem('simplecache', 'cacheapi',
+ default=False)
configitem('simplecache', 'cacheobjects',
default=False)
configitem('simplecache', 'redirectsfile',
default=None)
+# API handler that makes cached keys available.
+def handlecacherequest(rctx, req, res, checkperm, urlparts):
+ if rctx.repo.ui.configbool('simplecache', 'cacheobjects'):
+ res.status = b'500 Internal Server Error'
+ res.setbodybytes(b'cacheobjects not supported for api server')
+ return
+
+ if not urlparts:
+ res.status = b'200 OK'
+ res.headers[b'Content-Type'] = b'text/plain'
+ res.setbodybytes(b'simple cache server')
+ return
+
+ key = b'/'.join(urlparts)
+
+ if key not in CACHE:
+ res.status = b'404 Not Found'
+ res.headers[b'Content-Type'] = b'text/plain'
+ res.setbodybytes(b'key not found in cache')
+ return
+
+ res.status = b'200 OK'
+ res.headers[b'Content-Type'] = b'application/mercurial-cbor'
+ res.setbodybytes(CACHE[key])
+
+def cachedescriptor(req, repo):
+ return {}
+
+wireprotoserver.API_HANDLERS[b'simplecache'] = {
+ 'config': (b'simplecache', b'cacheapi'),
+ 'handler': handlecacherequest,
+ 'apidescriptor': cachedescriptor,
+}
+
@interfaceutil.implementer(repository.iwireprotocolcommandcacher)
class memorycacher(object):
- def __init__(self, ui, command, encodefn):
+ def __init__(self, ui, command, encodefn, redirecttargets, redirecthashes,
+ req):
self.ui = ui
self.encodefn = encodefn
+ self.redirecttargets = redirecttargets
+ self.redirecthashes = redirecthashes
+ self.req = req
self.key = None
self.cacheobjects = ui.configbool('simplecache', 'cacheobjects')
+ self.cacheapi = ui.configbool('simplecache', 'cacheapi')
self.buffered = []
ui.log('simplecache', 'cacher constructed for %s\n', command)
@@ -65,6 +107,37 @@
entry = CACHE[self.key]
self.ui.log('simplecache', 'cache hit for %s\n', self.key)
+ redirectable = True
+
+ if not self.cacheapi:
+ redirectable = False
+ elif not self.redirecttargets:
+ redirectable = False
+ else:
+ clienttargets = set(self.redirecttargets)
+ ourtargets = set(t[b'name'] for t in loadredirecttargets(self.ui))
+
+ # We only ever redirect to a single target (for now). So we don't
+ # need to store which target matched.
+ if not clienttargets & ourtargets:
+ redirectable = False
+
+ if redirectable:
+ paths = self.req.dispatchparts[:-3]
+ paths.append(b'simplecache')
+ paths.append(self.key)
+
+ url = b'%s/%s' % (self.req.advertisedbaseurl, b'/'.join(paths))
+
+ #url = b'http://example.com/%s' % self.key
+ self.ui.log('simplecache', 'sending content redirect for %s to '
+ '%s\n', self.key, url)
+ response = wireprototypes.alternatelocationresponse(
+ url=url,
+ mediatype=b'application/mercurial-cbor')
+
+ return {'objs': [response]}
+
if self.cacheobjects:
return {
'objs': entry,
@@ -91,8 +164,10 @@
return []
-def makeresponsecacher(orig, repo, proto, command, args, objencoderfn):
- return memorycacher(repo.ui, command, objencoderfn)
+def makeresponsecacher(orig, repo, proto, command, args, objencoderfn,
+ redirecttargets, redirecthashes):
+ return memorycacher(repo.ui, command, objencoderfn, redirecttargets,
+ redirecthashes, proto._req)
def loadredirecttargets(ui):
path = ui.config('simplecache', 'redirectsfile')
diff --git a/tests/test-wireproto-serverreactor.py b/tests/test-wireproto-serverreactor.py
--- a/tests/test-wireproto-serverreactor.py
+++ b/tests/test-wireproto-serverreactor.py
@@ -69,6 +69,7 @@
b'requestid': 1,
b'command': b'mycommand',
b'args': {},
+ b'redirect': None,
b'data': None,
})
@@ -86,6 +87,7 @@
b'requestid': 41,
b'command': b'mycommand',
b'args': {b'foo': b'bar'},
+ b'redirect': None,
b'data': None,
})
@@ -100,6 +102,7 @@
b'requestid': 1,
b'command': b'mycommand',
b'args': {b'foo': b'bar', b'biz': b'baz'},
+ b'redirect': None,
b'data': None,
})
@@ -115,6 +118,7 @@
b'requestid': 1,
b'command': b'mycommand',
b'args': {},
+ b'redirect': None,
b'data': b'data!',
})
@@ -137,6 +141,7 @@
b'requestid': 1,
b'command': b'mycommand',
b'args': {},
+ b'redirect': None,
b'data': b'data1data2data3',
})
@@ -160,6 +165,7 @@
b'key': b'val',
b'foo': b'bar',
},
+ b'redirect': None,
b'data': b'value1value2',
})
@@ -235,6 +241,7 @@
b'requestid': 1,
b'command': b'command',
b'args': {},
+ b'redirect': None,
b'data': None,
})
@@ -291,12 +298,14 @@
b'requestid': 3,
b'command': b'command3',
b'args': {b'biz': b'baz', b'key': b'val'},
+ b'redirect': None,
b'data': None,
})
self.assertEqual(results[5][1], {
b'requestid': 1,
b'command': b'command1',
b'args': {b'foo': b'bar', b'key1': b'val'},
+ b'redirect': None,
b'data': None,
})
diff --git a/tests/test-wireproto-content-redirects.t b/tests/test-wireproto-content-redirects.t
--- a/tests/test-wireproto-content-redirects.t
+++ b/tests/test-wireproto-content-redirects.t
@@ -1,11 +1,20 @@
$ . $TESTDIR/wireprotohelpers.sh
+ $ cat >> $HGRCPATH << EOF
+ > [extensions]
+ > blackbox =
+ > [blackbox]
+ > track = simplecache
+ > EOF
+
$ hg init server
$ enablehttpv2 server
$ cd server
$ cat >> .hg/hgrc << EOF
> [extensions]
> simplecache = $TESTDIR/wireprotosimplecache.py
+ > [simplecache]
+ > cacheapi = true
> EOF
$ echo a0 > a
@@ -1183,5 +1192,178 @@
}
]
+Set up the server to issue content redirects to its built-in API server.
+
+ $ cat > redirects.py << EOF
+ > [
+ > {
+ > b'name': b'local',
+ > b'protocol': b'http',
+ > b'uris': [b'http://example.com/'],
+ > },
+ > ]
+ > EOF
+
+Request to eventual cache URL should return 404 (validating the cache server works)
+
+ $ sendhttpraw << EOF
+ > httprequest GET api/simplecache/missingkey
+ > user-agent: test
+ > EOF
+ using raw connection to peer
+ s> GET /api/simplecache/missingkey HTTP/1.1\r\n
+ s> Accept-Encoding: identity\r\n
+ s> user-agent: test\r\n
+ s> host: $LOCALIP:$HGPORT\r\n (glob)
+ s> \r\n
+ s> makefile('rb', None)
+ s> HTTP/1.1 404 Not Found\r\n
+ s> Server: testing stub value\r\n
+ s> Date: $HTTP_DATE$\r\n
+ s> Content-Type: text/plain\r\n
+ s> Content-Length: 22\r\n
+ s> \r\n
+ s> key not found in cache
+
+Send a cacheable request
+
+ $ sendhttpv2peer << EOF
+ > command manifestdata
+ > nodes eval:[b'\x99\x2f\x47\x79\x02\x9a\x3d\xf8\xd0\x66\x6d\x00\xbb\x92\x4f\x69\x63\x4e\x26\x41']
+ > tree eval:b''
+ > fields eval:[b'parents']
+ > EOF
+ creating http peer for wire protocol version 2
+ sending manifestdata command
+ s> POST /api/exp-http-v2-0002/ro/manifestdata HTTP/1.1\r\n
+ s> Accept-Encoding: identity\r\n
+ s> accept: application/mercurial-exp-framing-0005\r\n
+ s> content-type: application/mercurial-exp-framing-0005\r\n
+ s> content-length: 128\r\n
+ s> host: $LOCALIP:$HGPORT\r\n (glob)
+ s> user-agent: Mercurial debugwireproto\r\n
+ s> \r\n
+ s> x\x00\x00\x01\x00\x01\x01\x11\xa3Dargs\xa3Ffields\x81GparentsEnodes\x81T\x99/Gy\x02\x9a=\xf8\xd0fm\x00\xbb\x92OicN&ADtree at DnameLmanifestdataHredirect\xa2Fhashes\x82Fsha256Dsha1Gtargets\x81Elocal
+ s> makefile('rb', None)
+ s> HTTP/1.1 200 OK\r\n
+ s> Server: testing stub value\r\n
+ s> Date: $HTTP_DATE$\r\n
+ s> Content-Type: application/mercurial-exp-framing-0005\r\n
+ s> Transfer-Encoding: chunked\r\n
+ s> \r\n
+ s> 13\r\n
+ s> \x0b\x00\x00\x01\x00\x02\x011
+ s> \xa1FstatusBok
+ s> \r\n
+ received frame(size=11; request=1; stream=2; streamflags=stream-begin; type=command-response; flags=continuation)
+ s> 63\r\n
+ s> [\x00\x00\x01\x00\x02\x001
+ s> \xa1Jtotalitems\x01\xa2DnodeT\x99/Gy\x02\x9a=\xf8\xd0fm\x00\xbb\x92OicN&AGparents\x82T\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00T\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00
+ s> \r\n
+ received frame(size=91; request=1; stream=2; streamflags=; type=command-response; flags=continuation)
+ s> 8\r\n
+ s> \x00\x00\x00\x01\x00\x02\x002
+ s> \r\n
+ s> 0\r\n
+ s> \r\n
+ received frame(size=0; request=1; stream=2; streamflags=; type=command-response; flags=eos)
+ response: gen[
+ {
+ b'totalitems': 1
+ },
+ {
+ b'node': b'\x99/Gy\x02\x9a=\xf8\xd0fm\x00\xbb\x92OicN&A',
+ b'parents': [
+ b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00',
+ b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
+ ]
+ }
+ ]
+
+Cached entry should be available on server
+
+ $ sendhttpraw << EOF
+ > httprequest GET api/simplecache/c045a581599d58608efd3d93d8129841f2af04a0
+ > user-agent: test
+ > EOF
+ using raw connection to peer
+ s> GET /api/simplecache/c045a581599d58608efd3d93d8129841f2af04a0 HTTP/1.1\r\n
+ s> Accept-Encoding: identity\r\n
+ s> user-agent: test\r\n
+ s> host: $LOCALIP:$HGPORT\r\n (glob)
+ s> \r\n
+ s> makefile('rb', None)
+ s> HTTP/1.1 200 OK\r\n
+ s> Server: testing stub value\r\n
+ s> Date: $HTTP_DATE$\r\n
+ s> Content-Type: application/mercurial-cbor\r\n
+ s> Content-Length: 91\r\n
+ s> \r\n
+ s> \xa1Jtotalitems\x01\xa2DnodeT\x99/Gy\x02\x9a=\xf8\xd0fm\x00\xbb\x92OicN&AGparents\x82T\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00T\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00
+ cbor> [
+ {
+ b'totalitems': 1
+ },
+ {
+ b'node': b'\x99/Gy\x02\x9a=\xf8\xd0fm\x00\xbb\x92OicN&A',
+ b'parents': [
+ b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00',
+ b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
+ ]
+ }
+ ]
+
+2nd request should result in content redirect response
+
+ $ sendhttpv2peer << EOF
+ > command manifestdata
+ > nodes eval:[b'\x99\x2f\x47\x79\x02\x9a\x3d\xf8\xd0\x66\x6d\x00\xbb\x92\x4f\x69\x63\x4e\x26\x41']
+ > tree eval:b''
+ > fields eval:[b'parents']
+ > EOF
+ creating http peer for wire protocol version 2
+ sending manifestdata command
+ s> POST /api/exp-http-v2-0002/ro/manifestdata HTTP/1.1\r\n
+ s> Accept-Encoding: identity\r\n
+ s> accept: application/mercurial-exp-framing-0005\r\n
+ s> content-type: application/mercurial-exp-framing-0005\r\n
+ s> content-length: 128\r\n
+ s> host: $LOCALIP:$HGPORT\r\n (glob)
+ s> user-agent: Mercurial debugwireproto\r\n
+ s> \r\n
+ s> x\x00\x00\x01\x00\x01\x01\x11\xa3Dargs\xa3Ffields\x81GparentsEnodes\x81T\x99/Gy\x02\x9a=\xf8\xd0fm\x00\xbb\x92OicN&ADtree at DnameLmanifestdataHredirect\xa2Fhashes\x82Fsha256Dsha1Gtargets\x81Elocal
+ s> makefile('rb', None)
+ s> HTTP/1.1 200 OK\r\n
+ s> Server: testing stub value\r\n
+ s> Date: $HTTP_DATE$\r\n
+ s> Content-Type: application/mercurial-exp-framing-0005\r\n
+ s> Transfer-Encoding: chunked\r\n
+ s> \r\n
+ s> *\r\n (glob)
+ s> \x*\x00\x00\x01\x00\x02\x011 (glob)
+ s> \xa2Hlocation\xa2ImediatypeX\x1aapplication/mercurial-cborCurl*http://*:$HGPORT/api/simplecache/c045a581599d58608efd3d93d8129841f2af04a0FstatusHredirect (glob)
+ s> \r\n
+ received frame(size=*; request=1; stream=2; streamflags=stream-begin; type=command-response; flags=continuation) (glob)
+ s> 8\r\n
+ s> \x00\x00\x00\x01\x00\x02\x001
+ s> \r\n
+ s> 8\r\n
+ received frame(size=0; request=1; stream=2; streamflags=; type=command-response; flags=continuation)
+ s> \x00\x00\x00\x01\x00\x02\x002
+ s> \r\n
+ s> 0\r\n
+ s> \r\n
+ received frame(size=0; request=1; stream=2; streamflags=; type=command-response; flags=eos)
+ abort: redirect responses not yet supported
+ [255]
+
$ cat error.log
$ killdaemons.py
+
+ $ cat .hg/blackbox.log
+ *> cacher constructed for manifestdata (glob)
+ *> cache miss for c045a581599d58608efd3d93d8129841f2af04a0 (glob)
+ *> storing cache entry for c045a581599d58608efd3d93d8129841f2af04a0 (glob)
+ *> cacher constructed for manifestdata (glob)
+ *> cache hit for c045a581599d58608efd3d93d8129841f2af04a0 (glob)
+ *> sending content redirect for c045a581599d58608efd3d93d8129841f2af04a0 to http://*:$HGPORT/api/simplecache/c045a581599d58608efd3d93d8129841f2af04a0 (glob)
diff --git a/tests/test-http-api-httpv2.t b/tests/test-http-api-httpv2.t
--- a/tests/test-http-api-httpv2.t
+++ b/tests/test-http-api-httpv2.t
@@ -430,10 +430,10 @@
s> Server: testing stub value\r\n
s> Date: $HTTP_DATE$\r\n
s> Content-Type: text/plain\r\n
- s> Content-Length: 205\r\n
+ s> Content-Length: 223\r\n
s> \r\n
s> received: 1 1 1 \xa2Dargs\xa2Dbar1CvalCfooDval1DnameHcommand1\n
- s> ["runcommand", {"args": {"bar1": "val", "foo": "val1"}, "command": "command1", "data": null, "requestid": 1}]\n
+ s> ["runcommand", {"args": {"bar1": "val", "foo": "val1"}, "command": "command1", "data": null, "redirect": null, "requestid": 1}]\n
s> received: <no frame>\n
s> {"action": "noop"}
diff --git a/mercurial/wireprotov2server.py b/mercurial/wireprotov2server.py
--- a/mercurial/wireprotov2server.py
+++ b/mercurial/wireprotov2server.py
@@ -312,7 +312,7 @@
res.headers[b'Content-Type'] = FRAMINGTYPE
try:
- objs = dispatch(repo, proto, command['command'])
+ objs = dispatch(repo, proto, command['command'], command['redirect'])
action, meta = reactor.oncommandresponsereadyobjects(
outstream, command['requestid'], objs)
@@ -339,7 +339,7 @@
def getdispatchrepo(repo, proto, command):
return repo.filtered('served')
-def dispatch(repo, proto, command):
+def dispatch(repo, proto, command, redirect):
"""Run a wire protocol command.
Returns an iterable of objects that will be sent to the client.
@@ -364,8 +364,17 @@
yield o
return
+ if redirect:
+ redirecttargets = redirect[b'targets']
+ redirecthashes = redirect[b'hashes']
+ else:
+ redirecttargets = []
+ redirecthashes = []
+
cacher = makeresponsecacher(repo, proto, command, args,
- cborutil.streamencode)
+ cborutil.streamencode,
+ redirecttargets=redirecttargets,
+ redirecthashes=redirecthashes)
# But we have no cacher. Do default handling.
if not cacher:
@@ -751,7 +760,8 @@
return cachekeyfn
-def makeresponsecacher(repo, proto, command, args, objencoderfn):
+def makeresponsecacher(repo, proto, command, args, objencoderfn,
+ redirecttargets, redirecthashes):
"""Construct a cacher for a cacheable command.
Returns an ``iwireprotocolcommandcacher`` instance.
diff --git a/mercurial/wireprototypes.py b/mercurial/wireprototypes.py
--- a/mercurial/wireprototypes.py
+++ b/mercurial/wireprototypes.py
@@ -368,3 +368,20 @@
and the content from this object is used instead.
"""
data = attr.ib()
+
+ at attr.s
+class alternatelocationresponse(object):
+ """Represents a response available at an alternate location.
+
+ Instances are sent in place of actual response objects when the server
+ is sending a "content redirect" response.
+
+ Only compatible with wire protocol version 2.
+ """
+ url = attr.ib()
+ mediatype = attr.ib()
+ size = attr.ib(default=None)
+ fullhashes = attr.ib(default=None)
+ fullhashseed = attr.ib(default=None)
+ serverdercerts = attr.ib(default=None)
+ servercadercerts = attr.ib(default=None)
diff --git a/mercurial/wireprotoframing.py b/mercurial/wireprotoframing.py
--- a/mercurial/wireprotoframing.py
+++ b/mercurial/wireprotoframing.py
@@ -21,6 +21,7 @@
from . import (
encoding,
error,
+ pycompat,
util,
wireprototypes,
)
@@ -429,6 +430,26 @@
flags=FLAG_COMMAND_RESPONSE_EOS,
payload=b'')
+def createalternatelocationresponseframe(stream, requestid, location):
+ data = {
+ b'status': b'redirect',
+ b'location': {
+ b'url': location.url,
+ b'mediatype': location.mediatype,
+ }
+ }
+
+ for a in (r'size', r'fullhashes', r'fullhashseed', r'serverdercerts',
+ r'servercadercerts'):
+ value = getattr(location, a)
+ if value is not None:
+ data[b'location'][pycompat.bytestr(a)] = value
+
+ return stream.makeframe(requestid=requestid,
+ typeid=FRAME_TYPE_COMMAND_RESPONSE,
+ flags=FLAG_COMMAND_RESPONSE_CONTINUATION,
+ payload=b''.join(cborutil.streamencode(data)))
+
def createcommanderrorresponse(stream, requestid, message, args=None):
# TODO should this be using a list of {'msg': ..., 'args': {}} so atom
# formatting works consistently?
@@ -813,6 +834,7 @@
def sendframes():
emitted = False
+ alternatelocationsent = False
emitter = bufferingcommandresponseemitter(stream, requestid)
while True:
try:
@@ -841,6 +863,25 @@
break
try:
+ # Alternate location responses can only be the first and
+ # only object in the output stream.
+ if isinstance(o, wireprototypes.alternatelocationresponse):
+ if emitted:
+ raise error.ProgrammingError(
+ 'alternatelocationresponse seen after initial '
+ 'output object')
+
+ yield createalternatelocationresponseframe(
+ stream, requestid, o)
+
+ alternatelocationsent = True
+ emitted = True
+ continue
+
+ if alternatelocationsent:
+ raise error.ProgrammingError(
+ 'object follows alternatelocationresponse')
+
if not emitted:
yield createcommandresponseokframe(stream, requestid)
emitted = True
@@ -977,6 +1018,7 @@
'requestid': requestid,
'command': request[b'name'],
'args': request[b'args'],
+ 'redirect': request.get(b'redirect'),
'data': entry['data'].getvalue() if entry['data'] else None,
}
To: indygreg, #hg-reviewers
Cc: mercurial-devel
More information about the Mercurial-devel
mailing list