[PATCH 3 of 4 RFC] Initial import of http2 extension
Augie Fackler
durin42 at gmail.com
Wed Mar 9 09:55:31 CST 2011
# HG changeset patch
# User Augie Fackler <durin42 at gmail.com>
# Date 1299685366 21600
# Node ID 268e83c0b1621565adda555801057fe7bd5ef9d8
# Parent 755fa87ed6990c898a0fb36bf5332e40fa1aa696
Initial import of http2 extension.
diff --git a/hgext/http2/__init__.py b/hgext/http2/__init__.py
new file mode 100644
--- /dev/null
+++ b/hgext/http2/__init__.py
@@ -0,0 +1,232 @@
+# http2: enable experimental faster http support
+#
+# Copyright (C) 2011 Google, Inc.
+#
+# This program is free software; you can redistribute it and/or modify it
+# under the terms of the GNU General Public License as published by the
+# Free Software Foundation; either version 2 of the License, or (at your
+# option) any later version.
+#
+# This program is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General
+# Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with this program; if not, write to the Free Software Foundation, Inc.,
+# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+"""improve http support (experimental)
+
+This extension swaps out the bulk of Mercurial's low-level http code
+with newer code that supports keepalives and 100-continue properly out
+of the box. It should fix the double-push issue in most cases.
+
+Currently missing features:
+ * Client-side x.509 certs for authentication
+ * Proper validation of server certificates
+ * Probably some proxy stuff, but I don't know what I missed
+
+Both of these features should be provided via modifications to the
+underlying http library to provide hooks so we don't have to do
+anything fancy here. The underlying http library is *not* GPL, but
+rather is 2-clause BSD so that we can (at some point) try and
+contribute it as an improved httplib to the greater Python community.
+"""
+
+import urllib
+import urllib2
+import httplib
+import socket
+import sys
+import os
+
+from mercurial import error
+from mercurial import hg
+from mercurial import httprepo
+from mercurial import node
+from mercurial import statichttprepo
+from mercurial import url
+
+from hgext.http2 import http
+
+# Mercurial requires that the http response object be sufficiently
+# file-like, so we provide a close() method here.
+class HTTPResponse(http.HTTPResponse):
+ def close(self):
+ pass
+
+class HTTPConnection(http.HTTPConnection):
+ response_class = HTTPResponse
+ def request(self, method, uri, body=None, headers={}):
+ if isinstance(body, url.httpsendfile):
+ body.seek(0)
+ http.HTTPConnection.request(self, method, uri, body=body,
+ headers=headers)
+
+class KeepAliveHandler:
+ def __init__(self):
+ self._connections = {}
+
+ #### Connection Management
+ def open_connections(self):
+ """return a list of connected hosts and the number of connections
+ to each. [('foo.com:80', 2), ('bar.org', 1)]"""
+ return [(host, len(li)) for host, li in self._connections.items()]
+
+ def close_connection(self, host):
+ """close connection(s) to <host>
+ host is the host:port spec, as in 'www.cnn.com:8080' as passed in.
+ no error occurs if there is no connection to that host."""
+ for h in self._connections.pop(host, []):
+ h.close()
+
+ def close_all(self):
+ """close all open connections"""
+ for host, conns in self._connections.iteritems():
+ for h in conns:
+ h.close()
+ self._connections = {}
+
+ def _remove_connection(self, host, connection, close=0):
+ if close: connection.close()
+ self._connections[host].remove(connection)
+
+ #### Transaction Execution
+ def http_open(self, req):
+ return self.do_open(http.HTTPConnection, req)
+
+ def do_open(self, http_class, req):
+ host = req.get_host()
+ if not host:
+ raise urllib2.URLError('no host given')
+
+ try:
+ conns = [c for c in self._connections.get(host, []) if not c.busy()]
+
+ if not conns:
+ h = http_class(host)
+ else:
+ h = conns.pop(0)
+ self._start_transaction(h, req)
+ r = h.getresponse()
+ self._connections.setdefault(host, []).append(h)
+ except (socket.error, httplib.HTTPException), err:
+ raise urllib2.URLError(err)
+
+ if r.will_close: self._connections[host].remove(h)
+
+ r._handler = self
+ r._host = host
+ r._url = req.get_full_url()
+ r.geturl = lambda : r._url
+ r._connection = h
+ r.code = r.status
+ r.msg = r.reason
+ r.info = lambda : r.headers
+
+ if r.status == 200:
+ return r
+ else:
+ return self.parent.error('http', req, r,
+ r.status, r.msg, r.headers)
+
+ def _start_transaction(self, h, req):
+ # What follows mostly reimplements HTTPConnection.request()
+ # except it adds self.parent.addheaders in the mix.
+ headers = req.headers.copy()
+ headers.update(req.unredirected_hdrs)
+ headers.update(self.parent.addheaders)
+ headers = dict((n.lower(), v) for n,v in headers.items())
+ try:
+ if req.has_data():
+ if 'content-type' not in headers:
+ headers['Content-type'] = 'application/x-www-form-urlencoded'
+ if 'content-length' not in headers:
+ h['Content-length'] = '%d' % len(data)
+ data = req.get_data()
+ h.request('POST', req.get_selector(), headers=headers, body=data)
+ else:
+ h.request('GET', req.get_selector(), headers=headers)
+ except (socket.error), err:
+ raise urllib2.URLError(err)
+
+
+class HandlerCommonMixin(object):
+ def _start_transaction(self, h, req):
+ url._generic_start_transaction(self, h, req)
+ return KeepAliveHandler._start_transaction(self, h, req)
+
+
+class HTTPHandler(KeepAliveHandler, urllib2.HTTPHandler, HandlerCommonMixin):
+ def http_open(self, req):
+ return self.do_open(HTTPConnection, req)
+
+ def __del__(self):
+ self.close_all()
+
+
+class HTTPSHandler(KeepAliveHandler, urllib2.HTTPSHandler, HandlerCommonMixin):
+ def __init__(self, ui):
+ KeepAliveHandler.__init__(self)
+ urllib2.HTTPSHandler.__init__(self)
+ self.ui = ui
+ self.pwmgr = url.passwordmgr(self.ui)
+
+ def https_open(self, req):
+ self.auth = self.pwmgr.readauthtoken(req.get_full_url())
+ return self.do_open(self._makeconnection, req)
+
+ def _makeconnection(self, host, port=443, *args, **kwargs):
+ keyfile = None
+ certfile = None
+
+ if args: # key_file
+ keyfile = args.pop(0)
+ if args: # cert_file
+ certfile = args.pop(0)
+
+ # if the user has specified different key/cert files in
+ # hgrc, we prefer these
+ if self.auth and 'key' in self.auth and 'cert' in self.auth:
+ keyfile = self.auth['key']
+ certfile = self.auth['cert']
+
+ # let host port take precedence
+ if ':' in host and '[' not in host or ']:' in host:
+ host, port = host.rsplit(':', 1)
+ port = int(port)
+ if '[' in host:
+ host = host[1:-1]
+
+ # TODO support keyfile and certfile!
+ return HTTPConnection(host, port, use_ssl=True, *args, **kwargs)
+
+
+url.httphandler = HTTPHandler
+url.httpshandler = HTTPSHandler
+
+# save original __init__ since we're going to overwrite our superclass
+_orig_init = httprepo.httprepository.__init__
+class HTTPContinueRepo(httprepo.httprepository):
+ # a list that is shared amongst all classes so we can correctly
+ # configure logging only once
+ configured_logging = []
+
+ def __init__(self, ui, path):
+ if path.startswith('https') and not url.has_https:
+ raise util.Abort(_('Python support for SSL and HTTPS '
+ 'is not installed'))
+ if ui.debugflag and not self.configured_logging:
+ import logging
+ logging.basicConfig(level=logging.INFO, stream=sys.stderr)
+ configured_logging = True
+ _orig_init(self, ui, path)
+
+ def do_cmd(self, cmd, headers={}, **args):
+ if cmd == 'unbundle':
+ headers['Expect'] = '100-Continue'
+ return httprepo.httprepository.do_cmd(self, cmd, headers=headers,
+ **args)
+
+httprepo.httprepository = HTTPContinueRepo
+httprepo.httpsrepository = HTTPContinueRepo
diff --git a/hgext/http2/http/__init__.py b/hgext/http2/http/__init__.py
new file mode 100644
--- /dev/null
+++ b/hgext/http2/http/__init__.py
@@ -0,0 +1,567 @@
+# Copyright 2010, Google Inc.
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+# * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+# * Redistributions in binary form must reproduce the above
+# copyright notice, this list of conditions and the following disclaimer
+# in the documentation and/or other materials provided with the
+# distribution.
+# * Neither the name of Google Inc. nor the names of its
+# contributors may be used to endorse or promote products derived from
+# this software without specific prior written permission.
+
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+"""Improved HTTP/1.1 client library
+
+This library contains an HTTPConnection which is similar to the one in
+httplib, but has several additional features:
+
+ * supports keepalives natively
+ * uses select() to block for incoming data
+ * notices when the server responds early to a request
+ * implements ssl inline instead of in a different class
+"""
+
+import cStringIO
+import errno
+import httplib
+import logging
+import rfc822
+import select
+import socket
+
+import socketutil
+
+logger = logging.getLogger(__name__)
+
+__all__ = ['HTTPConnection', 'HTTPResponse']
+
+HTTP_VER_1_0 = 'HTTP/1.0'
+HTTP_VER_1_1 = 'HTTP/1.1'
+
+_LEN_CLOSE_IS_END = -1
+
+OUTGOING_BUFFER_SIZE = 1 << 15
+INCOMING_BUFFER_SIZE = 1 << 20
+
+HDR_ACCEPT_ENCODING = 'accept-encoding'
+HDR_CONNECTION_CTRL = 'connection'
+HDR_CONTENT_LENGTH = 'content-length'
+HDR_XFER_ENCODING = 'transfer-encoding'
+
+XFER_ENCODING_CHUNKED = 'chunked'
+
+CONNECTION_CLOSE = 'close'
+
+EOL = '\r\n'
+_END_HEADERS = EOL * 2
+
+# Based on some searching around, 1 second seems like a reasonable default here.
+TIMEOUT_ASSUME_CONTINUE = 1
+TIMEOUT_DEFAULT = None
+
+
+class HTTPResponse(object):
+ """Response from an HTTP server.
+
+ The response will continue to load as available. If you need the
+ complete response before continuing, check the .complete() method.
+ """
+ def __init__(self, sock, timeout):
+ self.sock = sock
+ self.raw_response = ''
+ self._body = None
+ self._headers_len = 0
+ self._content_len = 0
+ self.headers = None
+ self.will_close = False
+ self.status_line = ''
+ self.status = None
+ self.http_version = None
+ self.reason = None
+ self._chunked = False
+ self._chunked_done = False
+ self._chunked_until_next = 0
+ self._chunked_skip_bytes = 0
+ self._chunked_preloaded_block = None
+
+ self._read_location = 0
+ self._eol = EOL
+
+ self._timeout = timeout
+
+ @property
+ def _end_headers(self):
+ return self._eol * 2
+
+ def complete(self):
+ """Returns true if this response is completely loaded.
+ """
+ if self._chunked:
+ return self._chunked_done
+ if self._content_len == _LEN_CLOSE_IS_END:
+ return False
+ return self._body is not None and len(self._body) >= self._content_len
+
+ def readline(self):
+ """Read a single line from the response body.
+
+ This may block until either a line ending is found or the
+ response is complete.
+ """
+ eol = self._body.find('\n', self._read_location)
+ while eol == -1 and not self.complete():
+ self._select()
+ eol = self._body.find('\n', self._read_location)
+ if eol != -1:
+ eol += 1
+ else:
+ eol = len(self._body)
+ data = self._body[self._read_location:eol]
+ self._read_location = eol
+ return data
+
+ def read(self, length=None):
+ # if length is None, unbounded read
+ while (not self.complete() # never select on a finished read
+ and (not length # unbounded, so we wait for complete()
+ or (self._read_location + length) > len(self._body))):
+ self._select()
+ if not length:
+ length = len(self._body) - self._read_location
+ elif len(self._body) < (self._read_location + length):
+ length = len(self._body) - self._read_location
+ r = self._body[self._read_location:self._read_location+length]
+ self._read_location += len(r)
+ return r
+
+ def _select(self):
+ r, _, _ = select.select([self.sock], [], [], self._timeout)
+ if not r:
+ # socket was not readable. If the response is not complete
+ # and we're not a _LEN_CLOSE_IS_END response, raise a timeout.
+ # If we are a _LEN_CLOSE_IS_END response and we have no data,
+ # raise a timeout.
+ if not (self.complete() or
+ (self._content_len == _LEN_CLOSE_IS_END and self._body)):
+ logger.info('timed out with timeout of %s', self._timeout)
+ raise HTTPTimeoutException('timeout reading data')
+ logger.info('cl: %r body: %r', self._content_len, self._body)
+ data = self.sock.recv(INCOMING_BUFFER_SIZE)
+ if not data:
+ if not self.headers:
+ self._load_response(self._end_headers)
+ self._content_len = 0
+ elif self._content_len == _LEN_CLOSE_IS_END:
+ self._content_len = len(self._body)
+ else:
+ self._load_response(data)
+
+ def _chunked_parsedata(self, data):
+ if self._chunked_preloaded_block:
+ data = self._chunked_preloaded_block + data
+ self._chunked_preloaded_block = None
+ while data:
+ logger.debug('looping with %d data remaining', len(data))
+ # Slice out anything we should skip
+ if self._chunked_skip_bytes:
+ if len(data) <= self._chunked_skip_bytes:
+ self._chunked_skip_bytes -= len(data)
+ data = ''
+ break
+ else:
+ data = data[self._chunked_skip_bytes:]
+ self._chunked_skip_bytes = 0
+
+ # determine how much is until the next chunk
+ if self._chunked_until_next:
+ amt = self._chunked_until_next
+ logger.debug('reading remaining %d of existing chunk', amt)
+ self._chunked_until_next = 0
+ body = data
+ else:
+ try:
+ amt, body = data.split(self._eol, 1)
+ except ValueError:
+ self._chunked_preloaded_block = data
+ logger.debug('saving %r as a preloaded block for chunked',
+ self._chunked_preloaded_block)
+ return
+ amt = int(amt, base=16)
+ logger.debug('reading chunk of length %d', amt)
+ if amt == 0:
+ self._chunked_done = True
+
+ # read through end of what we have or the chunk
+ self._body += body[:amt]
+ if len(body) >= amt:
+ data = body[amt:]
+ self._chunked_skip_bytes = len(self._eol)
+ else:
+ self._chunked_until_next = amt - len(body)
+ self._chunked_skip_bytes = 0
+ data = ''
+
+ def _load_response(self, data):
+ if self._chunked:
+ self._chunked_parsedata(data)
+ return
+ elif self._body is not None:
+ self._body += data
+ return
+
+ # We haven't seen end of headers yet
+ self.raw_response += data
+ # This is a bogus server with bad line endings
+ if self._eol not in self.raw_response:
+ for bad_eol in ('\n', '\r'):
+ if (bad_eol in self.raw_response
+ # verify that bad_eol is not the end of the incoming data
+ # as this could be a response line that just got
+ # split between \r and \n.
+ and (self.raw_response.index(bad_eol) <
+ (len(self.raw_response) - 1))):
+ logger.info('bogus line endings detected, '
+ 'using %r for EOL', bad_eol)
+ self._eol = bad_eol
+ break
+ # exit early if not at end of headers
+ if self._end_headers not in self.raw_response or self.headers:
+ return
+
+ # handle 100-continue response
+ hdrs, body = self.raw_response.split(self._end_headers, 1)
+ http_ver, status = hdrs.split(' ', 1)
+ if status.startswith('100'):
+ self.raw_response = body
+ logger.debug('continue seen, setting body to %r', body)
+ return
+
+ # arriving here means we should parse response headers
+ # as all headers have arrived completely
+ hdrs, body = self.raw_response.split(self._end_headers, 1)
+ del self.raw_response
+ if self._eol in hdrs:
+ self.status_line, hdrs = hdrs.split(self._eol, 1)
+ else:
+ self.status_line = hdrs
+ hdrs = ''
+ # TODO HTTP < 1.0 support
+ (self.http_version, self.status,
+ self.reason) = self.status_line.split(' ', 2)
+ self.status = int(self.status)
+ if self._eol != EOL:
+ hdrs = hdrs.replace(self._eol, '\r\n')
+ headers = rfc822.Message(cStringIO.StringIO(hdrs))
+ if HDR_CONTENT_LENGTH in headers:
+ self._content_len = int(headers[HDR_CONTENT_LENGTH])
+ if HDR_CONNECTION_CTRL in headers:
+ self.will_close = (
+ headers[HDR_CONNECTION_CTRL].lower() == CONNECTION_CLOSE)
+ if self._content_len == 0:
+ self._content_len = _LEN_CLOSE_IS_END
+ if (HDR_XFER_ENCODING in headers
+ and headers[HDR_XFER_ENCODING].lower() == XFER_ENCODING_CHUNKED):
+ self._body = ''
+ self._chunked_parsedata(body)
+ self._chunked = True
+ if self._body is None:
+ self._body = body
+ self.headers = headers
+
+
+class HTTPConnection(object):
+ """Connection to a single http server.
+
+ Supports 100-continue and keepalives natively. Uses select() for
+ non-blocking socket operations.
+ """
+ http_version = HTTP_VER_1_1
+ response_class = HTTPResponse
+
+ def __init__(self, host, port=None, use_ssl=None, timeout=TIMEOUT_DEFAULT,
+ continue_timeout=TIMEOUT_ASSUME_CONTINUE,
+ proxy_hostport=None, **ssl_opts):
+ """Create a new HTTPConnection.
+
+ Args:
+ host: The host to which we'll connect.
+ port: Optional. The port over which we'll connect. Default 80 for
+ non-ssl, 443 for ssl.
+ use_ssl: Optional. Wether to use ssl. Defaults to False if port is
+ not 443, true if port is 443.
+ timeout: Optional. Connection timeout, default is TIMEOUT_DEFAULT.
+ continue_timeout: Optional. Timeout for waiting on an expected
+ "100 Continue" response. Default is TIMEOUT_ASSUME_CONTINUE.
+ proxy_hostport: Optional. Tuple of (host, port) to use as an http
+ proxy for the connection. Default is to not use a proxy.
+ """
+ if port is None and host.count(':') == 1 or ']:' in host:
+ host, port = host.rsplit(':', 1)
+ port = int(port)
+ if '[' in host:
+ host = host[1:-1]
+ if use_ssl is None and port is None:
+ use_ssl = False
+ port = 80
+ elif use_ssl is None:
+ use_ssl = (port == 443)
+ elif port is None:
+ port = (use_ssl and 443 or 80)
+ self.port = port
+ if use_ssl and not socketutil.have_ssl:
+ raise Exception('ssl requested but unavailable on this Python')
+ self.ssl = use_ssl
+ self.ssl_opts = ssl_opts
+ self.host = host
+ self.sock = None
+ self._current_response = None
+ if proxy_hostport is None:
+ self._proxy_host = self._proxy_port = None
+ else:
+ self._proxy_host, self._proxy_port = proxy_hostport
+
+ self.timeout = timeout
+ self.continue_timeout = continue_timeout
+
+ def _connect(self):
+ """Connect to the host and port specified in __init__."""
+ if self.sock:
+ return
+ if self._proxy_host is not None:
+ sock = socketutil.create_connection((self._proxy_host,
+ self._proxy_port))
+ if self.ssl:
+ # TODO proxy header support
+ sock.send('CONNECT %s:%s %s\r\n\r\n' %
+ (self.host, self.port, HTTP_VER_1_0))
+ self.sock = sock
+ else:
+ sock = socketutil.create_connection((self.host, self.port))
+ if self.ssl:
+ sock = socketutil.wrap_socket(sock, **self.ssl_opts)
+ sock.setblocking(0)
+ self.sock = sock
+
+ def buildheaders(self, method, url, headers):
+ self._connect()
+ outgoing = ['%s %s %s%s' % (method, url, self.http_version, EOL)]
+ headers['host'] = ('Host', self.host)
+ headers[HDR_ACCEPT_ENCODING] = (HDR_ACCEPT_ENCODING, 'identity')
+ for hdr, val in headers.itervalues():
+ outgoing.append('%s: %s%s' % (hdr, val, EOL))
+ outgoing.append(EOL)
+ return ''.join(outgoing)
+
+ def close(self):
+ """Close the connection to the server.
+
+ This is a no-op if the connection is already closed. The
+ connection may automatically close if requessted by the server
+ or required by the nature of a response.
+ """
+ if self.sock is None:
+ return
+ self.sock.close()
+ self.sock = None
+ logger.info('closed connection to %s on %s', self.host, self.port)
+
+ def busy(self):
+ """Returns True if this connection object is currently in use.
+
+ If a response is still pending, this will return True, even if
+ the request has finished sending. In the future,
+ HTTPConnection may transparently juggle multiple connections
+ to the server, in which case this will be useful to detect if
+ any of those connections is ready for use.
+ """
+ return not (self._current_response is None
+ or self._current_response.complete())
+
+ def request(self, method, url, body=None, headers={},
+ expect_continue=False):
+ """Send a request to the server.
+
+ For increased flexibility, this does not return the response
+ object. Future versions of HTTPConnection that juggle multiple
+ sockets will be able to send (for example) 5 requests all at
+ once, and then let the requests arrive as data is
+ available. Use the `getresponse()` method to retrieve the
+ response.
+ """
+ cr = self._current_response
+ if cr is not None:
+ if cr.complete():
+ self._current_response = None
+ if cr.will_close:
+ self.close()
+ else:
+ raise httplib.CannotSendRequest()
+
+ logger.info('sending %s request for %s to %s on port %s',
+ method, url, self.host, self.port)
+ hdrs = dict((k.lower(), (k, v)) for k, v in headers.iteritems())
+ if hdrs.get('expect', ('', ''))[1].lower() == '100-continue':
+ expect_continue = True
+ elif expect_continue:
+ hdrs['expect'] = ('Expect', '100-Continue')
+
+ chunked = False
+ if body and HDR_CONTENT_LENGTH not in hdrs:
+ if getattr(body, '__len__', False):
+ hdrs[HDR_CONTENT_LENGTH] = (HDR_CONTENT_LENGTH, len(body))
+ elif getattr(body, 'read', False):
+ hdrs[HDR_XFER_ENCODING] = (HDR_XFER_ENCODING,
+ XFER_ENCODING_CHUNKED)
+ chunked = True
+ else:
+ raise BadRequestData('body has no __len__() nor read()')
+
+ outgoing_headers = self.buildheaders(method, url, hdrs)
+ response = None
+ first = True
+
+ def reconnect(where):
+ logger.info('reconnecting during %s', where)
+ self.close()
+ self._connect()
+
+ while ((outgoing_headers or body)
+ and not (response and response.complete())):
+ select_timeout = self.timeout
+ out = outgoing_headers or body
+ blocking_on_continue = False
+ if expect_continue and not outgoing_headers and not (
+ response and response.headers):
+ logger.info('waiting for continue response from server')
+ select_timeout = self.continue_timeout
+ blocking_on_continue = True
+ out = False
+ if out:
+ w = [self.sock]
+ else:
+ w = []
+ r, w, x = select.select([self.sock], w, [], select_timeout)
+ # if we were expecting a 100 continue and it's been long
+ # enough, just go ahead and assume it's ok. This is the
+ # recommended behavior from the RFC.
+ if r == w == x == []:
+ if blocking_on_continue:
+ expect_continue = False
+ logger.info('no response to continue expectation from '
+ 'server, optimistically sending request body')
+ else:
+ raise HTTPTimeoutException('timeout sending data')
+ # TODO exceptional conditions with select? (what are those be?)
+ # TODO if the response is loading, must we finish sending at all?
+ #
+ # Certainly not if it's going to close the connection and/or
+ # the response is already done...I think.
+ was_first = first
+
+ # outgoing data
+ if w and out:
+ try:
+ if getattr(out, 'read', False):
+ data = out.read(OUTGOING_BUFFER_SIZE)
+ if not data:
+ continue
+ if len(data) < OUTGOING_BUFFER_SIZE:
+ if chunked:
+ body = '0' + EOL + EOL
+ else:
+ body = None
+ if chunked:
+ out = hex(len(data))[2:] + EOL + data + EOL
+ else:
+ out = data
+ amt = w[0].send(out)
+ except socket.error, e:
+ if e[0] == socket.SSL_ERROR_WANT_WRITE and self.ssl:
+ # This means that SSL hasn't flushed its buffer into
+ # the socket yet.
+ # TODO: find a way to block on ssl flushing its buffer
+ # similar to selecting on a raw socket.
+ continue
+ elif (e[0] not in (errno.ECONNRESET, errno.EPIPE)
+ and not first):
+ raise
+ reconnect('write')
+ amt = self.sock.send(out)
+ logger.debug('sent %d', amt)
+ first = False
+ # stash data we think we sent in case the socket breaks
+ # when we read from it
+ if was_first:
+ sent_data = out[:amt]
+ if out is body:
+ body = out[amt:]
+ else:
+ outgoing_headers = out[amt:]
+
+ # incoming data
+ if r:
+ try:
+ data = r[0].recv(INCOMING_BUFFER_SIZE)
+ if not data:
+ logger.info('socket appears closed in read')
+ outgoing_headers = body = None
+ break
+ if response is None:
+ response = self.response_class(r[0], self.timeout)
+ response._load_response(data)
+ if (response._content_len == _LEN_CLOSE_IS_END
+ and len(data) == 0):
+ response._content_len = len(response._body)
+ except socket.error, e:
+ if e[0] != errno.EPIPE and not was_first:
+ raise
+ if (response._content_len
+ and response._content_len != _LEN_CLOSE_IS_END):
+ outgoing_headers = sent_data + outgoing_headers
+ reconnect('read')
+
+ # close if the server response said to or responded before eating
+ # the whole request
+ if response is None:
+ response = self.response_class(self.sock, self.timeout)
+ complete = response.complete()
+ data_left = bool(outgoing_headers or body)
+ if data_left:
+ logger.info('stopped sending request early, '
+ 'will close the socket to be safe.')
+ response.will_close = True
+ if response.will_close and complete:
+ self.close()
+ self._current_response = response
+
+ def getresponse(self):
+ if self._current_response is None:
+ raise httplib.ResponseNotReady()
+ r = self._current_response
+ if r.complete():
+ self._current_response = None
+ while r.headers is None:
+ r._select()
+ return r
+
+class HTTPTimeoutException(httplib.HTTPException):
+ """A timeout occurred while waiting on the server."""
+
+class BadRequestData(httplib.HTTPException):
+ """Request body object has neither __len__ nor read."""
diff --git a/hgext/http2/http/socketutil.py b/hgext/http2/http/socketutil.py
new file mode 100644
--- /dev/null
+++ b/hgext/http2/http/socketutil.py
@@ -0,0 +1,125 @@
+# Copyright 2010, Google Inc.
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+# * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+# * Redistributions in binary form must reproduce the above
+# copyright notice, this list of conditions and the following disclaimer
+# in the documentation and/or other materials provided with the
+# distribution.
+# * Neither the name of Google Inc. nor the names of its
+# contributors may be used to endorse or promote products derived from
+# this software without specific prior written permission.
+
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+"""Abstraction to simplify socket use for Python < 2.6
+
+This will attempt to use the ssl module and the new
+socket.create_connection method, but fall back to the old
+methods if those are unavailable.
+"""
+import logging
+import socket
+
+logger = logging.getLogger(__name__)
+
+try:
+ import ssl
+ ssl.wrap_socket # make demandimporters load the module
+ have_ssl = True
+except ImportError:
+ import httplib
+ import urllib2
+ have_ssl = getattr(urllib2, 'HTTPSHandler', False)
+ ssl = False
+
+
+try:
+ create_connection = socket.create_connection
+except AttributeError:
+ def create_connection(address):
+ host, port = address
+ msg = "getaddrinfo returns an empty list"
+ sock = None
+ for res in socket.getaddrinfo(host, port, 0,
+ socket.SOCK_STREAM):
+ af, socktype, proto, _canonname, sa = res
+ try:
+ sock = socket.socket(af, socktype, proto)
+ logger.info("connect: (%s, %s)", host, port)
+ sock.connect(sa)
+ except socket.error, msg:
+ logger.info('connect fail: %s %s', host, port)
+ if sock:
+ sock.close()
+ sock = None
+ continue
+ break
+ if not sock:
+ raise socket.error, msg
+ return sock
+
+if ssl:
+ wrap_socket = ssl.wrap_socket
+ CERT_NONE = ssl.CERT_NONE
+ CERT_OPTIONAL = ssl.CERT_OPTIONAL
+ CERT_REQUIRED = ssl.CERT_REQUIRED
+ PROTOCOL_SSLv2 = ssl.PROTOCOL_SSLv2
+ PROTOCOL_SSLv3 = ssl.PROTOCOL_SSLv3
+ PROTOCOL_SSLv23 = ssl.PROTOCOL_SSLv23
+ PROTOCOL_TLSv1 = ssl.PROTOCOL_TLSv1
+else:
+ class FakeSocket(httplib.FakeSocket):
+ """Socket wrapper that supports SSL.
+ """
+ # backport the behavior from Python 2.6, which is to busy wait
+ # on the socket instead of anything nice. Sigh.
+ # See http://bugs.python.org/issue3890 for more info.
+ def recv(self, buflen = 1024, flags = 0):
+ """ssl-aware wrapper around socket.recv
+ """
+ if flags != 0:
+ raise ValueError(
+ "non-zero flags not allowed in calls to recv() on %s" %
+ self.__class__)
+ while True:
+ try:
+ return self._ssl.read(buflen)
+ except socket.sslerror, x:
+ if x.args[0] == socket.SSL_ERROR_WANT_READ:
+ continue
+ else:
+ raise x
+
+ PROTOCOL_SSLv2 = 0
+ PROTOCOL_SSLv3 = 1
+ PROTOCOL_SSLv23 = 2
+ PROTOCOL_TLSv1 = 3
+
+ CERT_NONE = 0
+ CERT_OPTIONAL = 1
+ CERT_REQUIRED = 2
+
+ def wrap_socket(sock, keyfile=None, certfile=None,
+ server_side=False, cert_reqs=CERT_NONE,
+ ssl_version=PROTOCOL_SSLv23, ca_certs=None,
+ do_handshake_on_connect=True,
+ suppress_ragged_eofs=True):
+ sslob = socket.ssl(sock)
+ # borrow httplib's workaround for no ssl.wrap_socket
+ sock = FakeSocket(sock, sslob)
+ return sock
diff --git a/hgext/http2/http/tests/__init__.py b/hgext/http2/http/tests/__init__.py
new file mode 100644
diff --git a/hgext/http2/http/tests/simple_http_test.py b/hgext/http2/http/tests/simple_http_test.py
new file mode 100644
--- /dev/null
+++ b/hgext/http2/http/tests/simple_http_test.py
@@ -0,0 +1,322 @@
+# Copyright 2010, Google Inc.
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+# * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+# * Redistributions in binary form must reproduce the above
+# copyright notice, this list of conditions and the following disclaimer
+# in the documentation and/or other materials provided with the
+# distribution.
+# * Neither the name of Google Inc. nor the names of its
+# contributors may be used to endorse or promote products derived from
+# this software without specific prior written permission.
+
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+import unittest
+
+import http
+
+# relative import to ease embedding the library
+import util
+
+
+class SimpleHttpTest(util.HttpTestBase, unittest.TestCase):
+
+ def _run_simple_test(self, host, server_data, expected_req, expected_data):
+ con = http.HTTPConnection(host)
+ con._connect()
+ con.sock.data = server_data
+ con.request('GET', '/')
+
+ self.assertEqual(expected_req, con.sock.sent)
+ self.assertEqual(expected_data, con.getresponse().read())
+
+ def test_broken_data_obj(self):
+ con = http.HTTPConnection('1.2.3.4:80')
+ con._connect()
+ self.assertRaises(http.BadRequestData,
+ con.request, 'POST', '/', body=1)
+
+ def test_multiline_header(self):
+ con = http.HTTPConnection('1.2.3.4:80')
+ con._connect()
+ con.sock.data = ['HTTP/1.1 200 OK\r\n',
+ 'Server: BogusServer 1.0\r\n',
+ 'Multiline: Value\r\n',
+ ' Rest of value\r\n',
+ 'Content-Length: 10\r\n',
+ '\r\n'
+ '1234567890'
+ ]
+ con.request('GET', '/')
+
+ expected_req = ('GET / HTTP/1.1\r\n'
+ 'Host: 1.2.3.4\r\n'
+ 'accept-encoding: identity\r\n\r\n')
+
+ self.assertEqual(('1.2.3.4', 80), con.sock.sa)
+ self.assertEqual(expected_req, con.sock.sent)
+ resp = con.getresponse()
+ self.assertEqual('1234567890', resp.read())
+ self.assertEqual(['Value\n Rest of value'],
+ resp.headers.getheaders('multiline'))
+
+ def testSimpleRequest(self):
+ con = http.HTTPConnection('1.2.3.4:80')
+ con._connect()
+ con.sock.data = ['HTTP/1.1 200 OK\r\n',
+ 'Server: BogusServer 1.0\r\n',
+ 'MultiHeader: Value\r\n'
+ 'MultiHeader: Other Value\r\n'
+ 'MultiHeader: One More!\r\n'
+ 'Content-Length: 10\r\n',
+ '\r\n'
+ '1234567890'
+ ]
+ con.request('GET', '/')
+
+ expected_req = ('GET / HTTP/1.1\r\n'
+ 'Host: 1.2.3.4\r\n'
+ 'accept-encoding: identity\r\n\r\n')
+
+ self.assertEqual(('1.2.3.4', 80), con.sock.sa)
+ self.assertEqual(expected_req, con.sock.sent)
+ resp = con.getresponse()
+ self.assertEqual('1234567890', resp.read())
+ self.assertEqual(['Value', 'Other Value', 'One More!'],
+ resp.headers.getheaders('multiheader'))
+ self.assertEqual(['BogusServer 1.0'],
+ resp.headers.getheaders('server'))
+
+ def testHeaderlessResponse(self):
+ con = http.HTTPConnection('1.2.3.4', use_ssl=False)
+ con._connect()
+ con.sock.data = ['HTTP/1.1 200 OK\r\n',
+ '\r\n'
+ '1234567890'
+ ]
+ con.request('GET', '/')
+
+ expected_req = ('GET / HTTP/1.1\r\n'
+ 'Host: 1.2.3.4\r\n'
+ 'accept-encoding: identity\r\n\r\n')
+
+ self.assertEqual(('1.2.3.4', 80), con.sock.sa)
+ self.assertEqual(expected_req, con.sock.sent)
+ resp = con.getresponse()
+ self.assertEqual('1234567890', resp.read())
+ self.assertEqual({}, dict(resp.headers))
+ self.assertEqual(resp.status, 200)
+
+ def testReadline(self):
+ con = http.HTTPConnection('1.2.3.4')
+ con._connect()
+ # make sure it trickles in one byte at a time
+ # so that we touch all the cases in readline
+ con.sock.data = list(''.join(
+ ['HTTP/1.1 200 OK\r\n',
+ 'Server: BogusServer 1.0\r\n',
+ 'Connection: Close\r\n',
+ '\r\n'
+ '1\n2\nabcdefg\n4\n5']))
+
+ expected_req = ('GET / HTTP/1.1\r\n'
+ 'Host: 1.2.3.4\r\n'
+ 'accept-encoding: identity\r\n\r\n')
+
+ con.request('GET', '/')
+ self.assertEqual(('1.2.3.4', 80), con.sock.sa)
+ self.assertEqual(expected_req, con.sock.sent)
+ r = con.getresponse()
+ for expected in ['1\n', '2\n', 'abcdefg\n', '4\n', '5']:
+ actual = r.readline()
+ self.assertEqual(expected, actual,
+ 'Expected %r, got %r' % (expected, actual))
+
+ def testIPv6(self):
+ self._run_simple_test('[::1]:8221',
+ ['HTTP/1.1 200 OK\r\n',
+ 'Server: BogusServer 1.0\r\n',
+ 'Content-Length: 10',
+ '\r\n\r\n'
+ '1234567890'],
+ ('GET / HTTP/1.1\r\n'
+ 'Host: ::1\r\n'
+ 'accept-encoding: identity\r\n\r\n'),
+ '1234567890')
+ self._run_simple_test('::2',
+ ['HTTP/1.1 200 OK\r\n',
+ 'Server: BogusServer 1.0\r\n',
+ 'Content-Length: 10',
+ '\r\n\r\n'
+ '1234567890'],
+ ('GET / HTTP/1.1\r\n'
+ 'Host: ::2\r\n'
+ 'accept-encoding: identity\r\n\r\n'),
+ '1234567890')
+ self._run_simple_test('[::3]:443',
+ ['HTTP/1.1 200 OK\r\n',
+ 'Server: BogusServer 1.0\r\n',
+ 'Content-Length: 10',
+ '\r\n\r\n'
+ '1234567890'],
+ ('GET / HTTP/1.1\r\n'
+ 'Host: ::3\r\n'
+ 'accept-encoding: identity\r\n\r\n'),
+ '1234567890')
+
+ def doPost(self, con, expect_body, body_to_send='This is some POST data'):
+ con.request('POST', '/', body=body_to_send,
+ expect_continue=True)
+ expected_req = ('POST / HTTP/1.1\r\n'
+ 'Host: 1.2.3.4\r\n'
+ 'content-length: %d\r\n'
+ 'Expect: 100-Continue\r\n'
+ 'accept-encoding: identity\r\n\r\n' % len(body_to_send))
+ if expect_body:
+ expected_req += body_to_send
+ return expected_req
+
+ def testEarlyContinueResponse(self):
+ con = http.HTTPConnection('1.2.3.4:80')
+ con._connect()
+ sock = con.sock
+ sock.data = ['HTTP/1.1 403 Forbidden\r\n',
+ 'Server: BogusServer 1.0\r\n',
+ 'Content-Length: 18',
+ '\r\n\r\n'
+ "You can't do that."]
+ expected_req = self.doPost(con, expect_body=False)
+ self.assertEqual(('1.2.3.4', 80), sock.sa)
+ self.assertStringEqual(expected_req, sock.sent)
+ self.assertEqual("You can't do that.", con.getresponse().read())
+ self.assertEqual(sock.closed, True)
+
+ def testDeniedAfterContinueTimeoutExpires(self):
+ con = http.HTTPConnection('1.2.3.4:80')
+ con._connect()
+ sock = con.sock
+ sock.data = ['HTTP/1.1 403 Forbidden\r\n',
+ 'Server: BogusServer 1.0\r\n',
+ 'Content-Length: 18\r\n',
+ 'Connection: close',
+ '\r\n\r\n'
+ "You can't do that."]
+ sock.read_wait_sentinel = 'Dear server, send response!'
+ sock.close_on_empty = True
+ # send enough data out that we'll chunk it into multiple
+ # blocks and the socket will close before we can send the
+ # whole request.
+ post_body = ('This is some POST data\n' * 1024 * 32 +
+ 'Dear server, send response!\n' +
+ 'This is some POST data\n' * 1024 * 32)
+ expected_req = self.doPost(con, expect_body=False,
+ body_to_send=post_body)
+ self.assertEqual(('1.2.3.4', 80), sock.sa)
+ self.assert_('POST data\n' in sock.sent)
+ self.assert_('Dear server, send response!\n' in sock.sent)
+ # We expect not all of our data was sent.
+ self.assertNotEqual(sock.sent, expected_req)
+ self.assertEqual("You can't do that.", con.getresponse().read())
+ self.assertEqual(sock.closed, True)
+
+ def testPostData(self):
+ con = http.HTTPConnection('1.2.3.4:80')
+ con._connect()
+ sock = con.sock
+ sock.read_wait_sentinel = 'POST data'
+ sock.early_data = ['HTTP/1.1 100 Co', 'ntinue\r\n\r\n']
+ sock.data = ['HTTP/1.1 200 OK\r\n',
+ 'Server: BogusServer 1.0\r\n',
+ 'Content-Length: 16',
+ '\r\n\r\n',
+ "You can do that."]
+ expected_req = self.doPost(con, expect_body=True)
+ self.assertEqual(('1.2.3.4', 80), sock.sa)
+ self.assertEqual(expected_req, sock.sent)
+ self.assertEqual("You can do that.", con.getresponse().read())
+ self.assertEqual(sock.closed, False)
+
+ def testServerWithoutContinue(self):
+ con = http.HTTPConnection('1.2.3.4:80')
+ con._connect()
+ sock = con.sock
+ sock.read_wait_sentinel = 'POST data'
+ sock.data = ['HTTP/1.1 200 OK\r\n',
+ 'Server: BogusServer 1.0\r\n',
+ 'Content-Length: 16',
+ '\r\n\r\n',
+ "You can do that."]
+ expected_req = self.doPost(con, expect_body=True)
+ self.assertEqual(('1.2.3.4', 80), sock.sa)
+ self.assertEqual(expected_req, sock.sent)
+ self.assertEqual("You can do that.", con.getresponse().read())
+ self.assertEqual(sock.closed, False)
+
+ def testServerWithSlowContinue(self):
+ con = http.HTTPConnection('1.2.3.4:80')
+ con._connect()
+ sock = con.sock
+ sock.read_wait_sentinel = 'POST data'
+ sock.data = ['HTTP/1.1 100 ', 'Continue\r\n\r\n',
+ 'HTTP/1.1 200 OK\r\n',
+ 'Server: BogusServer 1.0\r\n',
+ 'Content-Length: 16',
+ '\r\n\r\n',
+ "You can do that."]
+ expected_req = self.doPost(con, expect_body=True)
+ self.assertEqual(('1.2.3.4', 80), sock.sa)
+ self.assertEqual(expected_req, sock.sent)
+ resp = con.getresponse()
+ self.assertEqual("You can do that.", resp.read())
+ self.assertEqual(200, resp.status)
+ self.assertEqual(sock.closed, False)
+
+ def testSlowConnection(self):
+ con = http.HTTPConnection('1.2.3.4:80')
+ con._connect()
+ # simulate one byte arriving at a time, to check for various
+ # corner cases
+ con.sock.data = list('HTTP/1.1 200 OK\r\n'
+ 'Server: BogusServer 1.0\r\n'
+ 'Content-Length: 10'
+ '\r\n\r\n'
+ '1234567890')
+ con.request('GET', '/')
+
+ expected_req = ('GET / HTTP/1.1\r\n'
+ 'Host: 1.2.3.4\r\n'
+ 'accept-encoding: identity\r\n\r\n')
+
+ self.assertEqual(('1.2.3.4', 80), con.sock.sa)
+ self.assertEqual(expected_req, con.sock.sent)
+ self.assertEqual('1234567890', con.getresponse().read())
+
+ def testTimeout(self):
+ con = http.HTTPConnection('1.2.3.4:80')
+ con._connect()
+ con.sock.data = []
+ con.request('GET', '/')
+ self.assertRaises(http.HTTPTimeoutException,
+ con.getresponse)
+
+ expected_req = ('GET / HTTP/1.1\r\n'
+ 'Host: 1.2.3.4\r\n'
+ 'accept-encoding: identity\r\n\r\n')
+
+ self.assertEqual(('1.2.3.4', 80), con.sock.sa)
+ self.assertEqual(expected_req, con.sock.sent)
diff --git a/hgext/http2/http/tests/test_bogus_responses.py b/hgext/http2/http/tests/test_bogus_responses.py
new file mode 100644
--- /dev/null
+++ b/hgext/http2/http/tests/test_bogus_responses.py
@@ -0,0 +1,67 @@
+# Copyright 2010, Google Inc.
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+# * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+# * Redistributions in binary form must reproduce the above
+# copyright notice, this list of conditions and the following disclaimer
+# in the documentation and/or other materials provided with the
+# distribution.
+# * Neither the name of Google Inc. nor the names of its
+# contributors may be used to endorse or promote products derived from
+# this software without specific prior written permission.
+
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+"""Tests against malformed responses.
+
+Server implementations that respond with only LF instead of CRLF have
+been observed. Checking against ones that use only CR is a hedge
+against that potential insanit.y
+"""
+import unittest
+
+import http
+
+# relative import to ease embedding the library
+import util
+
+
+class SimpleHttpTest(util.HttpTestBase, unittest.TestCase):
+
+ def bogusEOL(self, eol):
+ con = http.HTTPConnection('1.2.3.4:80')
+ con._connect()
+ con.sock.data = ['HTTP/1.1 200 OK%s' % eol,
+ 'Server: BogusServer 1.0%s' % eol,
+ 'Content-Length: 10',
+ eol * 2,
+ '1234567890']
+ con.request('GET', '/')
+
+ expected_req = ('GET / HTTP/1.1\r\n'
+ 'Host: 1.2.3.4\r\n'
+ 'accept-encoding: identity\r\n\r\n')
+
+ self.assertEqual(('1.2.3.4', 80), con.sock.sa)
+ self.assertEqual(expected_req, con.sock.sent)
+ self.assertEqual('1234567890', con.getresponse().read())
+
+ def testOnlyLinefeed(self):
+ self.bogusEOL('\n')
+
+ def testOnlyCarriageReturn(self):
+ self.bogusEOL('\r')
diff --git a/hgext/http2/http/tests/test_chunked_transfer.py b/hgext/http2/http/tests/test_chunked_transfer.py
new file mode 100644
--- /dev/null
+++ b/hgext/http2/http/tests/test_chunked_transfer.py
@@ -0,0 +1,134 @@
+# Copyright 2010, Google Inc.
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+# * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+# * Redistributions in binary form must reproduce the above
+# copyright notice, this list of conditions and the following disclaimer
+# in the documentation and/or other materials provided with the
+# distribution.
+# * Neither the name of Google Inc. nor the names of its
+# contributors may be used to endorse or promote products derived from
+# this software without specific prior written permission.
+
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+import cStringIO
+import unittest
+
+import http
+
+# relative import to ease embedding the library
+import util
+
+
+def chunkedblock(x, eol='\r\n'):
+ r"""Make a chunked transfer-encoding block.
+
+ >>> chunkedblock('hi')
+ '2\r\nhi\r\n'
+ >>> chunkedblock('hi' * 10)
+ '14\r\nhihihihihihihihihihi\r\n'
+ >>> chunkedblock('hi', eol='\n')
+ '2\nhi\n'
+ """
+ return ''.join((hex(len(x))[2:], eol, x, eol))
+
+
+class ChunkedTransferTest(util.HttpTestBase, unittest.TestCase):
+ def testChunkedUpload(self):
+ con = http.HTTPConnection('1.2.3.4:80')
+ con._connect()
+ sock = con.sock
+ sock.read_wait_sentinel = 'end-of-body'
+ sock.data = ['HTTP/1.1 200 OK\r\n',
+ 'Server: BogusServer 1.0\r\n',
+ 'Content-Length: 6',
+ '\r\n\r\n',
+ "Thanks"]
+
+ zz = 'zz\n'
+ con.request('POST', '/', body=cStringIO.StringIO((zz * (0x8010/3)) + 'end-of-body'))
+ expected_req = ('POST / HTTP/1.1\r\n'
+ 'transfer-encoding: chunked\r\n'
+ 'Host: 1.2.3.4\r\n'
+ 'accept-encoding: identity\r\n\r\n')
+ expected_req += chunkedblock('zz\n' * (0x8000 / 3) + 'zz')
+ expected_req += chunkedblock('\n' + 'zz\n' * ((0x1b - len('end-of-body')) / 3)
+ + 'end-of-body')
+ expected_req += '0\r\n\r\n'
+ self.assertEqual(('1.2.3.4', 80), sock.sa)
+ self.assertStringEqual(expected_req, sock.sent)
+ self.assertEqual("Thanks", con.getresponse().read())
+ self.assertEqual(sock.closed, False)
+
+ def testChunkedDownload(self):
+ con = http.HTTPConnection('1.2.3.4:80')
+ con._connect()
+ sock = con.sock
+ sock.data = ['HTTP/1.1 200 OK\r\n',
+ 'Server: BogusServer 1.0\r\n',
+ 'transfer-encoding: chunked',
+ '\r\n\r\n',
+ chunkedblock('hi '),
+ chunkedblock('there'),
+ chunkedblock(''),
+ ]
+ con.request('GET', '/')
+ self.assertStringEqual('hi there', con.getresponse().read())
+
+ def testChunkedDownloadBadEOL(self):
+ con = http.HTTPConnection('1.2.3.4:80')
+ con._connect()
+ sock = con.sock
+ sock.data = ['HTTP/1.1 200 OK\n',
+ 'Server: BogusServer 1.0\n',
+ 'transfer-encoding: chunked',
+ '\n\n',
+ chunkedblock('hi ', eol='\n'),
+ chunkedblock('there', eol='\n'),
+ chunkedblock('', eol='\n'),
+ ]
+ con.request('GET', '/')
+ self.assertStringEqual('hi there', con.getresponse().read())
+
+ def testChunkedDownloadPartialChunkBadEOL(self):
+ con = http.HTTPConnection('1.2.3.4:80')
+ con._connect()
+ sock = con.sock
+ sock.data = ['HTTP/1.1 200 OK\n',
+ 'Server: BogusServer 1.0\n',
+ 'transfer-encoding: chunked',
+ '\n\n',
+ chunkedblock('hi ', eol='\n'),
+ ] + list(chunkedblock('there\n' * 5, eol='\n')) + [chunkedblock('', eol='\n')]
+ con.request('GET', '/')
+ self.assertStringEqual('hi there\nthere\nthere\nthere\nthere\n',
+ con.getresponse().read())
+
+ def testChunkedDownloadPartialChunk(self):
+ con = http.HTTPConnection('1.2.3.4:80')
+ con._connect()
+ sock = con.sock
+ sock.data = ['HTTP/1.1 200 OK\r\n',
+ 'Server: BogusServer 1.0\r\n',
+ 'transfer-encoding: chunked',
+ '\r\n\r\n',
+ chunkedblock('hi '),
+ ] + list(chunkedblock('there\n' * 5)) + [chunkedblock('')]
+ con.request('GET', '/')
+ self.assertStringEqual('hi there\nthere\nthere\nthere\nthere\n',
+ con.getresponse().read())
diff --git a/hgext/http2/http/tests/test_proxy_support.py b/hgext/http2/http/tests/test_proxy_support.py
new file mode 100644
--- /dev/null
+++ b/hgext/http2/http/tests/test_proxy_support.py
@@ -0,0 +1,115 @@
+# Copyright 2010, Google Inc.
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+# * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+# * Redistributions in binary form must reproduce the above
+# copyright notice, this list of conditions and the following disclaimer
+# in the documentation and/or other materials provided with the
+# distribution.
+# * Neither the name of Google Inc. nor the names of its
+# contributors may be used to endorse or promote products derived from
+# this software without specific prior written permission.
+
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+import unittest
+
+import http
+
+# relative import to ease embedding the library
+import util
+
+
+class ProxyHttpTest(util.HttpTestBase, unittest.TestCase):
+
+ def _run_simple_test(self, host, server_data, expected_req, expected_data):
+ con = http.HTTPConnection(host)
+ con._connect()
+ con.sock.data = server_data
+ con.request('GET', '/')
+
+ self.assertEqual(expected_req, con.sock.sent)
+ self.assertEqual(expected_data, con.getresponse().read())
+
+ def testSimpleRequest(self):
+ con = http.HTTPConnection('1.2.3.4:80', proxy_hostport=('magicproxy', 4242))
+ con._connect()
+ con.sock.data = ['HTTP/1.1 200 OK\r\n',
+ 'Server: BogusServer 1.0\r\n',
+ 'MultiHeader: Value\r\n'
+ 'MultiHeader: Other Value\r\n'
+ 'MultiHeader: One More!\r\n'
+ 'Content-Length: 10\r\n',
+ '\r\n'
+ '1234567890'
+ ]
+ con.request('GET', '/')
+
+ expected_req = ('GET / HTTP/1.1\r\n'
+ 'Host: 1.2.3.4\r\n'
+ 'accept-encoding: identity\r\n\r\n')
+
+ self.assertEqual(('127.0.0.42', 4242), con.sock.sa)
+ self.assertEqual(expected_req, con.sock.sent)
+ resp = con.getresponse()
+ self.assertEqual('1234567890', resp.read())
+ self.assertEqual(['Value', 'Other Value', 'One More!'],
+ resp.headers.getheaders('multiheader'))
+ self.assertEqual(['BogusServer 1.0'],
+ resp.headers.getheaders('server'))
+
+ def testSSLRequest(self):
+ con = http.HTTPConnection('1.2.3.4:443', proxy_hostport=('magicproxy', 4242))
+ con._connect()
+ con.sock.data = ['HTTP/1.1 100 Continue\r\n\r\n',
+ 'HTTP/1.1 200 OK\r\n',
+ 'Server: BogusServer 1.0\r\n',
+ 'Content-Length: 10\r\n',
+ '\r\n'
+ '1234567890'
+ ]
+ con.request('GET', '/')
+
+ expected_req = ('CONNECT 1.2.3.4:443 HTTP/1.0\r\n\r\n'
+ 'GET / HTTP/1.1\r\n'
+ 'Host: 1.2.3.4\r\n'
+ 'accept-encoding: identity\r\n\r\n')
+
+ self.assertEqual(('127.0.0.42', 4242), con.sock.sa)
+ self.assertStringEqual(expected_req, con.sock.sent)
+ resp = con.getresponse()
+ self.assertEqual(resp.status, 200)
+ self.assertEqual('1234567890', resp.read())
+ self.assertEqual(['BogusServer 1.0'],
+ resp.headers.getheaders('server'))
+
+ def testSSLProxyFailure(self):
+ con = http.HTTPConnection('1.2.3.4:443', proxy_hostport=('magicproxy', 4242))
+ con._connect()
+ con.sock.data = ['HTTP/1.1 407 Proxy Authentication Required\r\n\r\n',
+ ]
+ con.request('GET', '/')
+
+ expected_req = ('CONNECT 1.2.3.4:443 HTTP/1.0\r\n\r\n'
+ 'GET / HTTP/1.1\r\n'
+ 'Host: 1.2.3.4\r\n'
+ 'accept-encoding: identity\r\n\r\n')
+
+ self.assertEqual(('127.0.0.42', 4242), con.sock.sa)
+ self.assertStringEqual(expected_req, con.sock.sent)
+ resp = con.getresponse()
+ self.assertEqual(resp.status, 407)
diff --git a/hgext/http2/http/tests/util.py b/hgext/http2/http/tests/util.py
new file mode 100644
--- /dev/null
+++ b/hgext/http2/http/tests/util.py
@@ -0,0 +1,157 @@
+# Copyright 2010, Google Inc.
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+# * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+# * Redistributions in binary form must reproduce the above
+# copyright notice, this list of conditions and the following disclaimer
+# in the documentation and/or other materials provided with the
+# distribution.
+# * Neither the name of Google Inc. nor the names of its
+# contributors may be used to endorse or promote products derived from
+# this software without specific prior written permission.
+
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+import difflib
+import socket
+
+import http
+
+
+class MockSocket(object):
+ """Mock non-blocking socket object.
+
+ This is ONLY capable of mocking a nonblocking socket.
+
+ Attributes:
+ early_data: data to always send as soon as end of headers is seen
+ data: a list of strings to return on recv(), with the
+ assumption that the socket would block between each
+ string in the list.
+ read_wait_sentinel: data that must be written to the socket before
+ beginning the response.
+ close_on_empty: If true, close the socket when it runs out of data
+ for the client.
+ """
+ def __init__(self, af, socktype, proto):
+ self.af = af
+ self.socktype = socktype
+ self.proto = proto
+
+ self.early_data = []
+ self.data = []
+ self.remote_closed = self.closed = False
+ self.close_on_empty = False
+ self.sent = ''
+ self.read_wait_sentinel = http._END_HEADERS
+
+ def close(self):
+ self.closed = True
+
+ def connect(self, sa):
+ self.sa = sa
+
+ def setblocking(self, timeout):
+ assert timeout == 0
+
+ def recv(self, amt=-1):
+ if self.early_data:
+ datalist = self.early_data
+ elif not self.data:
+ return ''
+ else:
+ datalist = self.data
+ if amt == -1:
+ return datalist.pop(0)
+ data = datalist.pop(0)
+ if len(data) > amt:
+ datalist.insert(0, data[amt:])
+ if not self.data and not self.early_data and self.close_on_empty:
+ self.remote_closed = True
+ return data[:amt]
+
+ @property
+ def ready_for_read(self):
+ return ((self.early_data and http._END_HEADERS in self.sent)
+ or (self.read_wait_sentinel in self.sent and self.data)
+ or self.closed)
+
+ def send(self, data):
+ # this is a horrible mock, but nothing needs us to raise the
+ # correct exception yet
+ assert not self.closed, 'attempted to write to a closed socket'
+ assert not self.remote_closed, ('attempted to write to a'
+ ' socket closed by the server')
+ if len(data) > 8192:
+ data = data[:8192]
+ self.sent += data
+ return len(data)
+
+
+def mockselect(r, w, x, timeout=0):
+ """Simple mock for select()
+ """
+ readable = filter(lambda s: s.ready_for_read, r)
+ return readable, w[:], []
+
+def mocksslwrap(sock, keyfile=None, certfile=None,
+ server_side=False, cert_reqs=http.socketutil.CERT_NONE,
+ ssl_version=http.socketutil.PROTOCOL_SSLv23, ca_certs=None,
+ do_handshake_on_connect=True,
+ suppress_ragged_eofs=True):
+ return sock
+
+
+def mockgetaddrinfo(host, port, unused, streamtype):
+ assert unused == 0
+ assert streamtype == socket.SOCK_STREAM
+ if host.count('.') != 3:
+ host = '127.0.0.42'
+ return [(socket.AF_INET, socket.SOCK_STREAM, socket.IPPROTO_TCP, '',
+ (host, port))]
+
+
+class HttpTestBase(object):
+ def setUp(self):
+ self.orig_socket = socket.socket
+ socket.socket = MockSocket
+
+ self.orig_getaddrinfo = socket.getaddrinfo
+ socket.getaddrinfo = mockgetaddrinfo
+
+ self.orig_select = http.select.select
+ http.select.select = mockselect
+
+ self.orig_sslwrap = http.socketutil.wrap_socket
+ http.socketutil.wrap_socket = mocksslwrap
+
+ def tearDown(self):
+ socket.socket = self.orig_socket
+ http.select.select = self.orig_select
+ http.socketutil.wrap_socket = self.orig_sslwrap
+ socket.getaddrinfo = self.orig_getaddrinfo
+
+ def assertStringEqual(self, l, r):
+ try:
+ self.assertEqual(l, r, 'failed string equality check, see stdout for details')
+ except:
+ add_nl = lambda li: map(lambda x: x+'\n', li)
+ print 'failed expectation:'
+ print ''.join(difflib.unified_diff(
+ add_nl(l.splitlines()), add_nl(r.splitlines()),
+ fromfile='expected', tofile='got'))
+ raise
More information about the Mercurial-devel
mailing list