[PATCH] keepalive: handle broken pipes gracefully during large POSTs
Augie Fackler
durin42 at gmail.com
Mon Nov 2 14:04:11 CST 2009
This patch fixes 'abort: broken pipe' failures against servers that
can eagerly return a 401 to a push, since if the push is big enough
the connection will be closed by the server.
I'm working on a followup that will use non-blocking sockets to be
able to notice early responses more rapidly, and should provide
reasonably significant bandwidth savings on sufficiently large pushes.
On Mon, Nov 2, 2009 at 12:30 PM, <durin42 at gmail.com> wrote:
> # HG changeset patch
> # User Augie Fackler <durin42 at gmail.com>
> # Date 1257177802 18000
> # Node ID 38e263556f1dfd4071c84fc6415aa149b5195259
> # Parent 8269fe2d48f6caa7808242803d64b7415ba0745a
> keepalive: handle broken pipes gracefully during large POSTs
>
> diff --git a/mercurial/keepalive.py b/mercurial/keepalive.py
> --- a/mercurial/keepalive.py
> +++ b/mercurial/keepalive.py
> @@ -23,6 +23,9 @@
> # - import md5 function from a local util module
> # Modified by Martin Geisler:
> # - moved md5 function from local util module to this module
> +# Modified by Augie Fackler:
> +# - add safesend method and use it to prevent broken pipe errors
> +# on large POST requests
>
> """An HTTP handler for urllib2 that supports HTTP 1.1 and keepalive.
>
> @@ -108,10 +111,11 @@
>
> # $Id: keepalive.py,v 1.14 2006/04/04 21:00:32 mstenner Exp $
>
> -import urllib2
> +import errno
> import httplib
> import socket
> import thread
> +import urllib2
>
> DEBUG = None
>
> @@ -495,10 +499,62 @@
> break
> return list
>
> +def safesend(self, str):
> + """Send `str' to the server.
> +
> + Shamelessly ripped off from httplib to patch a bad behavior.
> + """
> + if getattr(self, '_broken_pipe_resp', None) is not None:
> + return
> +
> + if self.sock is None:
> + if self.auto_open:
> + self.connect()
> + else:
> + raise httplib.NotConnected()
> +
> + # send the data to the server. if we get a broken pipe, then close
> + # the socket. we want to reconnect when somebody tries to send again.
> + #
> + # NOTE: we DO propagate the error, though, because we cannot simply
> + # ignore the error... the caller will know if they can retry.
> + if self.debuglevel > 0:
> + print "send:", repr(str)
> + try:
> + blocksize=8192
> + if hasattr(str,'read') :
> + if self.debuglevel > 0: print "sendIng a read()able"
> + data=str.read(blocksize)
> + while data:
> + self.sock.sendall(data)
> + data=str.read(blocksize)
> + else:
> + self.sock.sendall(str)
> + except socket.error, v:
> + reraise = True
> + if v[0] == errno.EPIPE: # Broken pipe
> + if self._HTTPConnection__state == httplib._CS_REQ_SENT:
> + self._broken_pipe_resp = None
> + self._broken_pipe_resp = self.getresponse()
> + reraise = False
> + self.close()
> + if reraise:
> + raise
> +
> +def wrapgetresponse(cls):
> + def safegetresponse(self):
> + r = getattr(self, '_broken_pipe_resp', None)
> + if r is not None:
> + return r
> + return cls.getresponse(self)
> + return safegetresponse
>
> class HTTPConnection(httplib.HTTPConnection):
> # use the modified response class
> response_class = HTTPResponse
> + send = safesend
> + getresponse = wrapgetresponse(httplib.HTTPConnection)
> +
>
> #########################################################################
> ##### TEST FUNCTIONS
> diff --git a/mercurial/url.py b/mercurial/url.py
> --- a/mercurial/url.py
> +++ b/mercurial/url.py
> @@ -408,10 +408,14 @@
> self.close_all()
>
> if has_https:
> - class httpsconnection(httplib.HTTPSConnection):
> + class BetterHTTPS(httplib.HTTPSConnection):
> + send = keepalive.safesend
> +
> + class httpsconnection(BetterHTTPS):
> response_class = keepalive.HTTPResponse
> # must be able to send big bundle as stream.
> - send = _gen_sendfile(httplib.HTTPSConnection)
> + send = _gen_sendfile(BetterHTTPS)
> + getresponse = keepalive.wrapgetresponse(httplib.HTTPSConnection)
>
> class httpshandler(keepalive.KeepAliveHandler, urllib2.HTTPSHandler):
> def __init__(self, ui):
> _______________________________________________
> Mercurial-devel mailing list
> Mercurial-devel at selenic.com
> http://selenic.com/mailman/listinfo/mercurial-devel
>
More information about the Mercurial-devel
mailing list