[PATCH] keepalive: handle broken pipes gracefully during large POSTs

Augie Fackler durin42 at gmail.com
Mon Nov 2 14:04:11 CST 2009


This patch fixes 'abort: broken pipe' failures against servers that
can eagerly return a 401 to a push, since if the push is big enough
the connection will be closed by the server.

I'm working on a followup that will use non-blocking sockets to be
able to notice early responses more rapidly, and should provide
reasonably significant bandwidth savings on sufficiently large pushes.

On Mon, Nov 2, 2009 at 12:30 PM,  <durin42 at gmail.com> wrote:
> # HG changeset patch
> # User Augie Fackler <durin42 at gmail.com>
> # Date 1257177802 18000
> # Node ID 38e263556f1dfd4071c84fc6415aa149b5195259
> # Parent  8269fe2d48f6caa7808242803d64b7415ba0745a
> keepalive: handle broken pipes gracefully during large POSTs
>
> diff --git a/mercurial/keepalive.py b/mercurial/keepalive.py
> --- a/mercurial/keepalive.py
> +++ b/mercurial/keepalive.py
> @@ -23,6 +23,9 @@
>  #  - import md5 function from a local util module
>  # Modified by Martin Geisler:
>  #  - moved md5 function from local util module to this module
> +# Modified by Augie Fackler:
> +#  - add safesend method and use it to prevent broken pipe errors
> +#    on large POST requests
>
>  """An HTTP handler for urllib2 that supports HTTP 1.1 and keepalive.
>
> @@ -108,10 +111,11 @@
>
>  # $Id: keepalive.py,v 1.14 2006/04/04 21:00:32 mstenner Exp $
>
> -import urllib2
> +import errno
>  import httplib
>  import socket
>  import thread
> +import urllib2
>
>  DEBUG = None
>
> @@ -495,10 +499,62 @@
>                 break
>         return list
>
> +def safesend(self, str):
> +    """Send `str' to the server.
> +
> +    Shamelessly ripped off from httplib to patch a bad behavior.
> +    """
> +    if getattr(self, '_broken_pipe_resp', None) is not None:
> +        return
> +
> +    if self.sock is None:
> +        if self.auto_open:
> +            self.connect()
> +        else:
> +            raise httplib.NotConnected()
> +
> +    # send the data to the server. if we get a broken pipe, then close
> +    # the socket. we want to reconnect when somebody tries to send again.
> +    #
> +    # NOTE: we DO propagate the error, though, because we cannot simply
> +    #       ignore the error... the caller will know if they can retry.
> +    if self.debuglevel > 0:
> +        print "send:", repr(str)
> +    try:
> +        blocksize=8192
> +        if hasattr(str,'read') :
> +            if self.debuglevel > 0: print "sendIng a read()able"
> +            data=str.read(blocksize)
> +            while data:
> +                self.sock.sendall(data)
> +                data=str.read(blocksize)
> +        else:
> +            self.sock.sendall(str)
> +    except socket.error, v:
> +        reraise = True
> +        if v[0] == errno.EPIPE:      # Broken pipe
> +            if self._HTTPConnection__state == httplib._CS_REQ_SENT:
> +                self._broken_pipe_resp = None
> +                self._broken_pipe_resp = self.getresponse()
> +                reraise = False
> +            self.close()
> +        if reraise:
> +            raise
> +
> +def wrapgetresponse(cls):
> +    def safegetresponse(self):
> +        r = getattr(self, '_broken_pipe_resp', None)
> +        if r is not None:
> +            return r
> +        return cls.getresponse(self)
> +    return safegetresponse
>
>  class HTTPConnection(httplib.HTTPConnection):
>     # use the modified response class
>     response_class = HTTPResponse
> +    send = safesend
> +    getresponse = wrapgetresponse(httplib.HTTPConnection)
> +
>
>  #########################################################################
>  #####   TEST FUNCTIONS
> diff --git a/mercurial/url.py b/mercurial/url.py
> --- a/mercurial/url.py
> +++ b/mercurial/url.py
> @@ -408,10 +408,14 @@
>         self.close_all()
>
>  if has_https:
> -    class httpsconnection(httplib.HTTPSConnection):
> +    class BetterHTTPS(httplib.HTTPSConnection):
> +        send = keepalive.safesend
> +
> +    class httpsconnection(BetterHTTPS):
>         response_class = keepalive.HTTPResponse
>         # must be able to send big bundle as stream.
> -        send = _gen_sendfile(httplib.HTTPSConnection)
> +        send = _gen_sendfile(BetterHTTPS)
> +        getresponse = keepalive.wrapgetresponse(httplib.HTTPSConnection)
>
>     class httpshandler(keepalive.KeepAliveHandler, urllib2.HTTPSHandler):
>         def __init__(self, ui):
> _______________________________________________
> Mercurial-devel mailing list
> Mercurial-devel at selenic.com
> http://selenic.com/mailman/listinfo/mercurial-devel
>



More information about the Mercurial-devel mailing list