D1568: lfs: using workers in lfs prefetch

mharbison72 (Matt Harbison) phabricator at mercurial-scm.org
Thu Jan 4 15:16:43 EST 2018


mharbison72 added a comment.


  It looks like keepalive.safesend() isn't sending everything.  I can send this to the mailing list if it gets mangled, but I figured I'd try to keep this thread together if possible.
  
    # HG changeset patch
    # User Matt Harbison <matt_harbison at yahoo.com>
    # Date 1515094298 18000
    #      Thu Jan 04 14:31:38 2018 -0500
    # Branch bzr_convert
    # Node ID b0abd77f295edbf1df58674bc1ef7a6bc57e4096
    # Parent  0653b44c70685f9ed6f5cf1689ca08f7bd2dbe34
    xxx-lfs: attempt to isolate upload corruption with workers
    
    (Please don't queue this- it's only to help Wojciech isolate this.)
    
    It doesn't look like all of the data is being sent over the socket, although it
    appears that all of the data is being read from the file.  Here's a sample
    transcript of an upload, with the subsequent wall of html removed.  In this
    case, all 4 files failed checksum verification on the server side.  The size in
    the "Sent %d bytes of data" line corresponds to what the server ends up with,
    and the other sizes match what the client starts with.
    
    I have no idea why this seems to work when 'lfs.url' is http instead of https.
    
    $ hg push
    ...
    pushing to lighttpd
    searching for changes
    12f746f3f2493b2f39da7ecf63d7ee19c6ac9ec6a4fcd8c229da8a522cb12765: size 5278176
    932b4ee4def2b434f85435d9e3e19ca8ba99ce9a065a61524b429a9d5e9b2e9c: size 5258384
    ccdf7e788769838f8285b3ee672ed573358202305ee361cfec7a4a4fb005bbc7: size 2062258
    dbc0ae9f8b05a7f84770ea303db5c1601500295548b3253e51f8889fcb38cc0a: size 5103733
    Broken pipe!
    Broken pipe!
    Broken pipe!
    No more data on ccdf...bbc7.  Read 2062258, expected 2062258
    Sent 939954 bytes of data
    No more data on dbc0...cc0a.  Read 5103733, expected 5103733
    Sent 4931701 bytes of data
    No more data on 932b...2e9c.  Read 5258384, expected 5258384
    Sent 5176464 bytes of data
    No more data on 12f7...2765.  Read 5278176, expected 5278176
    Sent 5138912 bytes of data
    <html with java.net.SocketTimeoutException stacktrace removed>
    
    diff --git a/hgext/lfs/blobstore.py b/hgext/lfs/blobstore.py
    --- a/hgext/lfs/blobstore.py
    +++ b/hgext/lfs/blobstore.py
    @@ -64,12 +64,14 @@ class filewithprogress(object):
         Useful to provide progress information for how many bytes are read.
         """
     
    -    def __init__(self, fp, callback):
    +    def __init__(self, fp, callback, name):
             self._fp = fp
             self._callback = callback # func(readsize)
             fp.seek(0, os.SEEK_END)
             self._len = fp.tell()
    +        self.total = 0
             fp.seek(0)
    +        self.name = name
     
         def __len__(self):
             return self._len
    @@ -79,9 +81,12 @@ class filewithprogress(object):
                 return b''
             data = self._fp.read(size)
             if data:
    +            self.total += len(data)
                 if self._callback:
                     self._callback(len(data))
             else:
    +            print('No more data on %s.  Read %d, expected %d' %
    +                  (self.name, self.total, self._len))
                 self._fp.close()
                 self._fp = None
             return data
    @@ -100,6 +105,12 @@ class local(object):
             self.cachevfs = lfsvfs(usercache)
             self.ui = repo.ui
     
    +    def open(self, oid):
    +        """Open a file descriptor for the named blob."""
    +        if self.cachevfs.exists(oid):
    +            return self.cachevfs(oid)
    +        return self.vfs(oid)
    +
         def write(self, oid, data, verify=True):
             """Write blob to local blobstore."""
             if verify:
    @@ -254,9 +265,9 @@ class _gitlfsremote(object):
             request = util.urlreq.request(href)
             if action == 'upload':
                 # If uploading blobs, read data from local blobstore.
    -            with localstore.vfs(oid) as fp:
    +            with localstore.open(oid) as fp:
                     _verifyfile(oid, fp)
    -            request.data = filewithprogress(localstore.vfs(oid), None)
    +            request.data = filewithprogress(localstore.open(oid), None, oid)
                 request.get_method = lambda: 'PUT'
     
             for k, v in headers:
    @@ -271,6 +282,8 @@ class _gitlfsremote(object):
                         break
                     response += data
             except util.urlerr.httperror as ex:
    +            print('%s: %s' % (oid, ex.read()))
    +            print('%s: %s' % (oid, ex.info()))
                 raise LfsRemoteError(_('HTTP error: %s (oid=%s, action=%s)')
                                      % (ex, oid, action))
     
    @@ -288,6 +301,7 @@ class _gitlfsremote(object):
             sizes = {}
             for obj in objects:
                 sizes[obj.get('oid')] = obj.get('size', 0)
    +            print('%s: size %d' % (obj.get('oid'), obj.get('size', 0)))
             topic = {'upload': _('lfs uploading'),
                      'download': _('lfs downloading')}[action]
             if len(objects) > 1:
    diff --git a/mercurial/keepalive.py b/mercurial/keepalive.py
    --- a/mercurial/keepalive.py
    +++ b/mercurial/keepalive.py
    @@ -553,14 +553,18 @@ def safesend(self, str):
                 if self.debuglevel > 0:
                     print("sending a read()able")
                 data = read(blocksize)
    +            total = 0
                 while data:
    +                total += len(data)
                     self.sock.sendall(data)
                     data = read(blocksize)
    +            print('Sent %d bytes of data' % total)
             else:
                 self.sock.sendall(str)
         except socket.error as v:
             reraise = True
             if v[0] == errno.EPIPE:      # Broken pipe
    +            print('Broken pipe!')
                 if self._HTTPConnection__state == httplib._CS_REQ_SENT:
                     self._broken_pipe_resp = None
                     self._broken_pipe_resp = self.getresponse()

REPOSITORY
  rHG Mercurial

REVISION DETAIL
  https://phab.mercurial-scm.org/D1568

To: wlis, #hg-reviewers, quark, mharbison72
Cc: mharbison72, quark, mercurial-devel


More information about the Mercurial-devel mailing list