D1568: lfs: using workers in lfs prefetch
mharbison72 (Matt Harbison)
phabricator at mercurial-scm.org
Thu Jan 4 15:16:43 EST 2018
mharbison72 added a comment.
It looks like keepalive.safesend() isn't sending everything. I can send this to the mailing list if it gets mangled, but I figured I'd try to keep this thread together if possible.
# HG changeset patch
# User Matt Harbison <matt_harbison at yahoo.com>
# Date 1515094298 18000
# Thu Jan 04 14:31:38 2018 -0500
# Branch bzr_convert
# Node ID b0abd77f295edbf1df58674bc1ef7a6bc57e4096
# Parent 0653b44c70685f9ed6f5cf1689ca08f7bd2dbe34
xxx-lfs: attempt to isolate upload corruption with workers
(Please don't queue this- it's only to help Wojciech isolate this.)
It doesn't look like all of the data is being sent over the socket, although it
appears that all of the data is being read from the file. Here's a sample
transcript of an upload, with the subsequent wall of html removed. In this
case, all 4 files failed checksum verification on the server side. The size in
the "Sent %d bytes of data" line corresponds to what the server ends up with,
and the other sizes match what the client starts with.
I have no idea why this seems to work when 'lfs.url' is http instead of https.
$ hg push
...
pushing to lighttpd
searching for changes
12f746f3f2493b2f39da7ecf63d7ee19c6ac9ec6a4fcd8c229da8a522cb12765: size 5278176
932b4ee4def2b434f85435d9e3e19ca8ba99ce9a065a61524b429a9d5e9b2e9c: size 5258384
ccdf7e788769838f8285b3ee672ed573358202305ee361cfec7a4a4fb005bbc7: size 2062258
dbc0ae9f8b05a7f84770ea303db5c1601500295548b3253e51f8889fcb38cc0a: size 5103733
Broken pipe!
Broken pipe!
Broken pipe!
No more data on ccdf...bbc7. Read 2062258, expected 2062258
Sent 939954 bytes of data
No more data on dbc0...cc0a. Read 5103733, expected 5103733
Sent 4931701 bytes of data
No more data on 932b...2e9c. Read 5258384, expected 5258384
Sent 5176464 bytes of data
No more data on 12f7...2765. Read 5278176, expected 5278176
Sent 5138912 bytes of data
<html with java.net.SocketTimeoutException stacktrace removed>
diff --git a/hgext/lfs/blobstore.py b/hgext/lfs/blobstore.py
--- a/hgext/lfs/blobstore.py
+++ b/hgext/lfs/blobstore.py
@@ -64,12 +64,14 @@ class filewithprogress(object):
Useful to provide progress information for how many bytes are read.
"""
- def __init__(self, fp, callback):
+ def __init__(self, fp, callback, name):
self._fp = fp
self._callback = callback # func(readsize)
fp.seek(0, os.SEEK_END)
self._len = fp.tell()
+ self.total = 0
fp.seek(0)
+ self.name = name
def __len__(self):
return self._len
@@ -79,9 +81,12 @@ class filewithprogress(object):
return b''
data = self._fp.read(size)
if data:
+ self.total += len(data)
if self._callback:
self._callback(len(data))
else:
+ print('No more data on %s. Read %d, expected %d' %
+ (self.name, self.total, self._len))
self._fp.close()
self._fp = None
return data
@@ -100,6 +105,12 @@ class local(object):
self.cachevfs = lfsvfs(usercache)
self.ui = repo.ui
+ def open(self, oid):
+ """Open a file descriptor for the named blob."""
+ if self.cachevfs.exists(oid):
+ return self.cachevfs(oid)
+ return self.vfs(oid)
+
def write(self, oid, data, verify=True):
"""Write blob to local blobstore."""
if verify:
@@ -254,9 +265,9 @@ class _gitlfsremote(object):
request = util.urlreq.request(href)
if action == 'upload':
# If uploading blobs, read data from local blobstore.
- with localstore.vfs(oid) as fp:
+ with localstore.open(oid) as fp:
_verifyfile(oid, fp)
- request.data = filewithprogress(localstore.vfs(oid), None)
+ request.data = filewithprogress(localstore.open(oid), None, oid)
request.get_method = lambda: 'PUT'
for k, v in headers:
@@ -271,6 +282,8 @@ class _gitlfsremote(object):
break
response += data
except util.urlerr.httperror as ex:
+ print('%s: %s' % (oid, ex.read()))
+ print('%s: %s' % (oid, ex.info()))
raise LfsRemoteError(_('HTTP error: %s (oid=%s, action=%s)')
% (ex, oid, action))
@@ -288,6 +301,7 @@ class _gitlfsremote(object):
sizes = {}
for obj in objects:
sizes[obj.get('oid')] = obj.get('size', 0)
+ print('%s: size %d' % (obj.get('oid'), obj.get('size', 0)))
topic = {'upload': _('lfs uploading'),
'download': _('lfs downloading')}[action]
if len(objects) > 1:
diff --git a/mercurial/keepalive.py b/mercurial/keepalive.py
--- a/mercurial/keepalive.py
+++ b/mercurial/keepalive.py
@@ -553,14 +553,18 @@ def safesend(self, str):
if self.debuglevel > 0:
print("sending a read()able")
data = read(blocksize)
+ total = 0
while data:
+ total += len(data)
self.sock.sendall(data)
data = read(blocksize)
+ print('Sent %d bytes of data' % total)
else:
self.sock.sendall(str)
except socket.error as v:
reraise = True
if v[0] == errno.EPIPE: # Broken pipe
+ print('Broken pipe!')
if self._HTTPConnection__state == httplib._CS_REQ_SENT:
self._broken_pipe_resp = None
self._broken_pipe_resp = self.getresponse()
REPOSITORY
rHG Mercurial
REVISION DETAIL
https://phab.mercurial-scm.org/D1568
To: wlis, #hg-reviewers, quark, mharbison72
Cc: mharbison72, quark, mercurial-devel
More information about the Mercurial-devel
mailing list