[PATCH] largefiles: send statlfile remote calls only for nonexisting locally files
liscju
piotr.listkiewicz at gmail.com
Tue May 10 11:28:52 UTC 2016
# HG changeset patch
# User liscju <piotr.listkiewicz at gmail.com>
# Date 1462781132 -7200
# Mon May 09 10:05:32 2016 +0200
# Node ID fa61e4c944c5f9abf50ebd5d361726adf164c681
# Parent af6d4a49e3615ac6cfa63b9c10811c28520434ea
largefiles: send statlfile remote calls only for nonexisting locally files
Files that are already in local store should be checked locally. The problem
with this implementation is how difference in messages between local and remote
checks should look like. For now local errors for file missing and content
corrupted looks like this:
'changeset cset: filename references missing storepath\n'
'changeset cset: filename references corrupted storepath\n'
for remote it looks like:
'changeset cset: filename missing\n'
'changeset cset: filename: contents differ\n'
Contents differ error for remote calls is never raised currently - for now
statlfile implementation lacks checking file content.
diff -r af6d4a49e361 -r fa61e4c944c5 hgext/largefiles/remotestore.py
--- a/hgext/largefiles/remotestore.py Thu May 05 15:12:43 2016 -0500
+++ b/hgext/largefiles/remotestore.py Mon May 09 10:05:32 2016 +0200
@@ -14,11 +14,13 @@ urlreq = util.urlreq
import lfutil
import basestore
+import localstore
class remotestore(basestore.basestore):
'''a largefile store accessed over a network'''
def __init__(self, ui, repo, url):
super(remotestore, self).__init__(ui, repo, url)
+ self._lstore = localstore.localstore(self.ui, self.repo, self.repo)
def put(self, source, hash):
if self.sendfile(source, hash):
@@ -65,27 +67,42 @@ class remotestore(basestore.basestore):
return lfutil.copyandhash(chunks, tmpfile)
+ def _hashesavailablelocally(self, hashes):
+ existslocallymap = self._lstore.exists(hashes)
+ localhashes = [hash for hash in hashes if existslocallymap[hash]]
+ return localhashes
+
def _verifyfiles(self, contents, filestocheck):
failed = False
expectedhashes = [expectedhash
for cset, filename, expectedhash in filestocheck]
- stats = self._stat(expectedhashes)
+ localhashes = self._hashesavailablelocally(expectedhashes)
+ stats = self._stat([expectedhash for expectedhash in expectedhashes
+ if expectedhash not in localhashes])
+
for cset, filename, expectedhash in filestocheck:
- stat = stats[expectedhash]
- if stat:
- if stat == 1:
- self.ui.warn(
- _('changeset %s: %s: contents differ\n')
- % (cset, filename))
+ if expectedhash in localhashes:
+ filetocheck = (cset, filename, expectedhash)
+ verifyresult = self._lstore._verifyfiles(contents,
+ [filetocheck])
+ if verifyresult:
failed = True
- elif stat == 2:
- self.ui.warn(
- _('changeset %s: %s missing\n')
- % (cset, filename))
- failed = True
- else:
- raise RuntimeError('verify failed: unexpected response '
- 'from statlfile (%r)' % stat)
+ else:
+ stat = stats[expectedhash]
+ if stat:
+ if stat == 1:
+ self.ui.warn(
+ _('changeset %s: %s: contents differ\n')
+ % (cset, filename))
+ failed = True
+ elif stat == 2:
+ self.ui.warn(
+ _('changeset %s: %s missing\n')
+ % (cset, filename))
+ failed = True
+ else:
+ raise RuntimeError('verify failed: unexpected response '
+ 'from statlfile (%r)' % stat)
return failed
def batch(self):
diff -r af6d4a49e361 -r fa61e4c944c5 tests/test-largefiles-wireproto.t
--- a/tests/test-largefiles-wireproto.t Thu May 05 15:12:43 2016 -0500
+++ b/tests/test-largefiles-wireproto.t Mon May 09 10:05:32 2016 +0200
@@ -324,27 +324,59 @@ largefiles should batch verify remote ca
$ hg serve -R batchverifymain -d -p $HGPORT --pid-file hg.pid \
> -A access.log
$ cat hg.pid >> $DAEMON_PIDS
- $ hg clone http://localhost:$HGPORT batchverifyclone
+ $ hg clone --noupdate http://localhost:$HGPORT batchverifyclone
requesting all changes
adding changesets
adding manifests
adding file changes
added 2 changesets with 2 changes to 2 files
- updating to branch default
- getting changed largefiles
- 2 largefiles updated, 0 removed
- 2 files updated, 0 files merged, 0 files removed, 0 files unresolved
- $ hg -R batchverifyclone verify --large
+ $ hg -R batchverifyclone verify --large --lfa
checking changesets
checking manifests
crosschecking files in changesets and manifests
checking files
2 files, 2 changesets, 2 total revisions
- searching 1 changesets for largefiles
+ searching 2 changesets for largefiles
verified existence of 2 revisions of 2 largefiles
$ tail -1 access.log
127.0.0.1 - - [*] "GET /?cmd=batch HTTP/1.1" 200 - x-hgarg-1:cmds=statlfile+sha%3D972a1a11f19934401291cc99117ec614933374ce%3Bstatlfile+sha%3Dc801c9cfe94400963fcb683246217d5db77f9a9a (glob)
- $ rm access.log
+ $ hg -R batchverifyclone update
+ getting changed largefiles
+ 2 largefiles updated, 0 removed
+ 2 files updated, 0 files merged, 0 files removed, 0 files unresolved
+
+Clear log file before next test
+
+ $ printf "" > access.log
+
+Verify should check file on remote server only when file is not
+available locally.
+
+ $ echo "ccc" >> batchverifymain/c
+ $ hg -R batchverifymain status
+ ? c
+ $ hg -R batchverifymain add --large batchverifymain/c
+ $ hg -R batchverifymain commit -m "c"
+ Invoking status precommit hook
+ A c
+ $ hg -R batchverifyclone pull
+ pulling from http://localhost:$HGPORT/
+ searching for changes
+ adding changesets
+ adding manifests
+ adding file changes
+ added 1 changesets with 1 changes to 1 files
+ (run 'hg update' to get a working copy)
+ $ hg -R batchverifyclone verify --lfa
+ checking changesets
+ checking manifests
+ crosschecking files in changesets and manifests
+ checking files
+ 3 files, 3 changesets, 3 total revisions
+ searching 3 changesets for largefiles
+ verified existence of 3 revisions of 3 largefiles
+ $ tail -1 access.log
+ 127.0.0.1 - - [*] "GET /?cmd=batch HTTP/1.1" 200 - x-hgarg-1:cmds=statlfile+sha%3Dc8559c3c9cfb42131794b7d8009230403b9b454c (glob)
$ killdaemons.py
More information about the Mercurial-devel
mailing list