[PATCH] largefiles: send statlfile remote calls only for nonexisting locally files
Augie Fackler
raf at durin42.com
Mon May 23 16:22:02 EDT 2016
On Tue, May 10, 2016 at 09:44:11PM -0400, Augie Fackler wrote:
> This looks reasonable to me. Mads, does it look good to you?
Hearing nothing, I'm going to go ahead and queue this. Thanks!
>
> On Tue, May 10, 2016 at 01:28:52PM +0200, liscju wrote:
> > # HG changeset patch
> > # User liscju <piotr.listkiewicz at gmail.com>
> > # Date 1462781132 -7200
> > # Mon May 09 10:05:32 2016 +0200
> > # Node ID fa61e4c944c5f9abf50ebd5d361726adf164c681
> > # Parent af6d4a49e3615ac6cfa63b9c10811c28520434ea
> > largefiles: send statlfile remote calls only for nonexisting locally files
> >
> > Files that are already in local store should be checked locally. The problem
> > with this implementation is how difference in messages between local and remote
> > checks should look like. For now local errors for file missing and content
> > corrupted looks like this:
> >
> > 'changeset cset: filename references missing storepath\n'
> > 'changeset cset: filename references corrupted storepath\n'
> >
> > for remote it looks like:
> >
> > 'changeset cset: filename missing\n'
> > 'changeset cset: filename: contents differ\n'
> >
> > Contents differ error for remote calls is never raised currently - for now
> > statlfile implementation lacks checking file content.
> >
> > diff -r af6d4a49e361 -r fa61e4c944c5 hgext/largefiles/remotestore.py
> > --- a/hgext/largefiles/remotestore.py Thu May 05 15:12:43 2016 -0500
> > +++ b/hgext/largefiles/remotestore.py Mon May 09 10:05:32 2016 +0200
> > @@ -14,11 +14,13 @@ urlreq = util.urlreq
> >
> > import lfutil
> > import basestore
> > +import localstore
> >
> > class remotestore(basestore.basestore):
> > '''a largefile store accessed over a network'''
> > def __init__(self, ui, repo, url):
> > super(remotestore, self).__init__(ui, repo, url)
> > + self._lstore = localstore.localstore(self.ui, self.repo, self.repo)
> >
> > def put(self, source, hash):
> > if self.sendfile(source, hash):
> > @@ -65,27 +67,42 @@ class remotestore(basestore.basestore):
> >
> > return lfutil.copyandhash(chunks, tmpfile)
> >
> > + def _hashesavailablelocally(self, hashes):
> > + existslocallymap = self._lstore.exists(hashes)
> > + localhashes = [hash for hash in hashes if existslocallymap[hash]]
> > + return localhashes
> > +
> > def _verifyfiles(self, contents, filestocheck):
> > failed = False
> > expectedhashes = [expectedhash
> > for cset, filename, expectedhash in filestocheck]
> > - stats = self._stat(expectedhashes)
> > + localhashes = self._hashesavailablelocally(expectedhashes)
> > + stats = self._stat([expectedhash for expectedhash in expectedhashes
> > + if expectedhash not in localhashes])
> > +
> > for cset, filename, expectedhash in filestocheck:
> > - stat = stats[expectedhash]
> > - if stat:
> > - if stat == 1:
> > - self.ui.warn(
> > - _('changeset %s: %s: contents differ\n')
> > - % (cset, filename))
> > + if expectedhash in localhashes:
> > + filetocheck = (cset, filename, expectedhash)
> > + verifyresult = self._lstore._verifyfiles(contents,
> > + [filetocheck])
> > + if verifyresult:
> > failed = True
> > - elif stat == 2:
> > - self.ui.warn(
> > - _('changeset %s: %s missing\n')
> > - % (cset, filename))
> > - failed = True
> > - else:
> > - raise RuntimeError('verify failed: unexpected response '
> > - 'from statlfile (%r)' % stat)
> > + else:
> > + stat = stats[expectedhash]
> > + if stat:
> > + if stat == 1:
> > + self.ui.warn(
> > + _('changeset %s: %s: contents differ\n')
> > + % (cset, filename))
> > + failed = True
> > + elif stat == 2:
> > + self.ui.warn(
> > + _('changeset %s: %s missing\n')
> > + % (cset, filename))
> > + failed = True
> > + else:
> > + raise RuntimeError('verify failed: unexpected response '
> > + 'from statlfile (%r)' % stat)
> > return failed
> >
> > def batch(self):
> > diff -r af6d4a49e361 -r fa61e4c944c5 tests/test-largefiles-wireproto.t
> > --- a/tests/test-largefiles-wireproto.t Thu May 05 15:12:43 2016 -0500
> > +++ b/tests/test-largefiles-wireproto.t Mon May 09 10:05:32 2016 +0200
> > @@ -324,27 +324,59 @@ largefiles should batch verify remote ca
> > $ hg serve -R batchverifymain -d -p $HGPORT --pid-file hg.pid \
> > > -A access.log
> > $ cat hg.pid >> $DAEMON_PIDS
> > - $ hg clone http://localhost:$HGPORT batchverifyclone
> > + $ hg clone --noupdate http://localhost:$HGPORT batchverifyclone
> > requesting all changes
> > adding changesets
> > adding manifests
> > adding file changes
> > added 2 changesets with 2 changes to 2 files
> > - updating to branch default
> > - getting changed largefiles
> > - 2 largefiles updated, 0 removed
> > - 2 files updated, 0 files merged, 0 files removed, 0 files unresolved
> > - $ hg -R batchverifyclone verify --large
> > + $ hg -R batchverifyclone verify --large --lfa
> > checking changesets
> > checking manifests
> > crosschecking files in changesets and manifests
> > checking files
> > 2 files, 2 changesets, 2 total revisions
> > - searching 1 changesets for largefiles
> > + searching 2 changesets for largefiles
> > verified existence of 2 revisions of 2 largefiles
> > $ tail -1 access.log
> > 127.0.0.1 - - [*] "GET /?cmd=batch HTTP/1.1" 200 - x-hgarg-1:cmds=statlfile+sha%3D972a1a11f19934401291cc99117ec614933374ce%3Bstatlfile+sha%3Dc801c9cfe94400963fcb683246217d5db77f9a9a (glob)
> > - $ rm access.log
> > + $ hg -R batchverifyclone update
> > + getting changed largefiles
> > + 2 largefiles updated, 0 removed
> > + 2 files updated, 0 files merged, 0 files removed, 0 files unresolved
> > +
> > +Clear log file before next test
> > +
> > + $ printf "" > access.log
> > +
> > +Verify should check file on remote server only when file is not
> > +available locally.
> > +
> > + $ echo "ccc" >> batchverifymain/c
> > + $ hg -R batchverifymain status
> > + ? c
> > + $ hg -R batchverifymain add --large batchverifymain/c
> > + $ hg -R batchverifymain commit -m "c"
> > + Invoking status precommit hook
> > + A c
> > + $ hg -R batchverifyclone pull
> > + pulling from http://localhost:$HGPORT/
> > + searching for changes
> > + adding changesets
> > + adding manifests
> > + adding file changes
> > + added 1 changesets with 1 changes to 1 files
> > + (run 'hg update' to get a working copy)
> > + $ hg -R batchverifyclone verify --lfa
> > + checking changesets
> > + checking manifests
> > + crosschecking files in changesets and manifests
> > + checking files
> > + 3 files, 3 changesets, 3 total revisions
> > + searching 3 changesets for largefiles
> > + verified existence of 3 revisions of 3 largefiles
> > + $ tail -1 access.log
> > + 127.0.0.1 - - [*] "GET /?cmd=batch HTTP/1.1" 200 - x-hgarg-1:cmds=statlfile+sha%3Dc8559c3c9cfb42131794b7d8009230403b9b454c (glob)
> >
> > $ killdaemons.py
> >
> > _______________________________________________
> > Mercurial-devel mailing list
> > Mercurial-devel at mercurial-scm.org
> > https://www.mercurial-scm.org/mailman/listinfo/mercurial-devel
> _______________________________________________
> Mercurial-devel mailing list
> Mercurial-devel at mercurial-scm.org
> https://www.mercurial-scm.org/mailman/listinfo/mercurial-devel
More information about the Mercurial-devel
mailing list