[PATCH 2 of 2 hg-bfiles] Refactored xxxstore.verify method, and implement sshstore.verify
david.douard at logilab.fr
david.douard at logilab.fr
Thu Nov 19 14:45:50 CST 2009
# HG changeset patch
# User David Douard <david.douard at logilab.fr>
# Date 1258392859 -3600
# Node ID 302666bd3ff5cea246ce74efa108f58e4d48c8f0
# Parent f51538471e58f2568535505a8d37b87cc41a1707
Refactored xxxstore.verify method, and implement sshstore.verify
The basestore.verify method now implements the main logics, using the low-level _verify_file method to check one file's validity. This latter is responsible for the real job of verifying a stored file, and is implemented in each basestore subclass that supports file verification.
diff --git a/bfiles.py b/bfiles.py
--- a/bfiles.py
+++ b/bfiles.py
@@ -704,7 +704,7 @@
if not data:
break
yield data
-
+
def _copy_and_hash(instream, outfile):
'''Read bytes from instream (iterable object), write them to outfile, computing the
SHA-1 hash of the data along the way. Close both infile and
@@ -786,7 +786,7 @@
success = []
missing = []
ui = self.ui
-
+
for (filename, hash) in files:
ui.note(_('getting %s\n') % filename)
outfilename = self.repo.wjoin(filename)
@@ -806,7 +806,7 @@
os.remove(tmpfilename)
missing.append((filename, hash))
continue
-
+
hhash = binascii.hexlify(bhash)
if hhash != hash:
ui.warn('%s: data corruption (expected %s, got %s)\n'
@@ -818,32 +818,11 @@
success.append((filename, hhash))
return (success, missing)
-
+
def verify(self, revs, contents=False):
'''Verify the existence (and, optionally, contents) of every big
file revision referenced by every changeset in revs.
Return 0 if all is well, non-zero on any errors.'''
- raise NotImplementedError('abstract method')
-
- def recvfile(self, tmpfile, filename, hash):
- raise NotImplementedError('abstract method')
-
-class localstore(basestore):
- '''Not necessarily a local store, just one using a regular filesystem
- path. Could be on the same machine or could be a network mount.'''
-
- # N.B. in this class, self.url is just the filesystem path
-
- def put(self, source, filename, hash):
- destdir = os.path.join(self.url, filename)
- dest = os.path.join(destdir, hash)
- if os.path.exists(dest):
- raise util.Abort(_('destination %r already exists') % dest)
-
- util.makedirs(destdir)
- shutil.copy(source, dest)
-
- def verify(self, revs, contents=False):
write = self.ui.write
failed = False
@@ -874,12 +853,33 @@
return int(failed)
def recvfile(self, tmpfile, filename, hash):
+ raise NotImplementedError('abstract method')
+
+ def _verify_file(self, cctx, cset, contents, standin, verified):
+ raise NotImplementedError('abstract method')
+
+class localstore(basestore):
+ '''Not necessarily a local store, just one using a regular filesystem
+ path. Could be on the same machine or could be a network mount.'''
+
+ # N.B. in this class, self.url is just the filesystem path
+
+ def put(self, source, filename, hash):
+ destdir = os.path.join(self.url, filename)
+ dest = os.path.join(destdir, hash)
+ if os.path.exists(dest):
+ raise util.Abort(_('destination %r already exists') % dest)
+
+ util.makedirs(destdir)
+ shutil.copy(source, dest)
+
+ def recvfile(self, tmpfile, filename, hash):
try:
infile = open(os.path.join(self.url, filename, hash), 'rb')
except IOError, err:
tmpfile.close()
raise error.RepoError('cannot open file')
- return _copy_and_hash(filestream(infile), tmpfile)
+ return _copy_and_hash(filestream(infile), tmpfile)
def _verify_file(self, cctx, cset, contents, standin, verified):
@@ -968,11 +968,10 @@
self.abort(error.RepoError(_("put failed: %s") % r))
def recvfile(self, tmpfile, filename, hash):
- storefile = os.path.join(self.path, filename, hash)
- hasher = hashlib.sha1()
-
- self.ui.debug('sshstore.recvfile(%s)\n' % (storefile))
- self.do_cmd("bfget", destname=storefile)
+ store_path = os.path.join(self.path, filename, hash)
+
+ self.ui.debug('sshstore.recvfile(%s)\n' % (store_path))
+ self.do_cmd("bfget", destname=store_path)
r = self._recv()
try:
size = int(r)
@@ -987,7 +986,45 @@
if not data:
break
yield data
-
+
+ def _verify_file(self, cctx, cset, contents, standin, verified):
+ filename = _split_standin(standin)
+ if not filename:
+ return False
+ fctx = cctx[standin]
+ key = (filename, fctx.filenode())
+ if key in verified:
+ return False
+
+ expect_hash = fctx.data()[0:40]
+ store_path = os.path.join(self.path, filename, expect_hash)
+ verified.add(key)
+
+ self.do_cmd("bfexists", destname=store_path)
+ r = self._recv()
+ if r not in ('ok', ''):
+ self.abort(error.RepoError(_("check failed, unexpected response: %s") % r))
+ if not r:
+ self.ui.warn(
+ _('changeset %s: %s missing\n'
+ ' (%s)\n')
+ % (cset, filename, store_path))
+ return True # failed
+
+ if contents:
+ self.do_cmd("bfverify", destname=store_path)
+ r = self._recv()
+ if r not in ('ok', ''):
+ self.abort(error.RepoError(_("check failed, unexpected response: %s") % r))
+ if not r:
+ self.ui.warn(
+ _('changeset %s: %s: contents differ\n'
+ ' (%s)\n')
+ % (cset, filename,
+ store_path))
+ return True # failed
+ return False
+
# XXX *snip* method heavily based on sshrepository.validate_repo
def validate_connection(self, ui, sshcmd, args, remotecmd):
# cleanup up previous run
@@ -1082,10 +1119,13 @@
self.url = url
(baseurl, authinfo) = url_.getauthinfo(self.url)
self.opener = url_.opener(self.ui, authinfo)
-
+
def put(self, source, filename, hash):
raise NotImplementedError('sorry, HTTP PUT not implemented yet')
+ def verify(self, revs, contents=False):
+ raise NotImplementedError('sorry, HTTP VERIFY not implemented yet')
+
def recvfile(self, tmpfile, filename, hash):
(baseurl, authinfo) = url_.getauthinfo(self.url)
url = urlparse.urljoin(baseurl,
@@ -1212,6 +1252,34 @@
finally:
fd.close()
+ def do_bfexists(self):
+ """respond to the bfexists command: check a file exists
+ """
+ key, fname = self.getarg()
+ if os.path.isfile(fname):
+ self.respond('ok')
+ else:
+ self.respond('')
+
+ def do_bfverify(self):
+ """respond to the bfhash command: check the hash of the file match its name
+ """
+ key, fname = self.getarg()
+ if not os.path.isfile(fname):
+ self.respond('file %s does not exists' % fname)
+ return
+ try:
+ fd = open(fname, "rb")
+ except IOError:
+ self.respond('cannot read file')
+ return
+ hhash = _hashfile(fd)
+ fd.close()
+ expect_hash = os.path.basename(fname)
+ if expect_hash == hhash:
+ self.respond('ok')
+ else:
+ self.respond('')
# -- hg commands declarations ------------------------------------------------
More information about the Mercurial-devel
mailing list