[PATCH 1 of 2 RFC V2] largefiles: abort push on client when file fit largefiles (issue3245)
liscju
piotr.listkiewicz at gmail.com
Fri Aug 26 06:23:12 UTC 2016
# HG changeset patch
# User liscju <piotr.listkiewicz at gmail.com>
# Date 1472122462 -7200
# Thu Aug 25 12:54:22 2016 +0200
# Node ID 7fa71874c771a76bcc0e04d77e435242b9b3ad12
# Parent 6a98f9408a504be455d4382801610daceac429e6
largefiles: abort push on client when file fit largefiles (issue3245)
So far setting minsize and patterns for largefiles worked only
locally changing which files were added/committed as largefiles.
It did not prevent pushing ordinary files that exceeded minsize
or matched pattern for largefiles on server.
To solve this there are two possible solutions:
1) Check ordinary files before pushing, if they match pattern
for largefiles on server - abort push
2) Check ordinary files when there are unbundled on server,
abort when there are files that match pattern for largefiles
This commit is an implementation of #1 idea to discuss and needs
some changes before being merged. The #2 idea is covered in
consecutive commit.
This solution at first downloads minsize and patterns from server,
check if any of files exceeds minsize or match pattern and if it
does aborts push.
This solution works only when server/client has version of hg that
support this checking. Idea #2 works for any client as far as
server has support for this, but the drawback of this idea is that
checks are done after sending files to server.
As pointed out in previous discussing on mailing list,
the question is if push should abort when
one of the revision contains ordinary file that fits largefile,
or it should check only heads of the pushed changesets. Another
possibility would be to just to give user the choice.
diff --git a/hgext/largefiles/basestore.py b/hgext/largefiles/basestore.py
--- a/hgext/largefiles/basestore.py
+++ b/hgext/largefiles/basestore.py
@@ -11,7 +11,12 @@ from __future__ import absolute_import
from mercurial.i18n import _
-from mercurial import node, util
+from mercurial import (
+ error,
+ match as matchmod,
+ node,
+ util,
+)
from . import lfutil
@@ -149,6 +154,34 @@ class basestore(object):
% (numrevs, numlfiles))
return int(failed)
+ def checkfiletopush(self, revs):
+ if self.repo.ui.configbool(lfutil.longname, 'disableclientcheck'):
+ return
+
+ lfsize, lfpats = self._getlfpatterns()
+
+ pats = []
+ if lfsize:
+ pats.append("set:size('>%sB')" % lfsize)
+
+ if lfpats:
+ pats.extend(lfpats)
+
+ if pats:
+ for rev in revs:
+ ctx = self.repo[rev]
+ # cwd parameter is equal to repo.root to prevent
+ # 'abort: pattern not under root' when hg used
+ # with -R option
+ lfmatcher = matchmod.match(self.repo.root,
+ self.repo.root, pats, ctx=ctx)
+ filesmatching = ctx.walk(lfmatcher)
+
+ for file in filesmatching:
+ raise error.Abort(_("Files %s in revision %s should be "
+ "marked as largefiles")
+ % (file, rev))
+
def _getfile(self, tmpfile, filename, hash):
'''Fetch one revision of one file from the store and write it
to tmpfile. Compute the hash of the file on-the-fly as it
@@ -164,3 +197,8 @@ class basestore(object):
Returns _true_ if any problems are found!
'''
raise NotImplementedError('abstract method')
+
+ def _getlfpatterns(self):
+ '''Get tuple of size and pattern of files that should be
+ treated as large files.'''
+ raise NotImplementedError('abstract method')
diff --git a/hgext/largefiles/localstore.py b/hgext/largefiles/localstore.py
--- a/hgext/largefiles/localstore.py
+++ b/hgext/largefiles/localstore.py
@@ -11,6 +11,11 @@ from __future__ import absolute_import
from mercurial.i18n import _
+from mercurial import (
+ error,
+ util,
+)
+
from . import (
basestore,
lfutil,
@@ -64,3 +69,16 @@ class localstore(basestore.basestore):
% (cset, filename, storepath))
failed = True
return failed
+
+ def _getlfpatterns(self):
+ lfsize = self.remote.ui.config(lfutil.longname, 'minsize', default=None)
+ if lfsize:
+ try:
+ lfsize = util.sizetoint(lfsize + "MB")
+ except error.ParseError:
+ lfsize = None
+
+ lfpats = self.remote.ui.configlist(
+ lfutil.longname, 'patterns', default=[])
+
+ return lfsize, lfpats
diff --git a/hgext/largefiles/proto.py b/hgext/largefiles/proto.py
--- a/hgext/largefiles/proto.py
+++ b/hgext/largefiles/proto.py
@@ -187,3 +187,29 @@ def httprepocallstream(self, cmd, **args
if cmd == 'batch' and self.capable('largefiles'):
args['cmds'] = headsre.sub('lheads', args['cmds'])
return httpoldcallstream(self, cmd, **args)
+
+def pushlfilespattern(repo, key, old, new):
+ repo.ui.warn(_("pushing key to largefiles not supported"))
+ return 0
+
+def listlfilespattern(repo):
+ lfsize = repo.ui.config(lfutil.longname, 'minsize', default=None)
+ if lfsize:
+ try:
+ lfsize = util.sizetoint(lfsize + "MB")
+ except error.ParseError:
+ lfsize = None
+
+ lfpats = repo.ui.configlist(
+ lfutil.longname, 'patterns', default=[])
+
+ keys = {}
+
+ if lfsize:
+ keys['lfminsize'] = str(lfsize)
+ else:
+ keys['lfminsize'] = ''
+
+ keys['lfpats'] = ','.join(lfpats)
+
+ return keys
diff --git a/hgext/largefiles/reposetup.py b/hgext/largefiles/reposetup.py
--- a/hgext/largefiles/reposetup.py
+++ b/hgext/largefiles/reposetup.py
@@ -23,6 +23,7 @@ from mercurial import (
from . import (
lfcommands,
lfutil,
+ storefactory,
)
def reposetup(ui, repo):
@@ -350,6 +351,14 @@ def reposetup(ui, repo):
actualfiles += regulars
return actualfiles
+ def checkpush(self, pushop):
+ super(lfilesrepo, self).checkpush(pushop)
+ store = storefactory.openstore(pushop.repo, pushop.remote)
+ revs = pushop.revs
+ if revs is None:
+ revs = pushop.repo.revs('all()')
+ store.checkfiletopush(revs)
+
repo.__class__ = lfilesrepo
# stack of hooks being executed before committing.
diff --git a/hgext/largefiles/uisetup.py b/hgext/largefiles/uisetup.py
--- a/hgext/largefiles/uisetup.py
+++ b/hgext/largefiles/uisetup.py
@@ -27,6 +27,7 @@ from mercurial import (
hg,
httppeer,
merge,
+ pushkey,
scmutil,
sshpeer,
subrepo,
@@ -201,3 +202,6 @@ def uisetup(ui):
if name == 'transplant':
extensions.wrapcommand(getattr(module, 'cmdtable'), 'transplant',
overrides.overridetransplant)
+
+ pushkey.register('largefiles', proto.pushlfilespattern,
+ proto.listlfilespattern)
diff --git a/hgext/largefiles/wirestore.py b/hgext/largefiles/wirestore.py
--- a/hgext/largefiles/wirestore.py
+++ b/hgext/largefiles/wirestore.py
@@ -37,3 +37,18 @@ class wirestore(remotestore.remotestore)
batch.statlfile(hash)
batch.submit()
return dict(zip(hashes, batch.results()))
+
+ def _getlfpatterns(self):
+ lfkeys = self.remote.listkeys('largefiles')
+
+ if lfkeys['lfminsize']:
+ lfminsize = int(lfkeys['lfminsize'])
+ else:
+ lfminsize = None
+
+ if lfkeys['lfpats']:
+ lfpats = lfkeys['lfpats'].split(',')
+ else:
+ lfpats = []
+
+ return lfminsize, lfpats
diff --git a/tests/test-largefiles-wireproto.t b/tests/test-largefiles-wireproto.t
--- a/tests/test-largefiles-wireproto.t
+++ b/tests/test-largefiles-wireproto.t
@@ -161,7 +161,6 @@ largefiles clients refuse to push largef
$ cat ../hg.pid >> $DAEMON_PIDS
$ hg push http://localhost:$HGPORT
pushing to http://localhost:$HGPORT/
- searching for changes
abort: http://localhost:$HGPORT/ does not appear to be a largefile store
[255]
$ cd ..
@@ -444,4 +443,87 @@ a large file from the server rather than
$ rm hg.pid access.log
$ killdaemons.py
+Before pushing revisions to the remote store it should check if
+they dont contain files that are enforced to be large file on remote
+store.
+
+ $ hg init enforcingserver
+ $ cat >> enforcingserver/.hg/hgrc <<EOF
+ > [extensions]
+ > largefiles=
+ > [largefiles]
+ > # Above 7 bytes it should be large file
+ > minsize=0.000007
+ > patterns=glob:*.jpg
+ > usercache=${USERCACHE}
+ > [web]
+ > push_ssl = false
+ > allow_push = *
+ > EOF
+
+ $ hg serve -R enforcingserver -d -p $HGPORT --pid-file hg.pid
+ $ cat hg.pid >> $DAEMON_PIDS
+
+ $ hg clone http://localhost:$HGPORT encorcingclient
+ no changes found
+ updating to branch default
+ 0 files updated, 0 files merged, 0 files removed, 0 files unresolved
+ $ cd encorcingclient
+
+If user want to push revision with file below 7 bytes everything is fine
+
+ $ echo "AAAA" >> a
+ $ hg add a
+ $ hg commit -m "a"
+ Invoking status precommit hook
+ A a
+ $ rm "${USERCACHE}"/*
+ $ hg push
+ pushing to http://localhost:$HGPORT/
+ searching for changes
+ remote: adding changesets
+ remote: adding manifests
+ remote: adding file changes
+ remote: added 1 changesets with 1 changes to 1 files
+
+If user wants to push revision with file above 7 bytes it aborts
+
+ $ echo "BBBBBBBBBBBBB" >> b
+ $ hg add b
+ $ hg commit -m "b"
+ Invoking status precommit hook
+ A b
+ $ hg push
+ pushing to http://localhost:$HGPORT/
+ abort: Files b in revision 1 should be marked as largefiles
+ [255]
+
+If user wants to push revision with file that match pattern for
+largefiles in server it aborts as well
+
+ $ echo "BB" > b
+ $ hg mv b b.jpg
+ $ hg commit --amend -m "b"
+ saved backup bundle to $TESTTMP/encorcingclient/.hg/strip-backup/3705bb1eeaab-aa2c33ea-amend-backup.hg (glob)
+ $ hg push
+ pushing to http://localhost:$HGPORT/
+ abort: Files b.jpg in revision 1 should be marked as largefiles
+ [255]
+
+Client can disable checking by putting disableclientcheck option in hgrc file
+
+ $ cat >> .hg/hgrc <<EOF
+ > [largefiles]
+ > disableclientcheck=yes
+ > EOF
+ $ hg push
+ pushing to http://localhost:$HGPORT/
+ searching for changes
+ remote: adding changesets
+ remote: adding manifests
+ remote: adding file changes
+ remote: added 1 changesets with 1 changes to 1 files
+
+ $ killdaemons.py
+
#endif
More information about the Mercurial-devel
mailing list