[PATCH 6 of 6] [RFC] extension: `dummy` and composition with `lfs`
Rémi Chaintron
remi at fb.com
Wed Nov 16 09:12:50 EST 2016
On 11/9/16, 5:17 PM, "Augie Fackler" <raf at durin42.com> wrote:
On Thu, Oct 27, 2016 at 04:04:04PM +0100, Remi Chaintron wrote:
> # HG changeset patch
> # User Remi Chaintron <remi at fb.com>
> # Date 1477579974 -3600
> # Thu Oct 27 15:52:54 2016 +0100
> # Branch stable
> # Node ID c7dc98eb24cd40e46e73dea275b4c5b245155efb
> # Parent 202b2a6872d3b3ffb1f373910c4eadffda8f727a
> [RFC] extension: `dummy` and composition with `lfs`
>
> This is a proof of concept for the composition of non-reflexive operations on
> filelogs using extensions relying on the flagprocessor design.
Could this be tested using censor instead?
I currently have a prototype to move various bits and pieces from censor related core code into the extension, but given the extensive rework and redesign involved, I found it more applicable to provide an easier proof of concept instead.
I’d love to get a proposal up for review in regards to moving censor to the extension, but I’m not positive I will have the time to get everything in order to make it happen before the end of the year.
>
> diff --git a/hgext/dummy.py b/hgext/dummy.py
> new file mode 100644
> --- /dev/null
> +++ b/hgext/dummy.py
> @@ -0,0 +1,83 @@
> +# coding=UTF-8
> +
> +from __future__ import absolute_import
> +
> +from mercurial import (
> + cmdutil,
> + commands,
> + extensions,
> + filelog,
> + revlog,
> +)
> +
> +def wrappedread(self, text):
> + return text[6:], True
> +
> +def getfilelogcontent(self, text):
> + return text, False
> +
> +extensionflag = revlog.REVIDX_ISDUMMY
> +transformmap = {
> + extensionflag: wrappedread
> +}
> +
> +def revision(orig, self, nodeorrev, _df=None):
> + self.flagprocessor.register(transformmap)
> + try:
> + return orig(self, nodeorrev, _df=_df)
> + finally:
> + self.flagprocessor.unregister(transformmap)
> +
> +
> +def addrevision(orig, self, text, transaction, link, p1, p2, cachedelta=None,
> + node=None, flags=revlog.REVLOG_DEFAULT_FLAGS):
> + self.flagprocessor.register(transformmap)
> + try:
> + if self._peek_isdummy(text):
> + if node is None:
> + node = revlog.hash(text, p1, p2)
> + text = 'DUMMY:%s' % (text)
> + flags |= extensionflag
> + return orig(self, text, transaction, link, p1, p2, cachedelta=cachedelta,
> + node=node, flags=flags)
> + finally:
> + self.flagprocessor.unregister(transformmap)
> +
> +def addgroup(orig, self, cg, linkmapper, transaction, addrevisioncb=None):
> + self.flagprocessor.register(transformmap)
> + try:
> + return orig(self, cg, linkmapper, transaction,
> + addrevisioncb=addrevisioncb)
> + finally:
> + self.flagprocessor.unregister(transformmap)
> +
> +def debugdata(orig, ui, repo, file_, rev=None, **opts):
> + transformmap[extensionflag] = getfilelogcontent
> + orig(ui, repo, file_, rev=rev, **opts)
> + transformmap[extensionflag] = wrappedread
> +
> +def push(orig, ui, repo, dest=None, **opts):
> + transformmap[extensionflag] = getfilelogcontent
> + orig(ui, repo, dest=dest, **opts)
> + transformmap[extensionflag] = wrappedread
> +
> +def pull(orig, ui, repo, source="default", **opts):
> + transformmap[extensionflag] = getfilelogcontent
> + orig(ui, repo, source=source, **opts)
> + transformmap[extensionflag] = wrappedread
> +
> +def _peek_isdummy(orig, self, text):
> + return 'dummy' in text
> +
> +def extsetup(ui):
> + wrapfunction = extensions.wrapfunction
> + wrapcommand = extensions.wrapcommand
> +
> + wrapfunction(filelog.filelog, 'revision', revision)
> + wrapfunction(filelog.filelog, 'addrevision', addrevision)
> + wrapfunction(filelog.filelog, 'addgroup', addgroup)
> + wrapfunction(filelog.filelog, '_peek_isdummy', _peek_isdummy)
> +
> + wrapcommand(commands.table, 'debugdata', debugdata)
> + wrapcommand(commands.table, 'push', push)
> + wrapcommand(commands.table, 'pull', pull)
> diff --git a/mercurial/revlog.py b/mercurial/revlog.py
> --- a/mercurial/revlog.py
> +++ b/mercurial/revlog.py
> @@ -55,8 +55,9 @@
> # revlog index flags
> REVIDX_ISCENSORED = (1 << 15) # revision has censor metadata, must be verified
> REVIDX_ISLARGEFILE = (1 << 14)
> +REVIDX_ISDUMMY = (1 << 13)
> REVIDX_DEFAULT_FLAGS = 0
> -REVIDX_KNOWN_FLAGS = REVIDX_ISCENSORED | REVIDX_ISLARGEFILE
> +REVIDX_KNOWN_FLAGS = REVIDX_ISCENSORED | REVIDX_ISLARGEFILE | REVIDX_ISDUMMY
>
> # max size of revlog with inline data
> _maxinline = 131072
> @@ -97,6 +98,7 @@
> self.flagsbypriority = [
> REVIDX_ISCENSORED,
> REVIDX_ISLARGEFILE,
> + REVIDX_ISDUMMY,
> ]
> self.transformmap = {}
> self.revlogobject = revlogobject
> @@ -1775,6 +1777,9 @@
> def islargefile(self, rev):
> return False
>
> + def _peek_isdummy(self, text):
> + return False
> +
> def _peek_iscensored(self, baserev, delta, flush):
> """Quickly check if a delta produces a censored revision."""
> return False
> diff --git a/tests/test-lfs.t b/tests/test-lfs.t
> --- a/tests/test-lfs.t
> +++ b/tests/test-lfs.t
> @@ -19,6 +19,7 @@
> $ cat >> .hg/hgrc <<EOF
> > [extensions]
> > lfs=$TESTDIR/../hgext/lfs/
> + > dummy=
> > [lfs]
> > threshold=1000B
> > blobstore=cache/localblobstore
> @@ -49,19 +50,43 @@
> version https://git-lfs.github.com/spec/v1
> oid sha256:1228f8759b018cca5b58d3e5d1740d0b827f06cecf8868fd17f35a87ef8aacf6
> size 1501
> -
> -# Check the contents of the file are fetched from the blobstore when requested
> - $ hg cat -r . largefile
> + $ hg cat --traceback -r . largefile
> AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
>
> -# Push the blobs to the server
> - $ hg push
> +# Commit a dummy file (will take the dummy flag)
> + $ echo 'dummy' > dummyfile
> + $ hg commit -Aqm 'add dummy file'
> +
> +# Check the dummy extension adds metadata on write
> + $ hg debugdata dummyfile 0
> + DUMMY:dummy
> +
> +# Check the extension removes the metadata on read
> + $ hg cat -r . dummyfile
> + dummy
> +
> +# Commit a lfs + dummy triggering file
> + $ echo dummyAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA > dummylargefile
> + $ hg commit -Aqm 'add dummy large file'
> +
> +# Check the filelog contains metadata (as dummy has priority on lfs)
> + $ hg debugdata dummylargefile 0
> + version https://git-lfs.github.com/spec/v1
> + oid sha256:22a3bdbccf152a786e35ca856dfef423c022968fc42f80f1c2eff8e8b116c37d
> + size 1512
> +
> +# Check the file contents are restored on read
> + $ hg cat -r . dummylargefile
> + dummyAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
> +
> +# Push changes to the server
> + $ hg push --traceback
> pushing to $TESTTMP/server (glob)
> searching for changes
> adding changesets
> adding manifests
> adding file changes
> - added 2 changesets with 2 changes to 2 files
> + added 4 changesets with 4 changes to 4 files
>
> # Initialize new client (not cloning) and setup extension
> $ cd ../
> @@ -76,6 +101,7 @@
> > default = $TESTTMP/server
> > [extensions]
> > lfs=$TESTDIR/../hgext/lfs/
> + > dummy=
> > [lfs]
> > threshold=1000B
> > blobstore=cache/localblobstore
> @@ -88,7 +114,7 @@
> adding changesets
> adding manifests
> adding file changes
> - added 2 changesets with 2 changes to 2 files
> + added 4 changesets with 4 changes to 4 files
> (run 'hg update' to get a working copy)
>
> # Check the blobstore is not yet populated
> @@ -97,7 +123,7 @@
>
> # Update to the last revision containing the large file
> $ hg update
> - 2 files updated, 0 files merged, 0 files removed, 0 files unresolved
> + 4 files updated, 0 files merged, 0 files removed, 0 files unresolved
>
> # Check the filelog contents actually contain metadata
> $ hg debugdata largefile 0
> @@ -110,6 +136,8 @@
> .hg/cache/localblobstore
> .hg/cache/localblobstore/12
> .hg/cache/localblobstore/12/28f8759b018cca5b58d3e5d1740d0b827f06cecf8868fd17f35a87ef8aacf6
> + .hg/cache/localblobstore/22
> + .hg/cache/localblobstore/22/a3bdbccf152a786e35ca856dfef423c022968fc42f80f1c2eff8e8b116c37d
>
> # Check the contents of the file are fetched from blobstore when requested
> $ hg cat -r . largefile
> _______________________________________________
> Mercurial-devel mailing list
> Mercurial-devel at mercurial-scm.org
> https://urldefense.proofpoint.com/v2/url?u=https-3A__www.mercurial-2Dscm.org_mailman_listinfo_mercurial-2Ddevel&d=DQIFAg&c=5VD0RTtNlTh3ycd41b3MUw&r=RjAECLzUoKvzn49UTozsUg&m=5RyPgFuzIirEktiCbTiLncwaRvGm6WhEv21cqfaP5-o&s=40rBjU8_W8S68vg_RCGfz4bO3GsV4mSj784C1r4Jk70&e=
More information about the Mercurial-devel
mailing list