[PATCH 4 of 4 v4] treemanifests: fix streaming clone
Augie Fackler
raf at durin42.com
Fri Feb 5 16:48:04 EST 2016
On Fri, Feb 05, 2016 at 03:17:26PM -0600, Martin von Zweigbergk wrote:
> # HG changeset patch
> # User Martin von Zweigbergk <martinvonz at google.com>
> # Date 1454603647 28800
> # Thu Feb 04 08:34:07 2016 -0800
> # Node ID 10d703242604ae8e9ecd505e4322153333b191a3
> # Parent 30acd1be0f83c981cf90f1b03666910f6a2b39d7
> treemanifests: fix streaming clone
Queued these. Thanks for cleaning up my mess.
>
> Similar to the previous patch, the .hg/store/meta/ directory does not
> get copied when when using "hg clone --uncompressed". Fix by including
> "meta/" in store.datafiles(). This seems safe to do, as there are only
> a few users of this method. "hg manifest" already filters the paths by
> "data/" prefix. The calls from largefiles also seem safe. The use in
> verify needs updating to prevent it from mistaking dirlogs for
> orphaned filelogs. That change is included in this patch.
>
> Since the dirlogs will now be in the fncache when using fncachestore,
> let's also update debugrebuildfncache(). That will also allow any
> existing treemanifest repos to get their dirlogs into the fncache.
>
> Also update test-treemanifest.t to use an a directory name that
> requires dot-encoding and uppercase-encoding so we test that the path
> encoding works.
>
> diff --git a/mercurial/repair.py b/mercurial/repair.py
> --- a/mercurial/repair.py
> +++ b/mercurial/repair.py
> @@ -273,6 +273,16 @@
>
> ui.progress(_('changeset'), None)
>
> + if 'treemanifest' in repo: # safe but unnecessary otherwise
> + for dir in util.dirs(seenfiles):
> + i = 'meta/%s/00manifest.i' % dir
> + d = 'meta/%s/00manifest.d' % dir
> +
> + if repo.store._exists(i):
> + newentries.add(i)
> + if repo.store._exists(d):
> + newentries.add(d)
> +
> addcount = len(newentries - oldentries)
> removecount = len(oldentries - newentries)
> for p in sorted(oldentries - newentries):
> diff --git a/mercurial/store.py b/mercurial/store.py
> --- a/mercurial/store.py
> +++ b/mercurial/store.py
> @@ -330,7 +330,7 @@
> return l
>
> def datafiles(self):
> - return self._walk('data', True)
> + return self._walk('data', True) + self._walk('meta', True)
>
> def topfiles(self):
> # yield manifest before changelog
> @@ -378,7 +378,7 @@
> self.opener = self.vfs
>
> def datafiles(self):
> - for a, b, size in self._walk('data', True):
> + for a, b, size in super(encodedstore, self).datafiles():
> try:
> a = decodefilename(a)
> except KeyError:
> @@ -460,7 +460,8 @@
> self.encode = encode
>
> def __call__(self, path, mode='r', *args, **kw):
> - if mode not in ('r', 'rb') and path.startswith('data/'):
> + if mode not in ('r', 'rb') and (path.startswith('data/') or
> + path.startswith('meta/')):
> self.fncache.add(path)
> return self.vfs(self.encode(path), mode, *args, **kw)
>
> diff --git a/mercurial/verify.py b/mercurial/verify.py
> --- a/mercurial/verify.py
> +++ b/mercurial/verify.py
> @@ -284,7 +284,7 @@
> for f, f2, size in repo.store.datafiles():
> if not f:
> self.err(None, _("cannot decode filename '%s'") % f2)
> - elif size > 0 or not revlogv1:
> + elif (size > 0 or not revlogv1) and f.startswith('data/'):
> storefiles.add(_normpath(f))
>
> files = sorted(set(filenodes) | set(filelinkrevs))
> diff --git a/tests/run-tests.py b/tests/run-tests.py
> --- a/tests/run-tests.py
> +++ b/tests/run-tests.py
> @@ -719,6 +719,8 @@
> (br':%d\b' % self._startport, b':$HGPORT'),
> (br':%d\b' % (self._startport + 1), b':$HGPORT1'),
> (br':%d\b' % (self._startport + 2), b':$HGPORT2'),
> + (br':%d\b' % (self._startport + 2), b':$HGPORT3'),
> + (br':%d\b' % (self._startport + 2), b':$HGPORT4'),
> (br'(?m)^(saved backup bundle to .*\.hg)( \(glob\))?$',
> br'\1 (glob)'),
> ]
> @@ -741,6 +743,8 @@
> env["HGPORT"] = str(self._startport)
> env["HGPORT1"] = str(self._startport + 1)
> env["HGPORT2"] = str(self._startport + 2)
> + env["HGPORT3"] = str(self._startport + 3)
> + env["HGPORT4"] = str(self._startport + 4)
> env["HGRCPATH"] = os.path.join(self._threadtmp, b'.hgrc')
> env["DAEMON_PIDS"] = os.path.join(self._threadtmp, b'daemon.pids')
> env["HGEDITOR"] = ('"' + sys.executable + '"'
> diff --git a/tests/test-treemanifest.t b/tests/test-treemanifest.t
> --- a/tests/test-treemanifest.t
> +++ b/tests/test-treemanifest.t
> @@ -367,7 +367,7 @@
> $ hg --config experimental.treemanifest=True init deeprepo
> $ cd deeprepo
>
> - $ mkdir a
> + $ mkdir .A
> $ mkdir b
> $ mkdir b/bar
> $ mkdir b/bar/orange
> @@ -376,8 +376,8 @@
> $ mkdir b/foo/apple
> $ mkdir b/foo/apple/bees
>
> - $ touch a/one.txt
> - $ touch a/two.txt
> + $ touch .A/one.txt
> + $ touch .A/two.txt
> $ touch b/bar/fruits.txt
> $ touch b/bar/orange/fly/gnat.py
> $ touch b/bar/orange/fly/housefly.txt
> @@ -393,8 +393,8 @@
> Test files from the root.
>
> $ hg files -r .
> - a/one.txt (glob)
> - a/two.txt (glob)
> + .A/one.txt (glob)
> + .A/two.txt (glob)
> b/bar/fruits.txt (glob)
> b/bar/orange/fly/gnat.py (glob)
> b/bar/orange/fly/housefly.txt (glob)
> @@ -412,7 +412,7 @@
>
> Test files for a subdirectory.
>
> - $ rm -r .hg/store/meta/a
> + $ rm -r .hg/store/meta/~2e_a
> $ hg files -r . b
> b/bar/fruits.txt (glob)
> b/bar/orange/fly/gnat.py (glob)
> @@ -422,7 +422,7 @@
>
> Test files with just includes and excludes.
>
> - $ rm -r .hg/store/meta/a
> + $ rm -r .hg/store/meta/~2e_a
> $ rm -r .hg/store/meta/b/bar/orange/fly
> $ rm -r .hg/store/meta/b/foo/apple/bees
> $ hg files -r . -I path:b/bar -X path:b/bar/orange/fly -I path:b/foo -X path:b/foo/apple/bees
> @@ -431,7 +431,7 @@
>
> Test files for a subdirectory, excluding a directory within it.
>
> - $ rm -r .hg/store/meta/a
> + $ rm -r .hg/store/meta/~2e_a
> $ rm -r .hg/store/meta/b/foo
> $ hg files -r . -X path:b/foo b
> b/bar/fruits.txt (glob)
> @@ -442,7 +442,7 @@
> Test files for a sub directory, including only a directory within it, and
> including an unrelated directory.
>
> - $ rm -r .hg/store/meta/a
> + $ rm -r .hg/store/meta/~2e_a
> $ rm -r .hg/store/meta/b/foo
> $ hg files -r . -I path:b/bar/orange -I path:a b
> b/bar/orange/fly/gnat.py (glob)
> @@ -452,7 +452,7 @@
> Test files for a pattern, including a directory, and excluding a directory
> within that.
>
> - $ rm -r .hg/store/meta/a
> + $ rm -r .hg/store/meta/~2e_a
> $ rm -r .hg/store/meta/b/foo
> $ rm -r .hg/store/meta/b/bar/orange
> $ hg files -r . glob:**.txt -I path:b/bar -X path:b/bar/orange
> @@ -488,8 +488,6 @@
> Tree manifest revlogs exist.
> $ find deepclone/.hg/store/meta | sort
> deepclone/.hg/store/meta
> - deepclone/.hg/store/meta/a
> - deepclone/.hg/store/meta/a/00manifest.i
> deepclone/.hg/store/meta/b
> deepclone/.hg/store/meta/b/00manifest.i
> deepclone/.hg/store/meta/b/bar
> @@ -504,6 +502,8 @@
> deepclone/.hg/store/meta/b/foo/apple/00manifest.i
> deepclone/.hg/store/meta/b/foo/apple/bees
> deepclone/.hg/store/meta/b/foo/apple/bees/00manifest.i
> + deepclone/.hg/store/meta/~2e_a
> + deepclone/.hg/store/meta/~2e_a/00manifest.i
> Verify passes.
> $ cd deepclone
> $ hg verify
> @@ -525,8 +525,12 @@
> added 3 changesets with 10 changes to 8 files
> updating to branch default
> 8 files updated, 0 files merged, 0 files removed, 0 files unresolved
> - $ grep store deeprepo-basicstore/.hg/requires
> + $ cd deeprepo-basicstore
> + $ grep store .hg/requires
> [1]
> + $ hg serve -p $HGPORT3 -d --pid-file=hg.pid --errorlog=errors.log
> + $ cat hg.pid >> $DAEMON_PIDS
> + $ cd ..
> $ hg clone --config format.usefncache=False \
> > --config experimental.changegroup3=True \
> > http://localhost:$HGPORT2 deeprepo-encodedstore
> @@ -537,8 +541,12 @@
> added 3 changesets with 10 changes to 8 files
> updating to branch default
> 8 files updated, 0 files merged, 0 files removed, 0 files unresolved
> - $ grep fncache deeprepo-encodedstore/.hg/requires
> + $ cd deeprepo-encodedstore
> + $ grep fncache .hg/requires
> [1]
> + $ hg serve -p $HGPORT4 -d --pid-file=hg.pid --errorlog=errors.log
> + $ cat hg.pid >> $DAEMON_PIDS
> + $ cd ..
>
> Local clone with basicstore
> $ hg clone -U deeprepo-basicstore local-clone-basicstore
> @@ -566,3 +574,55 @@
> crosschecking files in changesets and manifests
> checking files
> 8 files, 3 changesets, 10 total revisions
> +
> +Stream clone with basicstore
> + $ hg clone --config experimental.changegroup3=True --uncompressed -U \
> + > http://localhost:$HGPORT3 stream-clone-basicstore
> + streaming all changes
> + 18 files to transfer, * of data (glob)
> + transferred * in * seconds (*) (glob)
> + searching for changes
> + no changes found
> + $ hg -R stream-clone-basicstore verify
> + checking changesets
> + checking manifests
> + crosschecking files in changesets and manifests
> + checking files
> + 8 files, 3 changesets, 10 total revisions
> +
> +Stream clone with encodedstore
> + $ hg clone --config experimental.changegroup3=True --uncompressed -U \
> + > http://localhost:$HGPORT4 stream-clone-encodedstore
> + streaming all changes
> + 18 files to transfer, * of data (glob)
> + transferred * in * seconds (*) (glob)
> + searching for changes
> + no changes found
> + $ hg -R stream-clone-encodedstore verify
> + checking changesets
> + checking manifests
> + crosschecking files in changesets and manifests
> + checking files
> + 8 files, 3 changesets, 10 total revisions
> +
> +Stream clone with fncachestore
> + $ hg clone --config experimental.changegroup3=True --uncompressed -U \
> + > http://localhost:$HGPORT2 stream-clone-fncachestore
> + streaming all changes
> + 18 files to transfer, * of data (glob)
> + transferred * in * seconds (*) (glob)
> + searching for changes
> + no changes found
> + $ hg -R stream-clone-fncachestore verify
> + checking changesets
> + checking manifests
> + crosschecking files in changesets and manifests
> + checking files
> + 8 files, 3 changesets, 10 total revisions
> +
> +Packed bundle
> + $ hg -R deeprepo debugcreatestreamclonebundle repo-packed.hg
> + writing 3349 bytes for 18 files
> + bundle requirements: generaldelta, revlogv1, treemanifest
> + $ hg debugbundle --spec repo-packed.hg
> + none-packed1;requirements%3Dgeneraldelta%2Crevlogv1%2Ctreemanifest
> _______________________________________________
> Mercurial-devel mailing list
> Mercurial-devel at mercurial-scm.org
> https://www.mercurial-scm.org/mailman/listinfo/mercurial-devel
More information about the Mercurial-devel
mailing list