[PATCH 5 of 6] hidden: add hiddenset and updatevisibility logic
Durham Goode
durham at fb.com
Thu May 18 14:23:59 EDT 2017
# HG changeset patch
# User Durham Goode <durham at fb.com>
# Date 1495129620 25200
# Thu May 18 10:47:00 2017 -0700
# Node ID 0979ce9d40803103441b6df1ebefafe80bb9a057
# Parent 2aef3c4be6ba81e9d2453760515517df6fc2b816
hidden: add hiddenset and updatevisibility logic
This adds a function that can incrementally update the hidden state for a
repository when given a list of nodes that have been changed. This will be
called by all actions that can affect hiddenness (bookmarks, tags, obsmarkers,
working copy parents). It uses a hash of the inputs that affect the hidden store
to ensure things are up-to-date.
This has the nice property that all logic for propagating hiddenness is
contained to one function, and all the consumers just have to report what
commits they touched. If an external extension wants to modify how hiding is
computed, it can just wrap this one function.
It also is nice because it updates the hidden state incrementally, instead of
recomputing the entire repo at once.
diff --git a/mercurial/hidden.py b/mercurial/hidden.py
--- a/mercurial/hidden.py
+++ b/mercurial/hidden.py
@@ -5,12 +5,38 @@
from __future__ import absolute_import
+import hashlib
+import heapq
+
from .i18n import _
-from .node import bin, hex
+from .node import bin, hex, nullid, nullrev
from . import (
error,
+ obsolete,
+ tags as tagsmod,
+ util,
)
+class lazychildset(object):
+ """Data structure for efficiently answering the query 'what are my children'
+ for a series of revisions."""
+ def __init__(self, repo):
+ self.repo = repo.unfiltered()
+ self._childmap = {}
+ self._minprocessed = len(repo.changelog)
+
+ def get(self, rev):
+ childmap = self._childmap
+ cl = self.repo.changelog
+ if rev < self._minprocessed:
+ for r in xrange(rev, self._minprocessed):
+ for p in cl.parentrevs(r):
+ if p != nullrev:
+ childmap.setdefault(p, []).append(r)
+ self._minprocessed = rev
+
+ return childmap.get(rev, ())
+
class rootset(object):
"""A commit set like object where a commit being in the set implies that all
descendants of that commit are also in the set.
@@ -135,3 +161,168 @@ class rootset(object):
def __len__(self):
self._load()
return len(self._cache)
+
+class hiddenset(rootset):
+ """A set representing what commits should be hidden. It contains logic to
+ update itself based on what nodes are reported as having been changed.
+ """
+ def __init__(self, repo, opener, name):
+ super(hiddenset, self).__init__(repo, opener, name)
+ self._load()
+
+ def _load(self):
+ if self.roots is None:
+ try:
+ cachehash = None
+ raw = self.opener.read(self.filename)
+ if len(raw) >= 20:
+ cachehash = raw[:20]
+ raw = raw[20:]
+ else:
+ raise RuntimeError("invalid cache")
+
+ realhash = self._gethash()
+ if cachehash == realhash:
+ self.roots = self._deserialize(raw)
+ else:
+ raise RuntimeError("cache hash mismatch")
+ except (IOError, RuntimeError):
+ # Rebuild the hidden roots from scratch
+ self.roots = set()
+ self._cache = set()
+ revs = self.repo.revs('not public()')
+ self.updatevisibility(revs)
+
+ self._fillcache()
+ self.dirty = False
+
+ def _write(self, fp):
+ fp.write(self._gethash())
+ fp.write(self._serialize())
+ self.dirty = False
+
+ def _gethash(self):
+ hiderevs, nohiderevs = self._gethiderevs()
+ cl = self.repo.changelog
+ hash = hashlib.sha1()
+ hash.update("hide")
+ hash.update(''.join(sorted(cl.node(r) for r in hiderevs)))
+ hash.update("nohide")
+ hash.update(''.join(sorted(cl.node(r) for r in nohiderevs)))
+
+ return hash.digest()
+
+ def _gethiderevs(self):
+ """Returns the set of revs that potentially could be hidden and a set of
+ revs that definitely should not be hidden."""
+ hiderevs = set()
+ nohiderevs = set()
+
+ repo = self.repo
+ cl = repo.changelog
+
+ # Hide obsolete commits
+ hiderevs.update(obsolete.getrevs(repo, 'obsolete'))
+
+ # Don't hide bookmarks
+ nohiderevs.update(cl.rev(node) for name, node in
+ repo._bookmarks.iteritems())
+
+ # Don't hide workingcopy parents
+ nohiderevs.update(cl.rev(node) for node in repo.dirstate.parents()
+ if node != nullid and node in cl.nodemap)
+
+ # Don't hide local tags
+ tags = {}
+ tagsmod.readlocaltags(repo.ui, repo, tags, {})
+ nohiderevs.update(cl.rev(t[0]) for t in tags.values()
+ if t[0] in cl.nodemap)
+
+ # Don't hide rebase commits
+ if util.safehasattr(repo, '_rebaseset'):
+ nohiderevs.update(repo._rebaseset)
+
+ return hiderevs, nohiderevs
+
+ def _shouldhide(self, rev, childmap, hiderevs, nohiderevs):
+ repo = self.repo
+ ispublic = repo._phasecache.phase(repo, rev) == 0
+ cl = repo.changelog
+ return (not ispublic and
+ rev in hiderevs and
+ rev not in nohiderevs and
+ all(cl.node(r) in self for r in childmap.get(rev)))
+
+ def _needupdate(self, revs):
+ repo = self.repo
+ cl = repo.changelog
+
+ hiderevs, nohiderevs = self._gethiderevs()
+
+ childmap = lazychildset(repo)
+ for rev in revs:
+ node = cl.node(rev)
+ # If the node should be visible...
+ if not self._shouldhide(rev, childmap, hiderevs, nohiderevs):
+ # ...but it's hidden
+ if node in self:
+ return True
+ # Otherwise, the node should be hidden
+ elif node not in self:
+ return True
+
+ return False
+
+ def updatevisibility(self, revs):
+ """Updates the visibility of the given revs, and propagates those
+ updates to the rev parents as necessary."""
+ self._load()
+
+ repo = self.repo
+ seen = set(revs)
+ if not seen or not self._needupdate(seen):
+ # Even if there are no changes, we need to rewrite the file so the
+ # cache hash is up to date with the latest changes.
+ with self.opener(self.filename, 'w+', atomictemp=True) as fp:
+ self._write(fp)
+ return
+
+ # Sort the revs so we can process them in topo order
+ heap = []
+ for rev in seen:
+ heapq.heappush(heap, -rev)
+
+ with repo.lock():
+ with repo.transaction('hidden') as tr:
+ hiderevs, nohiderevs = self._gethiderevs()
+
+ cl = repo.changelog
+ childmap = lazychildset(repo)
+ while heap:
+ rev = -heapq.heappop(heap)
+
+ # If it should be visible...
+ node = cl.node(rev)
+ if not self._shouldhide(rev, childmap, hiderevs,
+ nohiderevs):
+ # And it's currently invisible...
+ if node in self:
+ # Make it visible and propagate
+ # (propogate first, since otherwise we can't access
+ # the parents)
+ for parent in cl.parentrevs(rev):
+ if parent != nullrev and parent not in seen:
+ heapq.heappush(heap, -parent)
+ seen.add(parent)
+ self.set(tr, cl.node(rev), False,
+ childmap=childmap)
+
+ # If it should not be visible, and it's not already hidden..
+ elif node not in self:
+ # Hide it and propagate (propogate first, since
+ # otherwise we can't access the parents)
+ for parent in cl.parentrevs(rev):
+ if parent != nullrev and parent not in seen:
+ heapq.heappush(heap, -parent)
+ seen.add(parent)
+ self.set(tr, node, True, childmap=childmap)
diff --git a/mercurial/localrepo.py b/mercurial/localrepo.py
--- a/mercurial/localrepo.py
+++ b/mercurial/localrepo.py
@@ -388,7 +388,7 @@ class localrepository(object):
@storecache('hidden.roots')
def hidden(self):
- return hidden.rootset(self, self.svfs, 'hidden')
+ return hidden.hiddenset(self, self.svfs, 'hidden')
def close(self):
self._writecaches()
@@ -1384,6 +1384,10 @@ class localrepository(object):
l = self._lock(self.svfs, "lock", wait, None,
self.invalidate, _('repository %s') % self.origroot)
self._lockref = weakref.ref(l)
+ # Force the hidden cache to load, so it can validate it's cache hash
+ # against the unmodified bookmark, obsolete, tags, and working copy
+ # states.
+ self.hidden
return l
def _wlockchecktransaction(self):
diff --git a/tests/test-empty.t b/tests/test-empty.t
--- a/tests/test-empty.t
+++ b/tests/test-empty.t
@@ -26,6 +26,7 @@ Check the basic files created:
Should be empty:
$ ls .hg/store
+ hidden.roots
Poke at a clone:
@@ -49,5 +50,6 @@ Poke at a clone:
Should be empty:
$ ls .hg/store
+ hidden.roots
$ cd ..
diff --git a/tests/test-fncache.t b/tests/test-fncache.t
--- a/tests/test-fncache.t
+++ b/tests/test-fncache.t
@@ -92,6 +92,7 @@ Non store repo:
.hg/data/tst.d.hg
.hg/data/tst.d.hg/foo.i
.hg/dirstate
+ .hg/hidden.roots
.hg/last-message.txt
.hg/phaseroots
.hg/requires
@@ -129,6 +130,7 @@ Non fncache repo:
.hg/store/data
.hg/store/data/tst.d.hg
.hg/store/data/tst.d.hg/_foo.i
+ .hg/store/hidden.roots
.hg/store/phaseroots
.hg/store/undo
.hg/store/undo.backupfiles
@@ -313,6 +315,7 @@ Aborted transactions can be recovered la
data/z.i
$ hg recover
rolling back interrupted transaction
+ warning: ignoring unknown working parent 4b0f1917ccc5!
checking changesets
checking manifests
crosschecking files in changesets and manifests
diff --git a/tests/test-hardlinks.t b/tests/test-hardlinks.t
--- a/tests/test-hardlinks.t
+++ b/tests/test-hardlinks.t
@@ -48,6 +48,7 @@ Prepare repo r1:
1 r1/.hg/store/data/d1/f2.i
1 r1/.hg/store/data/f1.i
1 r1/.hg/store/fncache
+ 1 r1/.hg/store/hidden.roots
1 r1/.hg/store/phaseroots
1 r1/.hg/store/undo
1 r1/.hg/store/undo.backup.fncache
@@ -87,6 +88,7 @@ Repos r1 and r2 should now contain hardl
2 r1/.hg/store/data/d1/f2.i
2 r1/.hg/store/data/f1.i
2 r1/.hg/store/fncache
+ 1 r1/.hg/store/hidden.roots
1 r1/.hg/store/phaseroots
1 r1/.hg/store/undo
1 r1/.hg/store/undo.backup.fncache
@@ -99,6 +101,7 @@ Repos r1 and r2 should now contain hardl
2 r2/.hg/store/data/d1/f2.i
2 r2/.hg/store/data/f1.i
2 r2/.hg/store/fncache
+ 1 r2/.hg/store/hidden.roots
Repo r3 should not be hardlinked:
@@ -108,6 +111,7 @@ Repo r3 should not be hardlinked:
1 r3/.hg/store/data/d1/f2.i
1 r3/.hg/store/data/f1.i
1 r3/.hg/store/fncache
+ 1 r3/.hg/store/hidden.roots
1 r3/.hg/store/phaseroots
1 r3/.hg/store/undo
1 r3/.hg/store/undo.backupfiles
@@ -134,6 +138,7 @@ Create a non-inlined filelog in r3:
1 r3/.hg/store/data/d1/f2.i
1 r3/.hg/store/data/f1.i
1 r3/.hg/store/fncache
+ 1 r3/.hg/store/hidden.roots
1 r3/.hg/store/phaseroots
1 r3/.hg/store/undo
1 r3/.hg/store/undo.backup.fncache
@@ -167,6 +172,7 @@ Push to repo r1 should break up most har
1 r2/.hg/store/data/d1/f2.i
2 r2/.hg/store/data/f1.i
[12] r2/\.hg/store/fncache (re)
+ 1 r2/.hg/store/hidden.roots
#if hardlink-whitelisted
$ nlinksdir r2/.hg/store/fncache
@@ -197,6 +203,7 @@ Committing a change to f1 in r1 must bre
1 r2/.hg/store/data/d1/f2.i
1 r2/.hg/store/data/f1.i
[12] r2/\.hg/store/fncache (re)
+ 1 r2/.hg/store/hidden.roots
#if hardlink-whitelisted
$ nlinksdir r2/.hg/store/fncache
@@ -245,6 +252,7 @@ r4 has hardlinks in the working dir (not
2 r4/.hg/store/data/d1/f2.i
2 r4/.hg/store/data/f1.i
2 r4/.hg/store/fncache
+ 2 r4/.hg/store/hidden.roots
2 r4/.hg/store/phaseroots
2 r4/.hg/store/undo
2 r4/.hg/store/undo.backup.fncache
@@ -294,6 +302,7 @@ Update back to revision 11 in r4 should
2 r4/.hg/store/data/d1/f2.i
2 r4/.hg/store/data/f1.i
2 r4/.hg/store/fncache
+ 2 r4/.hg/store/hidden.roots
2 r4/.hg/store/phaseroots
2 r4/.hg/store/undo
2 r4/.hg/store/undo.backup.fncache
diff --git a/tests/test-hook.t b/tests/test-hook.t
--- a/tests/test-hook.t
+++ b/tests/test-hook.t
@@ -190,6 +190,7 @@ more there after
00manifest.i
data
fncache
+ hidden.roots
journal.phaseroots
phaseroots
undo
diff --git a/tests/test-inherit-mode.t b/tests/test-inherit-mode.t
--- a/tests/test-inherit-mode.t
+++ b/tests/test-inherit-mode.t
@@ -79,6 +79,7 @@ new directories are setgid
00660 ./.hg/store/data/dir/bar.i
00660 ./.hg/store/data/foo.i
00660 ./.hg/store/fncache
+ 00660 ./.hg/store/hidden.roots
00660 ./.hg/store/phaseroots
00660 ./.hg/store/undo
00660 ./.hg/store/undo.backupfiles
@@ -125,6 +126,7 @@ group can still write everything
00660 ../push/.hg/store/data/dir/bar.i
00660 ../push/.hg/store/data/foo.i
00660 ../push/.hg/store/fncache
+ 00660 ../push/.hg/store/hidden.roots
00660 ../push/.hg/store/undo
00660 ../push/.hg/store/undo.backupfiles
00660 ../push/.hg/store/undo.phaseroots
diff --git a/tests/test-upgrade-repo.t b/tests/test-upgrade-repo.t
--- a/tests/test-upgrade-repo.t
+++ b/tests/test-upgrade-repo.t
@@ -195,6 +195,7 @@ Upgrading a repository that is already m
repository locked and read-only
creating temporary repository to stage migrated data: $TESTTMP/modern/.hg/upgrade.* (glob)
(it is safe to interrupt this process any time before data migration completes)
+ copying hidden.roots
data fully migrated to temporary repository
marking source repository as being upgraded; clients will be unable to read from repository
starting in-place swap of repository data
@@ -241,6 +242,7 @@ Upgrading a repository to generaldelta w
migrating changelog containing 3 revisions (184 bytes in store; 181 bytes tracked data)
finished migrating 3 changelog revisions; change in size: 0 bytes
finished migrating 9 total revisions; total change in store size: 0 bytes
+ copying hidden.roots
copying phaseroots
data fully migrated to temporary repository
marking source repository as being upgraded; clients will be unable to read from repository
@@ -277,6 +279,7 @@ store directory has files we expect
00manifest.i
data
fncache
+ hidden.roots
phaseroots
undo
undo.backupfiles
@@ -303,6 +306,7 @@ old store should be backed up
00manifest.i
data
fncache
+ hidden.roots
phaseroots
undo
undo.backup.fncache
@@ -339,6 +343,7 @@ store files with special filenames aren'
finished migrating 1 changelog revisions; change in size: 0 bytes
finished migrating 3 total revisions; total change in store size: 0 bytes
copying .XX_special_filename
+ copying hidden.roots
copying phaseroots
data fully migrated to temporary repository
marking source repository as being upgraded; clients will be unable to read from repository
diff --git a/tests/test-verify.t b/tests/test-verify.t
--- a/tests/test-verify.t
+++ b/tests/test-verify.t
@@ -78,6 +78,7 @@ Entire changelog missing
$ rm .hg/store/00changelog.*
$ hg verify -q
+ warning: ignoring unknown working parent c5ddb05ab828!
0: empty or missing changelog
manifest at 0: d0b6632564d4 not in changesets
manifest at 1: 941fc4534185 not in changesets
@@ -116,6 +117,7 @@ Entire changelog and manifest log missin
$ rm .hg/store/00changelog.*
$ rm .hg/store/00manifest.*
$ hg verify -q
+ warning: ignoring unknown working parent c5ddb05ab828!
warning: orphan revlog 'data/file.i'
1 warnings encountered!
$ cp -R .hg/store-full/. .hg/store
@@ -125,6 +127,7 @@ Entire changelog and filelog missing
$ rm .hg/store/00changelog.*
$ rm .hg/store/data/file.*
$ hg verify -q
+ warning: ignoring unknown working parent c5ddb05ab828!
0: empty or missing changelog
manifest at 0: d0b6632564d4 not in changesets
manifest at 1: 941fc4534185 not in changesets
@@ -158,6 +161,7 @@ Changelog missing entry
$ cp -f .hg/store-partial/00changelog.* .hg/store
$ hg verify -q
+ warning: ignoring unknown working parent c5ddb05ab828!
manifest@?: rev 1 points to nonexistent changeset 1
manifest@?: 941fc4534185 not in changesets
file@?: rev 1 points to nonexistent changeset 1
@@ -193,6 +197,7 @@ Changelog and manifest log missing entry
$ cp -f .hg/store-partial/00changelog.* .hg/store
$ cp -f .hg/store-partial/00manifest.* .hg/store
$ hg verify -q
+ warning: ignoring unknown working parent c5ddb05ab828!
file@?: rev 1 points to nonexistent changeset 1
(expected 0)
file@?: c10f2164107d not in manifests
@@ -206,6 +211,7 @@ Changelog and filelog missing entry
$ cp -f .hg/store-partial/00changelog.* .hg/store
$ cp -f .hg/store-partial/data/file.* .hg/store/data
$ hg verify -q
+ warning: ignoring unknown working parent c5ddb05ab828!
manifest@?: rev 1 points to nonexistent changeset 1
manifest@?: 941fc4534185 not in changesets
file@?: manifest refers to unknown revision c10f2164107d
More information about the Mercurial-devel
mailing list