[PATCH 1 of 2 v2] repair: add functionality to rebuild fncache

Gregory Szorc gregory.szorc at gmail.com
Mon Jun 22 17:04:54 UTC 2015


# HG changeset patch
# User Gregory Szorc <gregory.szorc at gmail.com>
# Date 1434992388 25200
#      Mon Jun 22 09:59:48 2015 -0700
# Node ID 8f10b76a327e7ac8c4f6b2691c31cedd7e8fa5e6
# Parent  7fdd1782fc4ee9da87d8af13e806dc9055db2c38
repair: add functionality to rebuild fncache

Currently, there is no way to recover from a missing or corrupt fncache
file in place (a clone is required). For certain use cases such as
servers and with large repositories, an in-place repair may be
desirable. This patch adds functionality for in-place repair of the
fncache.

The `hg debugrebuildfncache` command is introduced. It ensures the
fncache is up to date by reconstructing the fncache from all seen files
encountered during a brute force traversal of the repository's entire
history.

The command will add missing entries and will prune excess ones.

Currently, the command no-ops unless the repository has the fncache
requirement. The command could later grow the ability to "upgrade" an
existing repository to be fncache enabled, if desired.

When testing this patch on a local clone of the Firefox repository, it
removed a bunch of entries. Investigation revealed that removed entries
belonged to empty (0 byte size) .i filelogs. The functionality for
pruning fncache of stripped revlogs was introduced in f49d60fa40a5, so
the presence of these entries likely predates this feature.

diff --git a/mercurial/commands.py b/mercurial/commands.py
--- a/mercurial/commands.py
+++ b/mercurial/commands.py
@@ -20,9 +20,9 @@ import merge as mergemod
 import minirst, revset, fileset
 import dagparser, context, simplemerge, graphmod, copies
 import random
 import setdiscovery, treediscovery, dagutil, pvec, localrepo
-import phases, obsolete, exchange, bundle2
+import phases, obsolete, exchange, bundle2, repair
 import ui as uimod
 
 table = {}
 
@@ -2727,8 +2727,13 @@ def debugrebuilddirstate(ui, repo, rev):
         repo.dirstate.rebuild(ctx.node(), ctx.manifest())
     finally:
         wlock.release()
 
+ at command('debugrebuildfncache', [], '')
+def debugrebuildfncache(ui, repo):
+    """rebuild the fncache file"""
+    repair.rebuildfncache(ui, repo)
+
 @command('debugrename',
     [('r', 'rev', '', _('revision to debug'), _('REV'))],
     _('[-r REV] FILE'))
 def debugrename(ui, repo, file1, *pats, **opts):
diff --git a/mercurial/repair.py b/mercurial/repair.py
--- a/mercurial/repair.py
+++ b/mercurial/repair.py
@@ -226,4 +226,73 @@ def strip(ui, repo, nodelist, backup=Tru
             # Remove partial backup only if there were no exceptions
             vfs.unlink(chgrpfile)
 
     repo.destroyed()
+
+def rebuildfncache(ui, repo):
+    """Rebuilds the fncache file from repo history.
+
+    Missing entries will be added. Extra entries will be removed.
+    """
+    repo = repo.unfiltered()
+
+    if 'fncache' not in repo.requirements:
+        ui.warn(_('(not rebuilding fncache because repository does not '
+                  'support fncache\n'))
+        return
+
+    lock = repo.lock()
+    try:
+        fnc = repo.store.fncache
+        # Trigger load of fncache.
+        if 'irrelevant' in fnc:
+            pass
+
+        oldentries = set(fnc.entries)
+        newentries = set()
+        seenfiles = set()
+
+        repolen = len(repo)
+        for rev in repo:
+            ui.progress(_('changeset'), rev, total=repolen)
+
+            ctx = repo[rev]
+            for f in ctx.files():
+                # This is to minimize I/O.
+                if f in seenfiles:
+                    continue
+                seenfiles.add(f)
+
+                i = 'data/%s.i' % f
+                d = 'data/%s.d' % f
+
+                if repo.store._exists(i):
+                    newentries.add(i)
+                if repo.store._exists(d):
+                    newentries.add(d)
+
+        ui.progress(_('changeset'), None)
+
+        addcount = len(newentries - oldentries)
+        removecount = len(oldentries - newentries)
+        for p in sorted(oldentries - newentries):
+            ui.write(_('removing %s\n') % p)
+        for p in sorted(newentries - oldentries):
+            ui.write(_('adding %s\n') % p)
+
+        if addcount or removecount:
+            ui.write(_('%d items added, %d removed from fncache\n') %
+                     (addcount, removecount))
+            fnc.entries = newentries
+            fnc._dirty = True
+
+            tr = repo.transaction('fncache')
+            try:
+                fnc.write(tr)
+                tr.close()
+            finally:
+                tr.release()
+        else:
+            ui.write(_('fncache already up to date\n'))
+    finally:
+        lock.release()
+
diff --git a/tests/test-completion.t b/tests/test-completion.t
--- a/tests/test-completion.t
+++ b/tests/test-completion.t
@@ -95,8 +95,9 @@ Show debug commands if there are no othe
   debugpathcomplete
   debugpushkey
   debugpvec
   debugrebuilddirstate
+  debugrebuildfncache
   debugrename
   debugrevlog
   debugrevspec
   debugsetparents
@@ -253,8 +254,9 @@ Show all commands + options
   debugpathcomplete: full, normal, added, removed
   debugpushkey: 
   debugpvec: 
   debugrebuilddirstate: rev
+  debugrebuildfncache: 
   debugrename: rev
   debugrevlog: changelog, manifest, dir, dump
   debugrevspec: optimize
   debugsetparents: 
diff --git a/tests/test-fncache.t b/tests/test-fncache.t
--- a/tests/test-fncache.t
+++ b/tests/test-fncache.t
@@ -282,4 +282,99 @@ Aborted transactions can be recovered la
   checking files
   1 files, 1 changesets, 1 total revisions
   $ cat .hg/store/fncache
   data/y.i
+
+  $ cd ..
+
+debugrebuildfncache does nothing unless repo has fncache requirement
+
+  $ hg --config format.usefncache=false init nofncache
+  $ cd nofncache
+  $ hg debugrebuildfncache
+  (not rebuilding fncache because repository does not support fncache
+
+  $ cd ..
+
+debugrebuildfncache works on empty repository
+
+  $ hg init empty
+  $ cd empty
+  $ hg debugrebuildfncache
+  fncache already up to date
+  $ cd ..
+
+debugrebuildfncache on an up to date repository no-ops
+
+  $ hg init repo
+  $ cd repo
+  $ echo initial > foo
+  $ echo initial > .bar
+  $ hg commit -A -m initial
+  adding .bar
+  adding foo
+
+  $ cat .hg/store/fncache | sort
+  data/.bar.i
+  data/foo.i
+
+  $ hg debugrebuildfncache
+  fncache already up to date
+
+debugrebuildfncache restores deleted fncache file
+
+  $ rm -f .hg/store/fncache
+  $ hg debugrebuildfncache
+  adding data/.bar.i
+  adding data/foo.i
+  2 items added, 0 removed from fncache
+
+  $ cat .hg/store/fncache | sort
+  data/.bar.i
+  data/foo.i
+
+Rebuild after rebuild should no-op
+
+  $ hg debugrebuildfncache
+  fncache already up to date
+
+A single missing file should get restored, an extra file should be removed
+
+  $ cat > .hg/store/fncache << EOF
+  > data/foo.i
+  > data/bad-entry.i
+  > EOF
+
+  $ hg debugrebuildfncache
+  removing data/bad-entry.i
+  adding data/.bar.i
+  1 items added, 1 removed from fncache
+
+  $ cat .hg/store/fncache | sort
+  data/.bar.i
+  data/foo.i
+
+  $ cd ..
+
+Try a simple variation without dotencode to ensure fncache is ignorant of encoding
+
+  $ hg --config format.dotencode=false init nodotencode
+  $ cd nodotencode
+  $ echo initial > foo
+  $ echo initial > .bar
+  $ hg commit -A -m initial
+  adding .bar
+  adding foo
+
+  $ cat .hg/store/fncache | sort
+  data/.bar.i
+  data/foo.i
+
+  $ rm .hg/store/fncache
+  $ hg debugrebuildfncache
+  adding data/.bar.i
+  adding data/foo.i
+  2 items added, 0 removed from fncache
+
+  $ cat .hg/store/fncache | sort
+  data/.bar.i
+  data/foo.i
diff --git a/tests/test-help.t b/tests/test-help.t
--- a/tests/test-help.t
+++ b/tests/test-help.t
@@ -795,8 +795,10 @@ Test list of internal help commands
    debugpvec     (no help text available)
    debugrebuilddirstate
                  rebuild the dirstate as it would look like for the given
                  revision
+   debugrebuildfncache
+                 rebuild the fncache file
    debugrename   dump rename information
    debugrevlog   show data and statistics about a revlog
    debugrevspec  parse and apply a revision specification
    debugsetparents


More information about the Mercurial-devel mailing list