[PATCH 08 of 10] repair: migrate revlogs during upgrade

Augie Fackler raf at durin42.com
Mon Nov 21 17:01:51 EST 2016


On Sat, Nov 05, 2016 at 09:40:24PM -0700, Gregory Szorc wrote:
> # HG changeset patch
> # User Gregory Szorc <gregory.szorc at gmail.com>
> # Date 1478393405 25200
> #      Sat Nov 05 17:50:05 2016 -0700
> # Node ID d2261c558ca9639fb81c182de15d75151cbad0f9
> # Parent  958bcf2577608bbb6d8ae078cde0ca451f3ab31a
> repair: migrate revlogs during upgrade

[...]

>
> diff --git a/mercurial/repair.py b/mercurial/repair.py
> --- a/mercurial/repair.py
> +++ b/mercurial/repair.py
> @@ -11,15 +11,19 @@ from __future__ import absolute_import
>  import errno
>  import hashlib
>  import tempfile
> +import time
>
>  from .i18n import _
>  from .node import short
>  from . import (
>      bundle2,
>      changegroup,
> +    changelog,
>      error,
>      exchange,
> +    manifest,
>      obsolete,
> +    revlog,
>      scmutil,
>      util,
>  )
> @@ -537,6 +541,87 @@ def upgradesummarizeactions(repo, action
>
>      return l, handled
>
> +def _revlogfrompath(repo, path):
> +    """Obtain a revlog from a repo path.
> +
> +    An instance of the appropriate class is returned.
> +    """
> +    if path == '00changelog.i':
> +        return changelog.changelog(repo.svfs)
> +    elif path.endswith('00manifest.i'):
> +        mandir = path[:-len('00manifest.i')]
> +        return manifest.manifestrevlog(repo.svfs, dir=mandir)

Hm. It might merit special handling if there are any new revlogs in
.hg/store/meta/, so maybe we should be paranoid and sniff for that
too?

I can't envision anything else ever requiring a custom class though,
so probably not needed.

> +    else:
> +        # Filelogs don't do anything special with settings. So we can use a
> +        # vanilla revlog.
> +        return revlog.revlog(repo.svfs, path)
> +
> +def _copyrevlogs(ui, srcrepo, dstrepo, tr):
> +    """Copy revlogs between 2 repos.
> +
> +    Full decoding/encoding is performed on both ends, ensuring that revlog
> +    settings on the destination are honored.
> +    """
> +    rlcount = 0
> +    revcount = 0
> +    srcsize = 0
> +    dstsize = 0
> +
> +    # Perform a pass to collect metadata. This validates we can open all
> +    # source files and allows a unified progress bar to be displayed.
> +    for unencoded, encoded, size in srcrepo.store.walk():
> +        srcsize += size
> +        if unencoded.endswith('.d'):
> +            continue
> +        rl = _revlogfrompath(srcrepo, unencoded)
> +        rlcount += 1
> +        revcount += len(rl)
> +
> +    ui.write(_('migrating %d revlogs containing %d revisions (%d bytes)\n') %
> +             (rlcount, revcount, srcsize))
> +
> +    if not rlcount:
> +        return
> +
> +    # Used to keep track of progress.
> +    convertedcount = [0]
> +    def oncopiedrevision(rl, rev, node):
> +        convertedcount[0] += 1
> +        srcrepo.ui.progress(_('revisions'), convertedcount[0], total=revcount)
> +
> +    # Do the actual copying.
> +    # FUTURE this operation can be farmed off to worker processes.
> +    seen = set()
> +    for unencoded, encoded, size in srcrepo.store.walk():
> +        if unencoded.endswith('.d'):
> +            continue
> +
> +        ui.progress(_('revisions'), convertedcount[0], total=revcount)
> +
> +        oldrl = _revlogfrompath(srcrepo, unencoded)
> +        newrl = _revlogfrompath(dstrepo, unencoded)
> +
> +        if isinstance(oldrl, manifest.manifestrevlog) and 'm' not in seen:
> +            seen.add('m')
> +            ui.write(_('migrating manifests...\n'))
> +        elif isinstance(oldrl, changelog.changelog) and 'c' not in seen:
> +            seen.add('c')
> +            ui.write(_('migrating changelog...\n'))
> +        elif 'f' not in seen:
> +            seen.add('f')
> +            ui.write(_('migrating file histories...\n'))
> +
> +        ui.note(_('copying %d revisions from %s\n') % (len(oldrl), unencoded))
> +        oldrl.clone(tr, newrl, addrevisioncb=oncopiedrevision)
> +
> +        dstsize += newrl.totalfilesize()
> +
> +    ui.progress(_('revisions'), None)
> +
> +    ui.write(_('revlogs migration complete; wrote %d bytes (delta %d bytes) '

nit: I'd make revlog singular here.

> +               'across %d revlogs and %d revisions\n') % (
> +             dstsize, dstsize - srcsize, rlcount, revcount))
> +
>  def _upgraderepo(ui, srcrepo, dstrepo, requirements):
>      """Do the low-level work of upgrading a repository.
>
> @@ -550,7 +635,15 @@ def _upgraderepo(ui, srcrepo, dstrepo, r
>      assert srcrepo.currentwlock()
>      assert dstrepo.currentwlock()
>
> -    # TODO copy store
> +    ui.write(_('(it is safe to interrupt this process any time before '
> +               'data migration completes)\n'))
> +
> +    with dstrepo.transaction('upgrade') as tr:
> +        _copyrevlogs(ui, srcrepo, dstrepo, tr)
> +
> +        # TODO copy non-revlog store files
> +
> +    ui.write(_('data fully migrated to temporary repository\n'))
>
>      ui.write(_('starting in-place swap of repository data\n'))
>      ui.warn(_('(clients may error or see inconsistent repository data until '
> @@ -567,6 +660,17 @@ def _upgraderepo(ui, srcrepo, dstrepo, r
>      util.copyfile(srcrepo.join('requires'), backupvfs.join('requires'))
>      scmutil.writerequires(srcrepo.vfs, requirements)
>
> +    # Now swap in the new store directory. Doing it as a rename should make
> +    # the operation nearly instantaneous and atomic (at least in well-behaved
> +    # environments).
> +    ui.write(_('replacing store...\n'))
> +    tstart = time.time()
> +    util.rename(srcrepo.spath, backupvfs.join('store'))
> +    util.rename(dstrepo.spath, srcrepo.spath)
> +    elapsed = time.time() - tstart
> +    ui.write(_('store replacement complete; repository was inconsistent for '
> +               '%0.1fs\n') % elapsed)
> +
>  def upgraderepo(ui, repo, dryrun=False):
>      """Upgrade a repository in place."""
>      # Avoid cycle: cmdutil -> repair -> localrepo -> cmdutil
> diff --git a/tests/test-upgrade-repo.t b/tests/test-upgrade-repo.t
> --- a/tests/test-upgrade-repo.t
> +++ b/tests/test-upgrade-repo.t
> @@ -9,10 +9,15 @@
>    starting repository upgrade
>    source repository locked and read-only
>    creating temporary repository to stage migrated data: $TESTTMP/empty/.hg/upgrade.* (glob)
> +  (it is safe to interrupt this process any time before data migration completes)
> +  migrating 0 revlogs containing 0 revisions (0 bytes)
> +  data fully migrated to temporary repository
>    starting in-place swap of repository data
>    (clients may error or see inconsistent repository data until this operation completes)
>    replaced files will be backed up at $TESTTMP/empty/.hg/upgradebackup.* (glob)
>    updating requirements in $TESTTMP/empty/.hg/requires
> +  replacing store...
> +  store replacement complete; repository was inconsistent for 0.0s

missing glob on inconsistent time (it shouldn't ever be very long, but
hey, buildbots tend to be underpowered) (throughout this test file,
not commenting on the rest)

>    removing temporary repository $TESTTMP/empty/.hg/upgrade.* (glob)
>
>  dry run works
> @@ -51,10 +56,15 @@ Various sub-optimal detections work
>    starting repository upgrade
>    source repository locked and read-only
>    creating temporary repository to stage migrated data: $TESTTMP/empty/.hg/upgrade.* (glob)
> +  (it is safe to interrupt this process any time before data migration completes)
> +  migrating 0 revlogs containing 0 revisions (0 bytes)
> +  data fully migrated to temporary repository
>    starting in-place swap of repository data
>    (clients may error or see inconsistent repository data until this operation completes)
>    replaced files will be backed up at $TESTTMP/empty/.hg/upgradebackup.* (glob)
>    updating requirements in $TESTTMP/empty/.hg/requires
> +  replacing store...
> +  store replacement complete; repository was inconsistent for 0.0s
>    removing temporary repository $TESTTMP/empty/.hg/upgrade.* (glob)
>
>    $ cd ..
> @@ -133,10 +143,19 @@ Upgrading a repository to generaldelta w
>    starting repository upgrade
>    source repository locked and read-only
>    creating temporary repository to stage migrated data: $TESTTMP/upgradegd/.hg/upgrade.* (glob)
> +  (it is safe to interrupt this process any time before data migration completes)
> +  migrating 5 revlogs containing 9 revisions (917 bytes)
> +  migrating file histories...
> +  migrating manifests...
> +  migrating changelog...
> +  revlogs migration complete; wrote 917 bytes (delta 0 bytes) across 5 revlogs and 9 revisions
> +  data fully migrated to temporary repository
>    starting in-place swap of repository data
>    (clients may error or see inconsistent repository data until this operation completes)
>    replaced files will be backed up at $TESTTMP/upgradegd/.hg/upgradebackup.* (glob)
>    updating requirements in $TESTTMP/upgradegd/.hg/requires
> +  replacing store...
> +  store replacement complete; repository was inconsistent for 0.0s
>    removing temporary repository $TESTTMP/upgradegd/.hg/upgrade.* (glob)
>
>  Original requirements backed up
> @@ -156,4 +175,43 @@ generaldelta added to original requireme
>    revlogv1
>    store
>
> +store directory has files we expect
> +
> +  $ ls .hg/store
> +  00changelog.i
> +  00manifest.i
> +  data
> +  fncache
> +  undo
> +  undo.backupfiles
> +  undo.phaseroots
> +
> +manifest should be generaldelta
> +
> +  $ hg debugrevlog -m | grep flags
> +  flags  : inline, generaldelta
> +
> +verify should be happy
> +
> +  $ hg verify
> +  checking changesets
> +  checking manifests
> +  crosschecking files in changesets and manifests
> +  checking files
> +  3 files, 3 changesets, 3 total revisions
> +
> +old store should be backed up
> +
> +  $ ls .hg/upgradebackup.*/store
> +  00changelog.i
> +  00manifest.i
> +  data
> +  fncache
> +  lock
> +  phaseroots
> +  undo
> +  undo.backup.fncache
> +  undo.backupfiles
> +  undo.phaseroots
> +
>    $ cd ..
> _______________________________________________
> Mercurial-devel mailing list
> Mercurial-devel at mercurial-scm.org
> https://www.mercurial-scm.org/mailman/listinfo/mercurial-devel


More information about the Mercurial-devel mailing list