[PATCH 2 of 3] lfs: prefetch lfs blobs when applying merge updates

Matt Harbison mharbison72 at gmail.com
Sun Feb 4 01:18:24 EST 2018


# HG changeset patch
# User Matt Harbison <matt_harbison at yahoo.com>
# Date 1517711172 18000
#      Sat Feb 03 21:26:12 2018 -0500
# Node ID e779b0c34d6989e45cabbf8da8cdfeb060134dbd
# Parent  359c61b92d8b47b866a8b527829643f45bba8970
lfs: prefetch lfs blobs when applying merge updates

In addition to merge, this method ultimately gets called by many commands:

  - backout
  - bisect
  - clone
  - fetch
  - graft
  - import (without --bypass)
  - pull -u
  - rebase
  - strip
  - share
  - transplant
  - unbundle
  - update

Additionally, it's also called by histedit, shelve, unshelve, and split, but it
seems that the related blobs should always be available locally for these.

For `hg update`, it happens after the normal argument checking and pre-update
hook processing, and remote corruption is detected prior to manipulating the
working directory.  Other commands could use this treatment (archive, cat,
revert, etc), but this covers so many of the frequently used bulk commands, it
seems like a good starting point.

Losing the verbose message that prints the file name before a corrupt blob
aborts the command is a little sad, because there's no easy way to go from oid
to file name.  I'd like to change that message to list the file name so it looks
cleaner and less cryptic, but the pointer object is nowhere near where it needs
to be to do this.  So punt on that for now.

diff --git a/hgext/lfs/__init__.py b/hgext/lfs/__init__.py
--- a/hgext/lfs/__init__.py
+++ b/hgext/lfs/__init__.py
@@ -137,6 +137,7 @@
     fileset,
     hg,
     localrepo,
+    merge,
     minifileset,
     node,
     pycompat,
@@ -331,6 +332,8 @@
     wrapfunction(hg, 'clone', wrapper.hgclone)
     wrapfunction(hg, 'postshare', wrapper.hgpostshare)
 
+    wrapfunction(merge, 'applyupdates', wrapper.mergemodapplyupdates)
+
     # Make bundle choose changegroup3 instead of changegroup2. This affects
     # "hg bundle" command. Note: it does not cover all bundle formats like
     # "packed1". Using "packed1" with lfs will likely cause trouble.
diff --git a/hgext/lfs/wrapper.py b/hgext/lfs/wrapper.py
--- a/hgext/lfs/wrapper.py
+++ b/hgext/lfs/wrapper.py
@@ -249,6 +249,34 @@
     if 'lfs' in destrepo.requirements:
         destrepo.vfs.append('hgrc', util.tonativeeol('\n[extensions]\nlfs=\n'))
 
+def mergemodapplyupdates(orig, repo, actions, wctx, mctx, overwrite, labels):
+    """Ensure that the required LFS blobs are present before applying updates,
+    fetching them as a group if needed.
+
+    This has the effect of ensuring all necessary LFS blobs are present before
+    making working directory changes during an update (including after clone and
+    share) or merge."""
+
+    pointers = []
+    localstore = repo.svfs.lfslocalblobstore
+
+    # Skipping 'a', 'am', 'f', 'r', 'dm', 'e', 'k', 'p' and 'pr', because they
+    # don't touch mctx.  'cd' is skipped, because changed/deleted never resolves
+    # to something from the remote side.
+    for a in 'g dc dg m'.split():
+        l = actions[a]
+        if len(l) > 0:
+            for f, msg, args in l:
+                p = pointerfromctx(mctx, f)
+                if p and not localstore.has(p.oid()):
+                    p.filename = f
+                    pointers.append(p)
+
+    if pointers:
+        repo.svfs.lfsremoteblobstore.readbatch(pointers, localstore)
+
+    return orig(repo, actions, wctx, mctx, overwrite, labels)
+
 def _canskipupload(repo):
     # if remotestore is a null store, upload is a no-op and can be skipped
     return isinstance(repo.svfs.lfsremoteblobstore, blobstore._nullremote)
diff --git a/tests/test-lfs-test-server.t b/tests/test-lfs-test-server.t
--- a/tests/test-lfs-test-server.t
+++ b/tests/test-lfs-test-server.t
@@ -66,10 +66,10 @@
   $ cd ../repo2
   $ hg update tip -v
   resolving manifests
-  getting a
   lfs: downloading 31cf46fbc4ecd458a0943c5b4881f1f5a6dd36c53d6167d5b69ac45149b38e5b (12 bytes)
   lfs: adding 31cf46fbc4ecd458a0943c5b4881f1f5a6dd36c53d6167d5b69ac45149b38e5b to the usercache
   lfs: processed: 31cf46fbc4ecd458a0943c5b4881f1f5a6dd36c53d6167d5b69ac45149b38e5b
+  getting a
   lfs: found 31cf46fbc4ecd458a0943c5b4881f1f5a6dd36c53d6167d5b69ac45149b38e5b in the local lfs store
   1 files updated, 0 files merged, 0 files removed, 0 files unresolved
 
@@ -99,17 +99,18 @@
   $ rm -rf `hg config lfs.usercache`
   $ hg --repo ../repo1 update tip -v
   resolving manifests
-  getting b
-  lfs: found 31cf46fbc4ecd458a0943c5b4881f1f5a6dd36c53d6167d5b69ac45149b38e5b in the local lfs store
-  getting c
+  lfs: need to transfer 2 objects (39 bytes)
+  lfs: downloading 37a65ab78d5ecda767e8622c248b5dbff1e68b1678ab0e730d5eb8601ec8ad19 (20 bytes)
+  lfs: adding 37a65ab78d5ecda767e8622c248b5dbff1e68b1678ab0e730d5eb8601ec8ad19 to the usercache
+  lfs: processed: 37a65ab78d5ecda767e8622c248b5dbff1e68b1678ab0e730d5eb8601ec8ad19
   lfs: downloading d11e1a642b60813aee592094109b406089b8dff4cb157157f753418ec7857998 (19 bytes)
   lfs: adding d11e1a642b60813aee592094109b406089b8dff4cb157157f753418ec7857998 to the usercache
   lfs: processed: d11e1a642b60813aee592094109b406089b8dff4cb157157f753418ec7857998
+  getting b
+  lfs: found 31cf46fbc4ecd458a0943c5b4881f1f5a6dd36c53d6167d5b69ac45149b38e5b in the local lfs store
+  getting c
   lfs: found d11e1a642b60813aee592094109b406089b8dff4cb157157f753418ec7857998 in the local lfs store
   getting d
-  lfs: downloading 37a65ab78d5ecda767e8622c248b5dbff1e68b1678ab0e730d5eb8601ec8ad19 (20 bytes)
-  lfs: adding 37a65ab78d5ecda767e8622c248b5dbff1e68b1678ab0e730d5eb8601ec8ad19 to the usercache
-  lfs: processed: 37a65ab78d5ecda767e8622c248b5dbff1e68b1678ab0e730d5eb8601ec8ad19
   lfs: found 37a65ab78d5ecda767e8622c248b5dbff1e68b1678ab0e730d5eb8601ec8ad19 in the local lfs store
   3 files updated, 0 files merged, 0 files removed, 0 files unresolved
 
@@ -123,11 +124,6 @@
 
   $ hg --repo ../repo1 update -C tip -v
   resolving manifests
-  getting a
-  lfs: found 31cf46fbc4ecd458a0943c5b4881f1f5a6dd36c53d6167d5b69ac45149b38e5b in the local lfs store
-  getting b
-  lfs: found 31cf46fbc4ecd458a0943c5b4881f1f5a6dd36c53d6167d5b69ac45149b38e5b in the local lfs store
-  getting c
   lfs: downloading d11e1a642b60813aee592094109b406089b8dff4cb157157f753418ec7857998 (19 bytes)
   abort: corrupt remote lfs object: d11e1a642b60813aee592094109b406089b8dff4cb157157f753418ec7857998
   [255]
diff --git a/tests/test-lfs.t b/tests/test-lfs.t
--- a/tests/test-lfs.t
+++ b/tests/test-lfs.t
@@ -760,7 +760,6 @@
   $ hg --config lfs.usercache=emptycache clone -v repo5 fromcorrupt2
   updating to branch default
   resolving manifests
-  getting l
   abort: corrupt remote lfs object: 22f66a3fc0b9bf3f012c814303995ec07099b3a9ce02a7af84b5970811074a3b
   [255]
 


More information about the Mercurial-devel mailing list