D6772: remotefilelog: reduce probability of race-condition in remotefilelog tests

[lothiraldan (Boris Feld)]

lothiraldan created this revision.
Herald added subscribers: mercurial-devel, mjpieters.
Herald added a reviewer: hg-reviewers.

REVISION SUMMARY
  ca1014ad3de4 <https://phab.mercurial-scm.org/rHGca1014ad3de4e95136952223ab7d217e54e05874> introduced a new parameter `ensurestart` to speed up
  remotefilelog background processes start. Unfortunately it seems to have
  increased the possibility of race-conditions in remotefilelog tests testing
  those background processes.
  
  With `ensurestart=False`, it seems that it's more probable to enter in a race
  condition with `debugwaitonprefetch` and `debugwaitonrepack` in remotefilelog
  background tests. Our CI seems to have a high probability of triggering this
  race condition so make it configurable to ensure tests are stable.

REPOSITORY
  rHG Mercurial

REVISION DETAIL
  https://phab.mercurial-scm.org/D6772

AFFECTED FILES
  hgext/remotefilelog/__init__.py
  hgext/remotefilelog/repack.py
  hgext/remotefilelog/shallowrepo.py
  tests/test-remotefilelog-bgprefetch.t
  tests/test-remotefilelog-repack-fast.t
  tests/test-remotefilelog-repack.t

CHANGE DETAILS

diff --git a/tests/test-remotefilelog-repack.t b/tests/test-remotefilelog-repack.t
--- a/tests/test-remotefilelog-repack.t
+++ b/tests/test-remotefilelog-repack.t
@@ -1,6 +1,12 @@
 #require no-windows
 
   $ . "$TESTDIR/remotefilelog-library.sh"
+# devel.remotefilelog.ensurestart: reduce race condition with
+# waiton{repack/prefetch}
+  $ cat >> $HGRCPATH <<EOF
+  > [devel]
+  > remotefilelog.ensurestart=True
+  > EOF
 
   $ hg init master
   $ cd master
diff --git a/tests/test-remotefilelog-repack-fast.t b/tests/test-remotefilelog-repack-fast.t
--- a/tests/test-remotefilelog-repack-fast.t
+++ b/tests/test-remotefilelog-repack-fast.t
@@ -1,10 +1,13 @@
 #require no-windows
 
   $ . "$TESTDIR/remotefilelog-library.sh"
-
+# devel.remotefilelog.ensurestart: reduce race condition with
+# waiton{repack/prefetch}
   $ cat >> $HGRCPATH <<EOF
   > [remotefilelog]
   > fastdatapack=True
+  > [devel]
+  > remotefilelog.ensurestart=True
   > EOF
 
   $ hg init master
diff --git a/tests/test-remotefilelog-bgprefetch.t b/tests/test-remotefilelog-bgprefetch.t
--- a/tests/test-remotefilelog-bgprefetch.t
+++ b/tests/test-remotefilelog-bgprefetch.t
@@ -1,6 +1,12 @@
 #require no-windows
 
   $ . "$TESTDIR/remotefilelog-library.sh"
+# devel.remotefilelog.ensurestart: reduce race condition with
+# waiton{repack/prefetch}
+  $ cat >> $HGRCPATH <<EOF
+  > [devel]
+  > remotefilelog.ensurestart=True
+  > EOF
 
   $ hg init master
   $ cd master
diff --git a/hgext/remotefilelog/shallowrepo.py b/hgext/remotefilelog/shallowrepo.py
--- a/hgext/remotefilelog/shallowrepo.py
+++ b/hgext/remotefilelog/shallowrepo.py
@@ -183,7 +183,7 @@
                                                             origctx=origctx)
 
         def backgroundprefetch(self, revs, base=None, repack=False, pats=None,
-                               opts=None):
+                               opts=None, ensurestart=False):
             """Runs prefetch in background with optional repack
             """
             cmd = [procutil.hgexecutable(), '-R', repo.origroot, 'prefetch']
@@ -193,7 +193,8 @@
                 cmd += ['-r', revs]
             # We know this command will find a binary, so don't block
             # on it starting.
-            procutil.runbgcommand(cmd, encoding.environ, ensurestart=False)
+            procutil.runbgcommand(cmd, encoding.environ,
+                                  ensurestart=ensurestart)
 
         def prefetch(self, revs, base=None, pats=None, opts=None):
             """Prefetches all the necessary file revisions for the given revs
diff --git a/hgext/remotefilelog/repack.py b/hgext/remotefilelog/repack.py
--- a/hgext/remotefilelog/repack.py
+++ b/hgext/remotefilelog/repack.py
@@ -34,7 +34,8 @@
 class RepackAlreadyRunning(error.Abort):
     pass
 
-def backgroundrepack(repo, incremental=True, packsonly=False):
+def backgroundrepack(repo, incremental=True, packsonly=False,
+                     ensurestart=False):
     cmd = [procutil.hgexecutable(), '-R', repo.origroot, 'repack']
     msg = _("(running background repack)\n")
     if incremental:
@@ -44,7 +45,7 @@
         cmd.append('--packsonly')
     repo.ui.warn(msg)
     # We know this command will find a binary, so don't block on it starting.
-    procutil.runbgcommand(cmd, encoding.environ, ensurestart=False)
+    procutil.runbgcommand(cmd, encoding.environ, ensurestart=ensurestart)
 
 def fullrepack(repo, options=None):
     """If ``packsonly`` is True, stores creating only loose objects are skipped.
diff --git a/hgext/remotefilelog/__init__.py b/hgext/remotefilelog/__init__.py
--- a/hgext/remotefilelog/__init__.py
+++ b/hgext/remotefilelog/__init__.py
@@ -224,6 +224,8 @@
 configitem('packs', 'maxpacksize', default=0)
 configitem('packs', 'maxchainlen', default=1000)
 
+configitem('devel', 'remotefilelog.ensurestart', default=False)
+
 #  default TTL limit is 30 days
 _defaultlimit = 60 * 60 * 24 * 30
 configitem('remotefilelog', 'nodettl', default=_defaultlimit)
@@ -949,19 +951,23 @@
         prefetchrevset = ui.config('remotefilelog', 'pullprefetch')
         bgrepack = repo.ui.configbool('remotefilelog', 'backgroundrepack')
         bgprefetch = repo.ui.configbool('remotefilelog', 'backgroundprefetch')
+        ensurestart = repo.ui.configbool('devel', 'remotefilelog.ensurestart')
 
         if prefetchrevset:
             ui.status(_("prefetching file contents\n"))
             revs = scmutil.revrange(repo, [prefetchrevset])
             base = repo['.'].rev()
             if bgprefetch:
-                repo.backgroundprefetch(prefetchrevset, repack=bgrepack)
+                repo.backgroundprefetch(prefetchrevset, repack=bgrepack,
+                                        ensurestart=ensurestart)
             else:
                 repo.prefetch(revs, base=base)
                 if bgrepack:
-                    repackmod.backgroundrepack(repo, incremental=True)
+                    repackmod.backgroundrepack(repo, incremental=True,
+                                               ensurestart=ensurestart)
         elif bgrepack:
-            repackmod.backgroundrepack(repo, incremental=True)
+            repackmod.backgroundrepack(repo, incremental=True,
+                                       ensurestart=ensurestart)
 
     return result
 
@@ -1085,9 +1091,12 @@
     revs = scmutil.revrange(repo, opts.get('rev'))
     repo.prefetch(revs, opts.get('base'), pats, opts)
 
+    ensurestart = repo.ui.configbool('devel', 'remotefilelog.ensurestart')
+
     # Run repack in background
     if opts.get('repack'):
-        repackmod.backgroundrepack(repo, incremental=True)
+        repackmod.backgroundrepack(repo, incremental=True,
+                                   ensurestart=ensurestart)
 
 @command('repack', [
      ('', 'background', None, _('run in a background process'), None),
@@ -1096,8 +1105,10 @@
     ], _('hg repack [OPTIONS]'))
 def repack_(ui, repo, *pats, **opts):
     if opts.get(r'background'):
+        ensurestart = repo.ui.configbool('devel', 'remotefilelog.ensurestart')
         repackmod.backgroundrepack(repo, incremental=opts.get(r'incremental'),
-                                   packsonly=opts.get(r'packsonly', False))
+                                   packsonly=opts.get(r'packsonly', False),
+                                   ensurestart=ensurestart)
         return
 
     options = {'packsonly': opts.get(r'packsonly')}



To: lothiraldan, #hg-reviewers
Cc: mjpieters, mercurial-devel