D7838: nodemap: use an intermediate "docket" file to carry small metadata

[marmoute (Pierre-Yves David)]

marmoute updated this revision to Diff 19754.

REPOSITORY
  rHG Mercurial

CHANGES SINCE LAST UPDATE
  https://phab.mercurial-scm.org/D7838?vs=19175&id=19754

CHANGES SINCE LAST ACTION
  https://phab.mercurial-scm.org/D7838/new/

REVISION DETAIL
  https://phab.mercurial-scm.org/D7838

AFFECTED FILES
  mercurial/revlogutils/nodemap.py
  tests/test-persistent-nodemap.t

CHANGE DETAILS

diff --git a/tests/test-persistent-nodemap.t b/tests/test-persistent-nodemap.t
--- a/tests/test-persistent-nodemap.t
+++ b/tests/test-persistent-nodemap.t
@@ -10,6 +10,8 @@
   > exp-persistent-nodemap=yes
   > EOF
   $ hg debugbuilddag .+5000
+  $ f --size .hg/store/00changelog.n
+  .hg/store/00changelog.n: size=18
   $ hg debugnodemap --dump-new | f --sha256 --size
   size=245760, sha256=bc400bf49f11e83bbd25630439feee6628a80a8602d2e38972eac44cc3efe10c
   $ hg debugnodemap --dump-disk | f --sha256 --bytes=256 --hexdump --size
diff --git a/mercurial/revlogutils/nodemap.py b/mercurial/revlogutils/nodemap.py
--- a/mercurial/revlogutils/nodemap.py
+++ b/mercurial/revlogutils/nodemap.py
@@ -8,6 +8,7 @@
 
 from __future__ import absolute_import
 
+import os
 import struct
 
 from .. import (
@@ -25,7 +26,20 @@
     """read the nodemap for a revlog from disk"""
     if revlog.nodemap_file is None:
         return None
-    return revlog.opener.tryread(revlog.nodemap_file)
+    pdata = revlog.opener.tryread(revlog.nodemap_file)
+    if not pdata:
+        return None
+    offset = 0
+    (version,) = S_VERSION.unpack(pdata[offset : offset + S_VERSION.size])
+    if version != ONDISK_VERSION:
+        return None
+    offset += S_VERSION.size
+    (uuid_size,) = S_HEADER.unpack(pdata[offset : offset + S_HEADER.size])
+    offset += S_HEADER.size
+    uid = pdata[offset : offset + uuid_size]
+
+    filename = _rawdata_filepath(revlog, uid)
+    return revlog.opener.tryread(filename)
 
 
 def setup_persistent_nodemap(tr, revlog):
@@ -54,13 +68,71 @@
         msg = "calling persist nodemap on a revlog without the feature enableb"
         raise error.ProgrammingError(msg)
     data = persistent_data(revlog.index)
+    uid = _make_uid()
+    datafile = _rawdata_filepath(revlog, uid)
+    # EXP-TODO: if this is a cache, this should use a cache vfs, not a
+    # store vfs
+    with revlog.opener(datafile, 'w') as fd:
+        fd.write(data)
     # EXP-TODO: if this is a cache, this should use a cache vfs, not a
     # store vfs
-    with revlog.opener(revlog.nodemap_file, 'w') as f:
-        f.write(data)
+    with revlog.opener(revlog.nodemap_file, 'w', atomictemp=True) as fp:
+        fp.write(_serialize_docket(uid))
     # EXP-TODO: if the transaction abort, we should remove the new data and
-    # reinstall the old one. (This will be simpler when the file format get a
-    # bit more advanced)
+    # reinstall the old one.
+
+
+### Nodemap docket file
+#
+# The nodemap data are stored on disk using 2 files:
+#
+# * a raw data files containing a persistent nodemap
+#   (see `Nodemap Trie` section)
+#
+# * a small "docket" file containing medatadata
+#
+# While the nodemap data can be multiple tens of megabytes, the "docket" is
+# small, it is easy to update it automatically or to duplicated its content
+# during a transaction.
+#
+# Multiple raw data can exist at the same time (The currently valid one and a
+# new one beind used by an in progress transaction). To accomodate this, the
+# filename hosting the raw data has a variable parts. The exact filename is
+# specified inside the "docket" file.
+#
+# The docket file contains information to find, qualify and validate the raw
+# data. Its content is currently very light, but it will expand as the on disk
+# nodemap gains the necessary features to be used in production.
+
+# version 0 is experimental, no BC garantee, do no use outside of tests.
+ONDISK_VERSION = 0
+
+S_VERSION = struct.Struct(">B")
+S_HEADER = struct.Struct(">B")
+
+ID_SIZE = 8
+
+
+def _make_uid():
+    """return a new unique identifier.
+
+    The identifier is random and composed of ascii characters."""
+    return nodemod.hex(os.urandom(ID_SIZE))
+
+
+def _serialize_docket(uid):
+    """return serialized bytes for a docket using the passed uid"""
+    data = []
+    data.append(S_VERSION.pack(ONDISK_VERSION))
+    data.append(S_HEADER.pack(len(uid)))
+    data.append(uid)
+    return b''.join(data)
+
+
+def _rawdata_filepath(revlog, uid):
+    """The (vfs relative) nodemap's rawdata file for a given uid"""
+    prefix = revlog.nodemap_file[:-2]
+    return b"%s-%s.nd" % (prefix, uid)
 
 
 ### Nodemap Trie



To: marmoute, #hg-reviewers
Cc: mercurial-devel