[PATCH RFC v6] scmutil: add a simple key-value file helper

Jun Wu quark at fb.com
Tue Dec 6 09:56:42 EST 2016


Excerpts from Kostia Balytskyi's message of 2016-12-06 06:32:02 -0800:
> # HG changeset patch
> # User Kostia Balytskyi <ikostia at fb.com>
> # Date 1481034715 28800
> #      Tue Dec 06 06:31:55 2016 -0800
> # Node ID 99bf22661062bb7d28ece67d94ebf421e3440731
> # Parent  243ecbd4f5c9f452275d4435866359cf84dc03ff
> scmutil: add a simple key-value file helper
> 
> The purpose of the added class is to serve purposes like save files of shelve
> or state files of shelve, rebase and histedit. Keys of these files can be
> alphanumeric and start with letters, while values must not contain newlines.
> Keys which start with an uppercase letter are required, while other keys
> are optional.
> 
> In light of Mercurial's reluctancy to use Python's json module, this tries
> to provide a reasonable alternative for a non-nested named data.
> Comparing to current approach of storing state in plain text files, where
> semantic meaning of lines of text is only determined by their oreder,
> simple key-value file allows for reordering lines and thus helps handle
> optional values.
> 
> Initial use-case I see for this is obs-shelve's shelve files. Later we
> can possibly migrate state files to this approach.
> 
> The test is in a new file beause I did not figure out where to put it
> within existing test suite. If you give me a better idea, I will gladly
> follow it.
> 
> diff --git a/mercurial/error.py b/mercurial/error.py
> --- a/mercurial/error.py
> +++ b/mercurial/error.py
> @@ -243,3 +243,19 @@ class UnsupportedBundleSpecification(Exc
>  
>  class CorruptedState(Exception):
>      """error raised when a command is not able to read its state from file"""
> +
> +class InvalidKeyValueFile(Exception):
> +    """error raised when the file can't be parsed as simple key-value file"""
> +

I think it's better to reuse CorruptedState for reading errors.

> +class InvalidKeyInFileException(Exception):
> +    """error raised when invalid key is attempted to be written
> +
> +    This is used in simple key-value file implementation"""
> +
> +class InvalidValueInFileException(Exception):
> +    """error raisesd when invalid value is attempted to be written
> +
> +    This is used in simple key-value file implementation"""
> +

The above two are better to be a ProgrammingError, per discussion in V1
with Augie. I'll send a ProgrammingError patch soon.

> +class MissingRequiredKeyInFileException(Exception):
> +    """error raised when simple key-value file misses a required key"""
> diff --git a/mercurial/scmutil.py b/mercurial/scmutil.py
> --- a/mercurial/scmutil.py
> +++ b/mercurial/scmutil.py
> @@ -1571,3 +1571,123 @@ class checkambigatclosing(closewrapbase)
>      def close(self):
>          self._origfh.close()
>          self._checkambig()
> +
> +class simplekeyvaluefile(object):
> +    """A simple file with key=value lines
> +
> +    Keys must be alphanumerics and start with a letter, values must not
> +    contain '\n' characters
> +
> +    >>> contents = {}
> +    >>> class fileobj(object):
> +    ...     def __init__(self, name):
> +    ...         self.name = name
> +    ...     def __enter__(self):
> +    ...         return self
> +    ...     def __exit__(self, *args, **kwargs):
> +    ...         pass
> +    ...     def write(self, text):
> +    ...         contents[self.name] = text
> +    ...     def read(self):
> +    ...         return contents[self.name]
> +    >>> class mockvfs(object):
> +    ...     def read(self, path):
> +    ...         return fileobj(path).read()
> +    ...     def readlines(self, path):
> +    ...         return fileobj(path).read().split('\\n')
> +    ...     def __call__(self, path, mode, atomictemp):
> +    ...         return fileobj(path)
> +    >>> vfs = mockvfs()
> +
> +    Basic testing of whether simple key-value file works:
> +    >>> d = {'key1': 'value1', 'Key2': 'value2'}
> +    >>> simplekeyvaluefile(vfs, 'kvfile').write(d)
> +    >>> print sorted(vfs.read('kvfile').split('\\n'))
> +    ['', 'Key2=value2', 'key1=value1']
> +
> +    Testing of whether invalid keys are detected:
> +    >>> d = {'0key1': 'value1', 'Key2': 'value2'}
> +    >>> simplekeyvaluefile(vfs, 'kvfile').write(d)
> +    Traceback (most recent call last):
> +        ...
> +    InvalidKeyInFileException: keys must start with a letter ...
> +    >>> d = {'key1@': 'value1', 'Key2': 'value2'}
> +    >>> simplekeyvaluefile(vfs, 'kvfile').write(d)
> +    Traceback (most recent call last):
> +        ...
> +    InvalidKeyInFileException: invalid key name in a simple key-value file
> +
> +    Testing of whether invalid values are detected:
> +    >>> d = {'key1': 'value1', 'Key2': 'value2\\n'}
> +    >>> simplekeyvaluefile(vfs, 'kvfile').write(d)
> +    Traceback (most recent call last):
> +        ...
> +    InvalidValueInFileException: invalid value in a simple key-value file
> +
> +    Test cases when necessary keys are present
> +    >>> d = {'key1': 'value1', 'Key2': 'value2'}
> +    >>> simplekeyvaluefile(vfs, 'allkeyshere').write(d)
> +    >>> class kvf(simplekeyvaluefile):
> +    ...     KEYS = [('key3', False), ('Key2', True)]
> +    >>> print sorted(kvf(vfs, 'allkeyshere').read().items())
> +    [('Key2', 'value'), ('key1', 'value')]
> +
> +    Test cases when necessary keys are absent
> +    >>> d = {'key1': 'value1', 'Key3': 'value2'}
> +    >>> simplekeyvaluefile(vfs, 'missingkeys').write(d)
> +    >>> class kvf(simplekeyvaluefile):
> +    ...     KEYS = [('key3', False), ('Key2', True)]
> +    >>> print sorted(kvf(vfs, 'missingkeys').read().items())
> +    Traceback (most recent call last):
> +        ...
> +    MissingRequiredKeyInFileException: missing a required key: 'Key2'
> +
> +    Test cases when file is not really a simple key-value file
> +    >>> contents['badfile'] = 'ababagalamaga\\n'
> +    >>> simplekeyvaluefile(vfs, 'badfile').read()
> +    Traceback (most recent call last):
> +        ...
> +    InvalidKeyValueFile: dictionary ... element #0 has length 1; 2 is required
> +    """
> +
> +    # if KEYS is non-empty, read values are validated against it:
> +    # each key is a tuple (keyname, required)
> +    KEYS = []
> +
> +    def __init__(self, vfs, path):
> +        self.vfs = vfs
> +        self.path = path
> +
> +    def validate(self, d):
> +        for key, req in self.KEYS:
> +            if req and key not in d:
> +                e = "missing a required key: '%s'" % key
> +                raise error.MissingRequiredKeyInFileException(e)
> +
> +    def read(self):
> +        lines = self.vfs.readlines(self.path)
> +        try:
> +            d = dict(line[:-1].split('=', 1) for line in lines if line)
> +        except ValueError as e:
> +            raise error.InvalidKeyValueFile(str(e))
> +        self.validate(d)
> +        return d
> +
> +    def write(self, data):
> +        """Write key=>value mapping to a file
> +        data is a dict. Keys must be alphanumerical and start with a letter.
> +        Values must not contain newline characters."""
> +        lines = []
> +        for k, v in data.items():
> +            if not k[0].isalpha():
> +                e = "keys must start with a letter in a key-value file"
> +                raise error.InvalidKeyInFileException(e)
> +            if not k.isalnum():
> +                e = "invalid key name in a simple key-value file"
> +                raise error.InvalidKeyInFileException(e)
> +            if '\n' in v:
> +                e = "invalid value in a simple key-value file"
> +                raise error.InvalidValueInFileException(e)
> +            lines.append("%s=%s\n" % (k, v))
> +        with self.vfs(self.path, mode='wb', atomictemp=True) as fp:
> +            fp.write(''.join(lines))
> diff --git a/tests/test-doctest.py b/tests/test-doctest.py
> --- a/tests/test-doctest.py
> +++ b/tests/test-doctest.py
> @@ -28,6 +28,7 @@ testmod('mercurial.patch')
>  testmod('mercurial.pathutil')
>  testmod('mercurial.parser')
>  testmod('mercurial.revset')
> +testmod('mercurial.scmutil', optionflags=doctest.ELLIPSIS)
>  testmod('mercurial.store')
>  testmod('mercurial.subrepo')
>  testmod('mercurial.templatefilters')


More information about the Mercurial-devel mailing list