[1,of,2,RFC] RFC: implement immutable config objects

[David Soria Parra]

On Mon, Mar 27, 2017 at 11:38:06AM -0700, Jun Wu wrote:
> # HG changeset patch
> # User Jun Wu <quark at fb.com>
> # Date 1490635856 25200
> #      Mon Mar 27 10:30:56 2017 -0700
> # Node ID 4eb7c76340791f379a34f9df4ec42e0c8b9b2a2f
> # Parent  4a8d065bbad80d3b3401010375bc80165404aa87
> RFC: implement immutable config objects

I like the overall approach and that it's parallel to config.config() and
ui.config. We might in later patches combine config.config into this.

> +class abstractimmutableconfig(object):
> +    """minimal interface defined for a read-only config accessor
> +
> +    The immutable config object should get its state set and frozen during
> +    __init__ and should not have any setconfig-like method.
> +
> +    The immutable config layer knows about sections, and should probably uses
> +    ordered dict for each section. To simplify things, this layer does not care
> +    about "source", "unset" or any filesystem IO. They're up to the upper layer
> +    to deal with. For example, the upper layer could store "None" as "unset"s,
> +    and store (value, source) as a tuple together.
> +    """
> +
> +    def __init__(self, title):
> +        """title: useful to identify the config"""
> +        self.title = title
> +
> +    def subconfigs(self, section=None):
> +        """return a list-ish of child config objects filtered by section"""
> +        raise NotImplementedError
> +
> +    def get(self, section, item):
> +        """return value or None"""
> +        return self.getsection(section).get(item)
> +
> +    def getsection(self, section):
> +        """return a dict-ish"""
> +        raise NotImplementedError
> +
> +    def sections(self):
> +        """return an iter-able"""
> +        raise NotImplementedError
> +
> +class atomicconfig(abstractimmutableconfig):
> +    """immutable config that converted from a list-ish"""
> +
> +    def __init__(self, title, entries=None):
> +        """
> +        title:   a title used to identify the atomicconfig
> +        entries: a list-ish of (section, item, value)
> +        """
> +        super(atomicconfig, self).__init__(title)
> +        self._sections = util.sortdict()
Can we use collections.defaultdict(util.sortdict) here?

> +        for entry in entries:
> +            section, item, value = entry
> +            if section not in self._sections:
> +                self._sections[section] = util.sortdict()
> +            if item is not None:
> +                self._sections[section][item] = value
> +
> +    def subconfigs(self, section=None):
> +        return ()
> +
> +    def getsection(self, section):
> +        return self._sections.get(section, {})
> +
> +    def sections(self):
> +        return self._sections.keys()
> +
> +class mergedconfig(abstractimmutableconfig):
> +    """immutable config that is a merge of a list of immutable configs"""
> +
> +    def __init__(self, title, subconfigs):
> +        super(mergedconfig, self).__init__(title)
> +        self._subconfigs = tuple(subconfigs)  # make it immutable
> +        self._cachedsections = {}
> +
> +    def subconfigs(self, section=None):
I am not sure about this API. We already have getsection() from
abstractimmutableconfig. I think this method does two things depending
on the arguments.
> +        if section is None:
> +            return self._subconfigs
> +        else:
> +            return self._sectionconfigs.get(section, ())
> +
> +    def sections(self):
> +        return self._sectionconfigs.keys()
> +
> +    @util.propertycache
> +    def _sectionconfigs(self):
> +        """{section: [subconfig]}"""
> +        sectionconfigs = {}
> +        for subconfig in self._subconfigs:
> +            for section in subconfig.sections():
> +                sectionconfigs.setdefault(section, []).append(subconfig)
> +        return sectionconfigs
> +
> +    def getsection(self, section):
> +        items = self._cachedsections.get(section, None)
> +        if items is None:
> +            subconfigs = self._sectionconfigs.get(section, [])
> +            if len(subconfigs) == 1:
> +                # no need to merge configs
> +                items = subconfigs[0].getsection(section)
> +            else:
> +                # merge configs
> +                items = util.sortdict()
> +                for subconfig in subconfigs:
> +                    subconfigitems = subconfig.getsection(section).items()
Could we do an `items.update(subconfig.getsection(section))`?
> +                    for item, value in subconfigitems:
> +                        items[item] = value
> +            self._cachedsections[section] = items
> +        return items
> +
> +    def append(self, subconfig):
> +        """return a new mergedconfig with the new subconfig appended"""
> +        return mergedconfig(self.title, list(self._subconfigs) + [subconfig])
> +
> +    def prepend(self, subconfig):
> +        """return a new mergedconfig with the new subconfig prepended"""
> +        return mergedconfig(self.title, [subconfig] + list(self._subconfigs))
> +
> +    def filter(self, func):
> +        """return a new mergedconfig with only selected subconfigs
> +        func: subconfig -> bool
> +        """
> +        return mergedconfig(self.title, filter(func, self._subconfigs))
> +
> +class fileconfig(atomicconfig):
> +    """immutable config constructed from config files"""
> +
> +    def __init__(self, title, fp, sections=None, remap=None):
> +        # Currently, just use the legacy, non-immutable "config" object to do
> +        # the parsing, remap stuff. In the future we may want to detach from it
> +        cfg = config()
> +        cfg.read(title, fp, sections=sections, remap=remap)
> +
> +        def cfgwalker():
> +            # visible config, with source
> +            for section in cfg:
> +                emptysection = True
> +                for item, value in cfg.items(section):
> +                    emptysection = False
> +                    source = cfg.source(section, item)
> +                    yield (section, item, (value, source))
> +                if emptysection:
> +                    # create the section
> +                    yield (section, None, (None, None))
> +            # "%unset" configs
> +            for (section, item) in cfg._unset:
> +                # TODO "%unset" could have line numbers
> +                if cfg.get(section, item) is None:
> +                    yield (section, item, (None, '%s:(unset)' % title))
> +
> +        super(fileconfig, self).__init__(title, cfgwalker())
> +
> +class filteredconfig(abstractimmutableconfig):
> +    """immutable config that changes other configs"""
> +
> +    def __init__(self, title, subconfig, filters=None):
> +        """
> +        filters: a dict-ish, {section: filterfunc or sortdict-ish}
> +                 a filterfunc takes sortdict-ish and returns sortdict-ish
> +                 a sortdict-ish will replace the section directly
> +        """
> +        super(filteredconfig, self).__init__(title)
> +        self._filters = filters or {}
> +        self._subconfig = subconfig
> +        self._cachedsections = {}
> +
> +    def subconfigs(self, section=None):
I think this is actually not needed except for dumpconfig, which might just be a
helper that knows about the internals.

> +        return (self._subconfig,)
> +
> +    def sections(self):
> +        return self._subconfig.sections()
> +
> +    def getsection(self, section):
> +        if section not in self._filters:
> +            return self._subconfig.getsection(section)
> +        items = self._cachedsections.get(section, None)
> +        if items is None:
> +            filter = self._filters[section]
> +            if util.safehasattr(filter, '__call__'):
this should be `callable`

> +                items = filter(self._subconfig.getsection(section))
> +            else:
> +                items = filter
> +            self._cachedsections[section] = items
> +        return items