[PATCH 5 of 5 import-refactor] setup: refactor handling of modules with C/Python implementations

[Gregory Szorc]

> On Nov 21, 2015, at 22:14, Gregory Szorc <gregory.szorc at gmail.com> wrote:
> 
> # HG changeset patch
> # User Gregory Szorc <gregory.szorc at gmail.com>
> # Date 1448172487 28800
> #      Sat Nov 21 22:08:07 2015 -0800
> # Node ID 7c0f0e0fa62799ce071b95753a8a45cfac8b4f05
> # Parent  1101c57a493083f9fd3dc2eb7832f862c587cd5a
> setup: refactor handling of modules with C/Python implementations

This one should be dropped: we need to reject modules loaded from .py or .pyc files from the legacy location otherwise old files from old installs could get loaded. I.e. we need to require modules loaded from mercurial.* come from C extensions and fall back to pure Python if they aren't.

> 
> Previously, .py files under mercurial/pure/ were copied to mercurial/*
> during installation if we were performing a pure Python installation.
> 
> Now that the new import hooks and module load policy is in place, this
> hackery from the past is no longer necessary. This patch refactors how
> setup.py deals with modules that have both Python and C implementations.
> 
> Modules under mercurial.* are now always present in the installation.
> The default is still to require the C extensions (unless running with
> PyPy or requesting a pure Python installation). But the modules are now
> there in case people want to run PyPy and CPython from the same
> installation. There is a slight risk the inclusion of these files in
> the distribution could lead to people blindly following poor
> instructions to not run the C extensions with CPython. So this change
> is somewhat contentious.
> 
> Modules under mercurial/pure/* are no longer copied to mercurial/*
> during install. Instead, we install them in their original location
> and leave it up to the module import hook and our module loading
> policy to dictate how and where modules are loaded.
> 
> Finally, `hg` is executed in pure Python mode as part of the
> installation. This should give a higher chance of success when running
> the installation.
> 
> diff --git a/setup.py b/setup.py
> --- a/setup.py
> +++ b/setup.py
> @@ -172,21 +172,19 @@ def runhg(cmd, env):
>     if err:
>         printf("stderr from '%s':" % (' '.join(cmd)), file=sys.stderr)
>         printf(b('\n').join([b('  ') + e for e in err]), file=sys.stderr)
>         return ''
>     return out
> 
> version = ''
> 
> -# Execute hg out of this directory with a custom environment which
> -# includes the pure Python modules in mercurial/pure. We also take
> -# care to not use any hgrc files and do no localization.
> -pypath = ['mercurial', os.path.join('mercurial', 'pure')]
> -env = {'PYTHONPATH': os.pathsep.join(pypath),
> +# Execute hg out of this directory with a custom environment which takes care
> +# to not use any hgrc files and do no localization.
> +env = {'HGMODULEPOLICY': 'py',
>        'HGRCPATH': '',
>        'LANGUAGE': 'C'}
> if 'LD_LIBRARY_PATH' in os.environ:
>     env['LD_LIBRARY_PATH'] = os.environ['LD_LIBRARY_PATH']
> if 'SystemRoot' in os.environ:
>     # Copy SystemRoot into the custom environment for Python 2.6
>     # under Windows. Otherwise, the subprocess will fail with
>     # error 0xc0150004. See: http://bugs.python.org/issue3440
> @@ -310,37 +308,23 @@ class hgbuildpy(build_py):
>     if convert2to3:
>         fixer_names = sorted(set(getfixers("lib2to3.fixes") +
>                                  getfixers("hgfixes")))
> 
>     def finalize_options(self):
>         build_py.finalize_options(self)
> 
>         if self.distribution.pure:
> -            if self.py_modules is None:
> -                self.py_modules = []
> -            for ext in self.distribution.ext_modules:
> -                if ext.name.startswith("mercurial."):
> -                    self.py_modules.append("mercurial.pure.%s" % ext.name[10:])
>             self.distribution.ext_modules = []
>         else:
>             h = os.path.join(get_python_inc(), 'Python.h')
>             if not os.path.exists(h):
>                 raise SystemExit('Python headers are required to build '
>                                  'Mercurial but weren\'t found in %s' % h)
> 
> -    def find_modules(self):
> -        modules = build_py.find_modules(self)
> -        for module in modules:
> -            if module[0] == "mercurial.pure":
> -                if module[1] != "__init__":
> -                    yield ("mercurial", module[1], module[2])
> -            else:
> -                yield module
> -
> class buildhgextindex(Command):
>     description = 'generate prebuilt index of hgext (for frozen package)'
>     user_options = []
>     _indexfilename = 'hgext/__index__.py'
> 
>     def initialize_options(self):
>         pass
> 
> @@ -481,21 +465,20 @@ cmdclass = {'build': hgbuild,
>             'build_py': hgbuildpy,
>             'build_hgextindex': buildhgextindex,
>             'install_lib': hginstalllib,
>             'install_scripts': hginstallscripts,
>             'build_hgexe': buildhgexe,
>             }
> 
> packages = ['mercurial', 'mercurial.hgweb', 'mercurial.httpclient',
> +            'mercurial.pure',
>             'hgext', 'hgext.convert', 'hgext.highlight', 'hgext.zeroconf',
>             'hgext.largefiles']
> 
> -pymodules = []
> -
> common_depends = ['mercurial/util.h']
> 
> osutil_ldflags = []
> 
> if sys.platform == 'darwin':
>     osutil_ldflags += ['-framework', 'ApplicationServices']
> 
> extmodules = [
> @@ -637,17 +620,16 @@ setup(name='mercurial',
>           'Operating System :: OS Independent',
>           'Operating System :: POSIX',
>           'Programming Language :: C',
>           'Programming Language :: Python',
>           'Topic :: Software Development :: Version Control',
>       ],
>       scripts=scripts,
>       packages=packages,
> -      py_modules=pymodules,
>       ext_modules=extmodules,
>       data_files=datafiles,
>       package_data=packagedata,
>       cmdclass=cmdclass,
>       distclass=hgdist,
>       options={'py2exe': {'packages': ['hgext', 'email']},
>                'bdist_mpkg': {'zipdist': False,
>                               'license': 'COPYING',