[PATCH] py3: handle os.environ.get() case in module loader

[Siddharth Agarwal]

On 8/4/16 12:11, Pulkit Goyal wrote:
> # HG changeset patch
> # User Pulkit Goyal <7895pulkit at gmail.com>
> # Date 1470337846 -19800
> #      Fri Aug 05 00:40:46 2016 +0530
> # Branch stable
> # Node ID 6bc900348ed8d695dddc9451f56e3adf38149127
> # Parent  44c45c5f2481e94b8eeb24b31fea4b49d3453835
> py3: handle os.environ.get() case in module loader
>
> The os.environ.get() doesnot accepts bytes on Python 3. Instead of adding u''
> in the source code, its preferred to add a case in the custom module loader
> which handles this.

I agree with Greg -- this makes things more complicated than necessary. 
We should just have a helper (e.g. util.environ) that gets assigned to 
os.environ on py2 and os.environb on py3. (And with possibly different 
behavior on Windows, similar to filenames.)

- Siddharth

>
> Firstly the code rewrite the token to include u'' to undo the effect by the
> transformer on the first argument. Then it searches for more arguments if
> present and does the same for them.
>
> diff -r 44c45c5f2481 -r 6bc900348ed8 mercurial/__init__.py
> --- a/mercurial/__init__.py	Thu Aug 04 00:32:19 2016 +0530
> +++ b/mercurial/__init__.py	Fri Aug 05 00:40:46 2016 +0530
> @@ -272,6 +272,47 @@
>                       except IndexError:
>                           pass
>   
> +                # os.environ.get() doesnot accepts byte strings on Python 3.
> +                # Rewrite the token to include the unicode literal prefix so
> +                # the string transformer above doesn't add the byte prefix.
> +                if (fn in ('get') and
> +                    prevtoken.type == token.OP and prevtoken.string == '.'
> +                    and tokens[i - 2].string == 'environ' and
> +                    tokens[i - 4].string == 'os'):
> +                    # (tokens[i-4:i-2], os.environ)
> +                    # (OP, '.')
> +                    # (NAME, 'get')
> +                    # (OP, '(')
> +                    # (STRING, 'HGENCODING')
> +                    # (OP, ')')
> +                    try:
> +                        st = tokens[i + 2]
> +                        if (st.type == token.STRING and
> +                            st.string[0] in ("'", '"')):
> +                            rt = tokenize.TokenInfo(st.type, 'u%s' % st.string,
> +                                                    st.start, st.end, st.line)
> +                            tokens[i + 2] = rt
> +
> +                        # This while loop deals with the further arguments in
> +                        # in os.environ.get(). It checks whether we have ')'
> +                        # as the next token or we have a ',' which indicates
> +                        # that we have more arguments. This loop terminates
> +                        # when we get the ')' token.
> +                        j = 3
> +                        while (tokens[i + j].string != ')' and
> +                            tokens[i + j].string == ','):
> +                            st = tokens[i + j + 1]
> +                            if (st.type == token.STRING and
> +                                st.string[0] in ("'", '"')):
> +                                rt = tokenize.TokenInfo(st.type, 'u%s' %
> +                                                        st.string, st.start,
> +                                                        st.end, st.line)
> +                                tokens[i + j + 1] = rt
> +                                j += 2
> +
> +                    except IndexError:
> +                        pass
> +
>               # Emit unmodified token.
>               yield t
>   
> @@ -279,7 +320,7 @@
>       # ``replacetoken`` or any mechanism that changes semantics of module
>       # loading is changed. Otherwise cached bytecode may get loaded without
>       # the new transformation mechanisms applied.
> -    BYTECODEHEADER = b'HG\x00\x01'
> +    BYTECODEHEADER = b'HG\x00\x02'
>   
>       class hgloader(importlib.machinery.SourceFileLoader):
>           """Custom module loader that transforms source code.
> diff -r 44c45c5f2481 -r 6bc900348ed8 tests/test-check-py3-compat.t
> --- a/tests/test-check-py3-compat.t	Thu Aug 04 00:32:19 2016 +0530
> +++ b/tests/test-check-py3-compat.t	Fri Aug 05 00:40:46 2016 +0530
> @@ -15,90 +15,90 @@
>   #if py3exe
>     $ hg files 'set:(**.py)' | sed 's|\\|/|g' | xargs $PYTHON3 contrib/check-py3-compat.py
>     doc/hgmanpage.py: invalid syntax: invalid syntax (<unknown>, line *) (glob)
> -  hgext/acl.py: error importing: <TypeError> str expected, not bytes (error at encoding.py:*) (glob)
> -  hgext/automv.py: error importing: <TypeError> str expected, not bytes (error at encoding.py:*) (glob)
> -  hgext/blackbox.py: error importing: <TypeError> str expected, not bytes (error at encoding.py:*) (glob)
> -  hgext/bugzilla.py: error importing: <TypeError> str expected, not bytes (error at encoding.py:*) (glob)
> -  hgext/censor.py: error importing: <TypeError> str expected, not bytes (error at encoding.py:*) (glob)
> -  hgext/chgserver.py: error importing: <TypeError> str expected, not bytes (error at encoding.py:*) (glob)
> -  hgext/children.py: error importing: <TypeError> str expected, not bytes (error at encoding.py:*) (glob)
> -  hgext/churn.py: error importing: <TypeError> str expected, not bytes (error at encoding.py:*) (glob)
> -  hgext/clonebundles.py: error importing: <TypeError> str expected, not bytes (error at encoding.py:*) (glob)
> +  hgext/acl.py: error importing: <TypeError> str expected, not bytes (error at i18n.py:*) (glob)
> +  hgext/automv.py: error importing: <TypeError> str expected, not bytes (error at i18n.py:*) (glob)
> +  hgext/blackbox.py: error importing: <TypeError> str expected, not bytes (error at i18n.py:*) (glob)
> +  hgext/bugzilla.py: error importing: <TypeError> str expected, not bytes (error at i18n.py:*) (glob)
> +  hgext/censor.py: error importing: <TypeError> str expected, not bytes (error at i18n.py:*) (glob)
> +  hgext/chgserver.py: error importing: <TypeError> str expected, not bytes (error at i18n.py:*) (glob)
> +  hgext/children.py: error importing: <TypeError> str expected, not bytes (error at i18n.py:*) (glob)
> +  hgext/churn.py: error importing: <TypeError> str expected, not bytes (error at i18n.py:*) (glob)
> +  hgext/clonebundles.py: error importing: <TypeError> str expected, not bytes (error at i18n.py:*) (glob)
>     hgext/color.py: invalid syntax: invalid syntax (<unknown>, line *) (glob)
> -  hgext/convert/bzr.py: error importing: <TypeError> str expected, not bytes (error at encoding.py:*) (glob)
> -  hgext/convert/common.py: error importing: <TypeError> str expected, not bytes (error at encoding.py:*) (glob)
> -  hgext/convert/convcmd.py: error importing: <TypeError> str expected, not bytes (error at encoding.py:*) (glob)
> -  hgext/convert/cvs.py: error importing: <TypeError> str expected, not bytes (error at encoding.py:*) (glob)
> -  hgext/convert/cvsps.py: error importing: <TypeError> str expected, not bytes (error at encoding.py:*) (glob)
> -  hgext/convert/darcs.py: error importing: <TypeError> str expected, not bytes (error at encoding.py:*) (glob)
> -  hgext/convert/filemap.py: error importing: <TypeError> str expected, not bytes (error at encoding.py:*) (glob)
> -  hgext/convert/git.py: error importing: <TypeError> str expected, not bytes (error at encoding.py:*) (glob)
> -  hgext/convert/gnuarch.py: error importing: <TypeError> str expected, not bytes (error at encoding.py:*) (glob)
> -  hgext/convert/hg.py: error importing: <TypeError> str expected, not bytes (error at encoding.py:*) (glob)
> -  hgext/convert/monotone.py: error importing: <TypeError> str expected, not bytes (error at encoding.py:*) (glob)
> -  hgext/convert/p4.py: error importing: <TypeError> str expected, not bytes (error at encoding.py:*) (glob)
> -  hgext/convert/subversion.py: error importing: <TypeError> str expected, not bytes (error at encoding.py:*) (glob)
> +  hgext/convert/bzr.py: error importing: <TypeError> str expected, not bytes (error at i18n.py:*) (glob)
> +  hgext/convert/common.py: error importing: <TypeError> str expected, not bytes (error at i18n.py:*) (glob)
> +  hgext/convert/convcmd.py: error importing: <TypeError> str expected, not bytes (error at i18n.py:*) (glob)
> +  hgext/convert/cvs.py: error importing: <TypeError> str expected, not bytes (error at i18n.py:*) (glob)
> +  hgext/convert/cvsps.py: error importing: <TypeError> str expected, not bytes (error at i18n.py:*) (glob)
> +  hgext/convert/darcs.py: error importing: <TypeError> str expected, not bytes (error at i18n.py:*) (glob)
> +  hgext/convert/filemap.py: error importing: <TypeError> str expected, not bytes (error at i18n.py:*) (glob)
> +  hgext/convert/git.py: error importing: <TypeError> str expected, not bytes (error at i18n.py:*) (glob)
> +  hgext/convert/gnuarch.py: error importing: <TypeError> str expected, not bytes (error at i18n.py:*) (glob)
> +  hgext/convert/hg.py: error importing: <TypeError> str expected, not bytes (error at i18n.py:*) (glob)
> +  hgext/convert/monotone.py: error importing: <TypeError> str expected, not bytes (error at i18n.py:*) (glob)
> +  hgext/convert/p4.py: error importing: <TypeError> str expected, not bytes (error at i18n.py:*) (glob)
> +  hgext/convert/subversion.py: error importing: <TypeError> str expected, not bytes (error at i18n.py:*) (glob)
>     hgext/convert/transport.py: error importing module: <ImportError> No module named 'svn.client' (line *) (glob)
> -  hgext/eol.py: error importing: <TypeError> str expected, not bytes (error at encoding.py:*) (glob)
> -  hgext/extdiff.py: error importing: <TypeError> str expected, not bytes (error at encoding.py:*) (glob)
> -  hgext/factotum.py: error importing: <TypeError> str expected, not bytes (error at encoding.py:*) (glob)
> -  hgext/fetch.py: error importing: <TypeError> str expected, not bytes (error at encoding.py:*) (glob)
> -  hgext/fsmonitor/state.py: error importing: <TypeError> str expected, not bytes (error at encoding.py:*) (glob)
> -  hgext/fsmonitor/watchmanclient.py: error importing: <TypeError> str expected, not bytes (error at encoding.py:*) (glob)
> -  hgext/gpg.py: error importing: <TypeError> str expected, not bytes (error at encoding.py:*) (glob)
> -  hgext/graphlog.py: error importing: <TypeError> str expected, not bytes (error at encoding.py:*) (glob)
> -  hgext/hgk.py: error importing: <TypeError> str expected, not bytes (error at encoding.py:*) (glob)
> -  hgext/highlight/highlight.py: error importing: <TypeError> str expected, not bytes (error at encoding.py:*) (glob)
> -  hgext/histedit.py: error importing: <TypeError> str expected, not bytes (error at encoding.py:*) (glob)
> -  hgext/journal.py: error importing: <TypeError> str expected, not bytes (error at encoding.py:*) (glob)
> -  hgext/keyword.py: error importing: <TypeError> str expected, not bytes (error at encoding.py:*) (glob)
> -  hgext/largefiles/basestore.py: error importing: <TypeError> str expected, not bytes (error at encoding.py:*) (glob)
> -  hgext/largefiles/lfcommands.py: error importing: <TypeError> str expected, not bytes (error at encoding.py:*) (glob)
> -  hgext/largefiles/lfutil.py: error importing: <TypeError> str expected, not bytes (error at encoding.py:*) (glob)
> -  hgext/largefiles/localstore.py: error importing: <TypeError> str expected, not bytes (error at encoding.py:*) (glob)
> -  hgext/largefiles/overrides.py: error importing: <TypeError> str expected, not bytes (error at encoding.py:*) (glob)
> -  hgext/largefiles/proto.py: error importing: <TypeError> str expected, not bytes (error at encoding.py:*) (glob)
> -  hgext/largefiles/remotestore.py: error importing: <TypeError> str expected, not bytes (error at encoding.py:*) (glob)
> -  hgext/largefiles/reposetup.py: error importing: <TypeError> str expected, not bytes (error at encoding.py:*) (glob)
> -  hgext/largefiles/storefactory.py: error importing: <TypeError> str expected, not bytes (error at encoding.py:*) (glob)
> -  hgext/largefiles/uisetup.py: error importing: <TypeError> str expected, not bytes (error at encoding.py:*) (glob)
> +  hgext/eol.py: error importing: <TypeError> str expected, not bytes (error at i18n.py:*) (glob)
> +  hgext/extdiff.py: error importing: <TypeError> str expected, not bytes (error at i18n.py:*) (glob)
> +  hgext/factotum.py: error importing: <TypeError> str expected, not bytes (error at i18n.py:*) (glob)
> +  hgext/fetch.py: error importing: <TypeError> str expected, not bytes (error at i18n.py:*) (glob)
> +  hgext/fsmonitor/state.py: error importing: <TypeError> str expected, not bytes (error at i18n.py:*) (glob)
> +  hgext/fsmonitor/watchmanclient.py: error importing: <TypeError> str expected, not bytes (error at i18n.py:*) (glob)
> +  hgext/gpg.py: error importing: <TypeError> str expected, not bytes (error at i18n.py:*) (glob)
> +  hgext/graphlog.py: error importing: <TypeError> str expected, not bytes (error at i18n.py:*) (glob)
> +  hgext/hgk.py: error importing: <TypeError> str expected, not bytes (error at i18n.py:*) (glob)
> +  hgext/highlight/highlight.py: error importing: <TypeError> str expected, not bytes (error at i18n.py:*) (glob)
> +  hgext/histedit.py: error importing: <TypeError> str expected, not bytes (error at i18n.py:*) (glob)
> +  hgext/journal.py: error importing: <TypeError> str expected, not bytes (error at i18n.py:*) (glob)
> +  hgext/keyword.py: error importing: <TypeError> str expected, not bytes (error at i18n.py:*) (glob)
> +  hgext/largefiles/basestore.py: error importing: <TypeError> str expected, not bytes (error at i18n.py:*) (glob)
> +  hgext/largefiles/lfcommands.py: error importing: <TypeError> str expected, not bytes (error at i18n.py:*) (glob)
> +  hgext/largefiles/lfutil.py: error importing: <TypeError> str expected, not bytes (error at i18n.py:*) (glob)
> +  hgext/largefiles/localstore.py: error importing: <TypeError> str expected, not bytes (error at i18n.py:*) (glob)
> +  hgext/largefiles/overrides.py: error importing: <TypeError> str expected, not bytes (error at i18n.py:*) (glob)
> +  hgext/largefiles/proto.py: error importing: <TypeError> str expected, not bytes (error at i18n.py:*) (glob)
> +  hgext/largefiles/remotestore.py: error importing: <TypeError> str expected, not bytes (error at i18n.py:*) (glob)
> +  hgext/largefiles/reposetup.py: error importing: <TypeError> str expected, not bytes (error at i18n.py:*) (glob)
> +  hgext/largefiles/storefactory.py: error importing: <TypeError> str expected, not bytes (error at i18n.py:*) (glob)
> +  hgext/largefiles/uisetup.py: error importing: <TypeError> str expected, not bytes (error at i18n.py:*) (glob)
>     hgext/largefiles/wirestore.py: error importing module: <SystemError> Parent module 'hgext.largefiles' not loaded, cannot perform relative import (line *) (glob)
> -  hgext/mq.py: error importing: <TypeError> str expected, not bytes (error at encoding.py:*) (glob)
> -  hgext/notify.py: error importing: <TypeError> str expected, not bytes (error at encoding.py:*) (glob)
> -  hgext/pager.py: error importing: <TypeError> str expected, not bytes (error at encoding.py:*) (glob)
> -  hgext/patchbomb.py: error importing: <TypeError> str expected, not bytes (error at encoding.py:*) (glob)
> -  hgext/purge.py: error importing: <TypeError> str expected, not bytes (error at encoding.py:*) (glob)
> -  hgext/rebase.py: error importing: <TypeError> str expected, not bytes (error at encoding.py:*) (glob)
> -  hgext/record.py: error importing: <TypeError> str expected, not bytes (error at encoding.py:*) (glob)
> -  hgext/relink.py: error importing: <TypeError> str expected, not bytes (error at encoding.py:*) (glob)
> -  hgext/schemes.py: error importing: <TypeError> str expected, not bytes (error at encoding.py:*) (glob)
> -  hgext/share.py: error importing: <TypeError> str expected, not bytes (error at encoding.py:*) (glob)
> -  hgext/shelve.py: error importing: <TypeError> str expected, not bytes (error at encoding.py:*) (glob)
> -  hgext/strip.py: error importing: <TypeError> str expected, not bytes (error at encoding.py:*) (glob)
> -  hgext/transplant.py: error importing: <TypeError> str expected, not bytes (error at encoding.py:*) (glob)
> -  hgext/win32mbcs.py: error importing: <TypeError> str expected, not bytes (error at encoding.py:*) (glob)
> -  hgext/win32text.py: error importing: <TypeError> str expected, not bytes (error at encoding.py:*) (glob)
> +  hgext/mq.py: error importing: <TypeError> str expected, not bytes (error at i18n.py:*) (glob)
> +  hgext/notify.py: error importing: <TypeError> str expected, not bytes (error at i18n.py:*) (glob)
> +  hgext/pager.py: error importing: <TypeError> str expected, not bytes (error at i18n.py:*) (glob)
> +  hgext/patchbomb.py: error importing: <TypeError> str expected, not bytes (error at i18n.py:*) (glob)
> +  hgext/purge.py: error importing: <TypeError> str expected, not bytes (error at i18n.py:*) (glob)
> +  hgext/rebase.py: error importing: <TypeError> str expected, not bytes (error at i18n.py:*) (glob)
> +  hgext/record.py: error importing: <TypeError> str expected, not bytes (error at i18n.py:*) (glob)
> +  hgext/relink.py: error importing: <TypeError> str expected, not bytes (error at i18n.py:*) (glob)
> +  hgext/schemes.py: error importing: <TypeError> str expected, not bytes (error at i18n.py:*) (glob)
> +  hgext/share.py: error importing: <TypeError> str expected, not bytes (error at i18n.py:*) (glob)
> +  hgext/shelve.py: error importing: <TypeError> str expected, not bytes (error at i18n.py:*) (glob)
> +  hgext/strip.py: error importing: <TypeError> str expected, not bytes (error at i18n.py:*) (glob)
> +  hgext/transplant.py: error importing: <TypeError> str expected, not bytes (error at i18n.py:*) (glob)
> +  hgext/win32mbcs.py: error importing: <TypeError> str expected, not bytes (error at i18n.py:*) (glob)
> +  hgext/win32text.py: error importing: <TypeError> str expected, not bytes (error at i18n.py:*) (glob)
>     mercurial/archival.py: invalid syntax: invalid syntax (<unknown>, line *) (glob)
> -  mercurial/bookmarks.py: error importing: <TypeError> str expected, not bytes (error at encoding.py:*) (glob)
> -  mercurial/branchmap.py: error importing: <TypeError> str expected, not bytes (error at encoding.py:*) (glob)
> +  mercurial/bookmarks.py: error importing: <TypeError> str expected, not bytes (error at i18n.py:*) (glob)
> +  mercurial/branchmap.py: error importing: <TypeError> str expected, not bytes (error at i18n.py:*) (glob)
>     mercurial/bundle2.py: invalid syntax: invalid syntax (<unknown>, line *) (glob)
> -  mercurial/bundlerepo.py: error importing: <TypeError> str expected, not bytes (error at encoding.py:*) (glob)
> -  mercurial/byterange.py: error importing: <TypeError> str expected, not bytes (error at encoding.py:*) (glob)
> -  mercurial/changegroup.py: error importing: <TypeError> str expected, not bytes (error at encoding.py:*) (glob)
> -  mercurial/changelog.py: error importing: <TypeError> str expected, not bytes (error at encoding.py:*) (glob)
> -  mercurial/cmdutil.py: error importing: <TypeError> str expected, not bytes (error at encoding.py:*) (glob)
> +  mercurial/bundlerepo.py: error importing: <TypeError> str expected, not bytes (error at i18n.py:*) (glob)
> +  mercurial/byterange.py: error importing: <TypeError> str expected, not bytes (error at i18n.py:*) (glob)
> +  mercurial/changegroup.py: error importing: <TypeError> str expected, not bytes (error at i18n.py:*) (glob)
> +  mercurial/changelog.py: error importing: <TypeError> str expected, not bytes (error at i18n.py:*) (glob)
> +  mercurial/cmdutil.py: error importing: <TypeError> str expected, not bytes (error at i18n.py:*) (glob)
>     mercurial/commands.py: invalid syntax: invalid syntax (<unknown>, line *) (glob)
> -  mercurial/commandserver.py: error importing: <TypeError> str expected, not bytes (error at encoding.py:*) (glob)
> -  mercurial/config.py: error importing: <TypeError> str expected, not bytes (error at encoding.py:*) (glob)
> -  mercurial/context.py: error importing: <TypeError> str expected, not bytes (error at encoding.py:*) (glob)
> -  mercurial/copies.py: error importing: <TypeError> str expected, not bytes (error at encoding.py:*) (glob)
> -  mercurial/crecord.py: error importing: <TypeError> str expected, not bytes (error at encoding.py:*) (glob)
> -  mercurial/dagparser.py: error importing: <TypeError> str expected, not bytes (error at encoding.py:*) (glob)
> -  mercurial/dagutil.py: error importing: <TypeError> str expected, not bytes (error at encoding.py:*) (glob)
> -  mercurial/destutil.py: error importing: <TypeError> str expected, not bytes (error at encoding.py:*) (glob)
> -  mercurial/dirstate.py: error importing: <TypeError> str expected, not bytes (error at encoding.py:*) (glob)
> -  mercurial/discovery.py: error importing: <TypeError> str expected, not bytes (error at encoding.py:*) (glob)
> -  mercurial/dispatch.py: error importing: <TypeError> str expected, not bytes (error at encoding.py:*) (glob)
> +  mercurial/commandserver.py: error importing: <TypeError> str expected, not bytes (error at i18n.py:*) (glob)
> +  mercurial/config.py: error importing: <TypeError> str expected, not bytes (error at i18n.py:*) (glob)
> +  mercurial/context.py: error importing: <TypeError> str expected, not bytes (error at i18n.py:*) (glob)
> +  mercurial/copies.py: error importing: <TypeError> str expected, not bytes (error at i18n.py:*) (glob)
> +  mercurial/crecord.py: error importing: <TypeError> str expected, not bytes (error at i18n.py:*) (glob)
> +  mercurial/dagparser.py: error importing: <TypeError> str expected, not bytes (error at i18n.py:*) (glob)
> +  mercurial/dagutil.py: error importing: <TypeError> str expected, not bytes (error at i18n.py:*) (glob)
> +  mercurial/destutil.py: error importing: <TypeError> str expected, not bytes (error at i18n.py:*) (glob)
> +  mercurial/dirstate.py: error importing: <TypeError> str expected, not bytes (error at i18n.py:*) (glob)
> +  mercurial/discovery.py: error importing: <TypeError> str expected, not bytes (error at i18n.py:*) (glob)
> +  mercurial/dispatch.py: error importing: <TypeError> str expected, not bytes (error at i18n.py:*) (glob)
>     mercurial/exchange.py: error importing: <TypeError> str expected, not bytes (error at i18n.py:*) (glob)
>     mercurial/extensions.py: error importing: <TypeError> str expected, not bytes (error at i18n.py:*) (glob)
>     mercurial/fancyopts.py: error importing: <TypeError> str expected, not bytes (error at i18n.py:*) (glob)
> _______________________________________________
> Mercurial-devel mailing list
> Mercurial-devel at mercurial-scm.org
> https://www.mercurial-scm.org/mailman/listinfo/mercurial-devel