[PATCH] py3: handle os.environ.get() case in module loader
Gregory Szorc
gregory.szorc at gmail.com
Thu Aug 4 23:15:24 EDT 2016
On Thu, Aug 4, 2016 at 12:11 PM, Pulkit Goyal <7895pulkit at gmail.com> wrote:
> # HG changeset patch
> # User Pulkit Goyal <7895pulkit at gmail.com>
> # Date 1470337846 -19800
> # Fri Aug 05 00:40:46 2016 +0530
> # Branch stable
> # Node ID 6bc900348ed8d695dddc9451f56e3adf38149127
> # Parent 44c45c5f2481e94b8eeb24b31fea4b49d3453835
> py3: handle os.environ.get() case in module loader
>
> The os.environ.get() doesnot accepts bytes on Python 3. Instead of adding
> u''
> in the source code, its preferred to add a case in the custom module loader
> which handles this.
>
> Firstly the code rewrite the token to include u'' to undo the effect by the
> transformer on the first argument. Then it searches for more arguments if
> present and does the same for them.
>
I'm not sure if we decided this was the proper course of action. But I'd
rather have a helper that accesses os.environ or os.environb (as
appropriate) and does the appropriate types transform (as appropriate)
rather than hacking more things into the module loader.
>
> diff -r 44c45c5f2481 -r 6bc900348ed8 mercurial/__init__.py
> --- a/mercurial/__init__.py Thu Aug 04 00:32:19 2016 +0530
> +++ b/mercurial/__init__.py Fri Aug 05 00:40:46 2016 +0530
> @@ -272,6 +272,47 @@
> except IndexError:
> pass
>
> + # os.environ.get() doesnot accepts byte strings on Python
> 3.
> + # Rewrite the token to include the unicode literal prefix
> so
> + # the string transformer above doesn't add the byte
> prefix.
> + if (fn in ('get') and
> + prevtoken.type == token.OP and prevtoken.string == '.'
> + and tokens[i - 2].string == 'environ' and
> + tokens[i - 4].string == 'os'):
> + # (tokens[i-4:i-2], os.environ)
> + # (OP, '.')
> + # (NAME, 'get')
> + # (OP, '(')
> + # (STRING, 'HGENCODING')
> + # (OP, ')')
> + try:
> + st = tokens[i + 2]
> + if (st.type == token.STRING and
> + st.string[0] in ("'", '"')):
> + rt = tokenize.TokenInfo(st.type, 'u%s' %
> st.string,
> + st.start, st.end,
> st.line)
> + tokens[i + 2] = rt
> +
> + # This while loop deals with the further
> arguments in
> + # in os.environ.get(). It checks whether we have
> ')'
> + # as the next token or we have a ',' which
> indicates
> + # that we have more arguments. This loop
> terminates
> + # when we get the ')' token.
> + j = 3
> + while (tokens[i + j].string != ')' and
> + tokens[i + j].string == ','):
> + st = tokens[i + j + 1]
> + if (st.type == token.STRING and
> + st.string[0] in ("'", '"')):
> + rt = tokenize.TokenInfo(st.type, 'u%s' %
> + st.string,
> st.start,
> + st.end, st.line)
> + tokens[i + j + 1] = rt
> + j += 2
> +
> + except IndexError:
> + pass
> +
> # Emit unmodified token.
> yield t
>
> @@ -279,7 +320,7 @@
> # ``replacetoken`` or any mechanism that changes semantics of module
> # loading is changed. Otherwise cached bytecode may get loaded without
> # the new transformation mechanisms applied.
> - BYTECODEHEADER = b'HG\x00\x01'
> + BYTECODEHEADER = b'HG\x00\x02'
>
> class hgloader(importlib.machinery.SourceFileLoader):
> """Custom module loader that transforms source code.
> diff -r 44c45c5f2481 -r 6bc900348ed8 tests/test-check-py3-compat.t
> --- a/tests/test-check-py3-compat.t Thu Aug 04 00:32:19 2016 +0530
> +++ b/tests/test-check-py3-compat.t Fri Aug 05 00:40:46 2016 +0530
> @@ -15,90 +15,90 @@
> #if py3exe
> $ hg files 'set:(**.py)' | sed 's|\\|/|g' | xargs $PYTHON3
> contrib/check-py3-compat.py
> doc/hgmanpage.py: invalid syntax: invalid syntax (<unknown>, line *)
> (glob)
> - hgext/acl.py: error importing: <TypeError> str expected, not bytes
> (error at encoding.py:*) (glob)
> - hgext/automv.py: error importing: <TypeError> str expected, not bytes
> (error at encoding.py:*) (glob)
> - hgext/blackbox.py: error importing: <TypeError> str expected, not bytes
> (error at encoding.py:*) (glob)
> - hgext/bugzilla.py: error importing: <TypeError> str expected, not bytes
> (error at encoding.py:*) (glob)
> - hgext/censor.py: error importing: <TypeError> str expected, not bytes
> (error at encoding.py:*) (glob)
> - hgext/chgserver.py: error importing: <TypeError> str expected, not
> bytes (error at encoding.py:*) (glob)
> - hgext/children.py: error importing: <TypeError> str expected, not bytes
> (error at encoding.py:*) (glob)
> - hgext/churn.py: error importing: <TypeError> str expected, not bytes
> (error at encoding.py:*) (glob)
> - hgext/clonebundles.py: error importing: <TypeError> str expected, not
> bytes (error at encoding.py:*) (glob)
> + hgext/acl.py: error importing: <TypeError> str expected, not bytes
> (error at i18n.py:*) (glob)
> + hgext/automv.py: error importing: <TypeError> str expected, not bytes
> (error at i18n.py:*) (glob)
> + hgext/blackbox.py: error importing: <TypeError> str expected, not bytes
> (error at i18n.py:*) (glob)
> + hgext/bugzilla.py: error importing: <TypeError> str expected, not bytes
> (error at i18n.py:*) (glob)
> + hgext/censor.py: error importing: <TypeError> str expected, not bytes
> (error at i18n.py:*) (glob)
> + hgext/chgserver.py: error importing: <TypeError> str expected, not
> bytes (error at i18n.py:*) (glob)
> + hgext/children.py: error importing: <TypeError> str expected, not bytes
> (error at i18n.py:*) (glob)
> + hgext/churn.py: error importing: <TypeError> str expected, not bytes
> (error at i18n.py:*) (glob)
> + hgext/clonebundles.py: error importing: <TypeError> str expected, not
> bytes (error at i18n.py:*) (glob)
> hgext/color.py: invalid syntax: invalid syntax (<unknown>, line *)
> (glob)
> - hgext/convert/bzr.py: error importing: <TypeError> str expected, not
> bytes (error at encoding.py:*) (glob)
> - hgext/convert/common.py: error importing: <TypeError> str expected, not
> bytes (error at encoding.py:*) (glob)
> - hgext/convert/convcmd.py: error importing: <TypeError> str expected,
> not bytes (error at encoding.py:*) (glob)
> - hgext/convert/cvs.py: error importing: <TypeError> str expected, not
> bytes (error at encoding.py:*) (glob)
> - hgext/convert/cvsps.py: error importing: <TypeError> str expected, not
> bytes (error at encoding.py:*) (glob)
> - hgext/convert/darcs.py: error importing: <TypeError> str expected, not
> bytes (error at encoding.py:*) (glob)
> - hgext/convert/filemap.py: error importing: <TypeError> str expected,
> not bytes (error at encoding.py:*) (glob)
> - hgext/convert/git.py: error importing: <TypeError> str expected, not
> bytes (error at encoding.py:*) (glob)
> - hgext/convert/gnuarch.py: error importing: <TypeError> str expected,
> not bytes (error at encoding.py:*) (glob)
> - hgext/convert/hg.py: error importing: <TypeError> str expected, not
> bytes (error at encoding.py:*) (glob)
> - hgext/convert/monotone.py: error importing: <TypeError> str expected,
> not bytes (error at encoding.py:*) (glob)
> - hgext/convert/p4.py: error importing: <TypeError> str expected, not
> bytes (error at encoding.py:*) (glob)
> - hgext/convert/subversion.py: error importing: <TypeError> str expected,
> not bytes (error at encoding.py:*) (glob)
> + hgext/convert/bzr.py: error importing: <TypeError> str expected, not
> bytes (error at i18n.py:*) (glob)
> + hgext/convert/common.py: error importing: <TypeError> str expected, not
> bytes (error at i18n.py:*) (glob)
> + hgext/convert/convcmd.py: error importing: <TypeError> str expected,
> not bytes (error at i18n.py:*) (glob)
> + hgext/convert/cvs.py: error importing: <TypeError> str expected, not
> bytes (error at i18n.py:*) (glob)
> + hgext/convert/cvsps.py: error importing: <TypeError> str expected, not
> bytes (error at i18n.py:*) (glob)
> + hgext/convert/darcs.py: error importing: <TypeError> str expected, not
> bytes (error at i18n.py:*) (glob)
> + hgext/convert/filemap.py: error importing: <TypeError> str expected,
> not bytes (error at i18n.py:*) (glob)
> + hgext/convert/git.py: error importing: <TypeError> str expected, not
> bytes (error at i18n.py:*) (glob)
> + hgext/convert/gnuarch.py: error importing: <TypeError> str expected,
> not bytes (error at i18n.py:*) (glob)
> + hgext/convert/hg.py: error importing: <TypeError> str expected, not
> bytes (error at i18n.py:*) (glob)
> + hgext/convert/monotone.py: error importing: <TypeError> str expected,
> not bytes (error at i18n.py:*) (glob)
> + hgext/convert/p4.py: error importing: <TypeError> str expected, not
> bytes (error at i18n.py:*) (glob)
> + hgext/convert/subversion.py: error importing: <TypeError> str expected,
> not bytes (error at i18n.py:*) (glob)
> hgext/convert/transport.py: error importing module: <ImportError> No
> module named 'svn.client' (line *) (glob)
> - hgext/eol.py: error importing: <TypeError> str expected, not bytes
> (error at encoding.py:*) (glob)
> - hgext/extdiff.py: error importing: <TypeError> str expected, not bytes
> (error at encoding.py:*) (glob)
> - hgext/factotum.py: error importing: <TypeError> str expected, not bytes
> (error at encoding.py:*) (glob)
> - hgext/fetch.py: error importing: <TypeError> str expected, not bytes
> (error at encoding.py:*) (glob)
> - hgext/fsmonitor/state.py: error importing: <TypeError> str expected,
> not bytes (error at encoding.py:*) (glob)
> - hgext/fsmonitor/watchmanclient.py: error importing: <TypeError> str
> expected, not bytes (error at encoding.py:*) (glob)
> - hgext/gpg.py: error importing: <TypeError> str expected, not bytes
> (error at encoding.py:*) (glob)
> - hgext/graphlog.py: error importing: <TypeError> str expected, not bytes
> (error at encoding.py:*) (glob)
> - hgext/hgk.py: error importing: <TypeError> str expected, not bytes
> (error at encoding.py:*) (glob)
> - hgext/highlight/highlight.py: error importing: <TypeError> str
> expected, not bytes (error at encoding.py:*) (glob)
> - hgext/histedit.py: error importing: <TypeError> str expected, not bytes
> (error at encoding.py:*) (glob)
> - hgext/journal.py: error importing: <TypeError> str expected, not bytes
> (error at encoding.py:*) (glob)
> - hgext/keyword.py: error importing: <TypeError> str expected, not bytes
> (error at encoding.py:*) (glob)
> - hgext/largefiles/basestore.py: error importing: <TypeError> str
> expected, not bytes (error at encoding.py:*) (glob)
> - hgext/largefiles/lfcommands.py: error importing: <TypeError> str
> expected, not bytes (error at encoding.py:*) (glob)
> - hgext/largefiles/lfutil.py: error importing: <TypeError> str expected,
> not bytes (error at encoding.py:*) (glob)
> - hgext/largefiles/localstore.py: error importing: <TypeError> str
> expected, not bytes (error at encoding.py:*) (glob)
> - hgext/largefiles/overrides.py: error importing: <TypeError> str
> expected, not bytes (error at encoding.py:*) (glob)
> - hgext/largefiles/proto.py: error importing: <TypeError> str expected,
> not bytes (error at encoding.py:*) (glob)
> - hgext/largefiles/remotestore.py: error importing: <TypeError> str
> expected, not bytes (error at encoding.py:*) (glob)
> - hgext/largefiles/reposetup.py: error importing: <TypeError> str
> expected, not bytes (error at encoding.py:*) (glob)
> - hgext/largefiles/storefactory.py: error importing: <TypeError> str
> expected, not bytes (error at encoding.py:*) (glob)
> - hgext/largefiles/uisetup.py: error importing: <TypeError> str expected,
> not bytes (error at encoding.py:*) (glob)
> + hgext/eol.py: error importing: <TypeError> str expected, not bytes
> (error at i18n.py:*) (glob)
> + hgext/extdiff.py: error importing: <TypeError> str expected, not bytes
> (error at i18n.py:*) (glob)
> + hgext/factotum.py: error importing: <TypeError> str expected, not bytes
> (error at i18n.py:*) (glob)
> + hgext/fetch.py: error importing: <TypeError> str expected, not bytes
> (error at i18n.py:*) (glob)
> + hgext/fsmonitor/state.py: error importing: <TypeError> str expected,
> not bytes (error at i18n.py:*) (glob)
> + hgext/fsmonitor/watchmanclient.py: error importing: <TypeError> str
> expected, not bytes (error at i18n.py:*) (glob)
> + hgext/gpg.py: error importing: <TypeError> str expected, not bytes
> (error at i18n.py:*) (glob)
> + hgext/graphlog.py: error importing: <TypeError> str expected, not bytes
> (error at i18n.py:*) (glob)
> + hgext/hgk.py: error importing: <TypeError> str expected, not bytes
> (error at i18n.py:*) (glob)
> + hgext/highlight/highlight.py: error importing: <TypeError> str
> expected, not bytes (error at i18n.py:*) (glob)
> + hgext/histedit.py: error importing: <TypeError> str expected, not bytes
> (error at i18n.py:*) (glob)
> + hgext/journal.py: error importing: <TypeError> str expected, not bytes
> (error at i18n.py:*) (glob)
> + hgext/keyword.py: error importing: <TypeError> str expected, not bytes
> (error at i18n.py:*) (glob)
> + hgext/largefiles/basestore.py: error importing: <TypeError> str
> expected, not bytes (error at i18n.py:*) (glob)
> + hgext/largefiles/lfcommands.py: error importing: <TypeError> str
> expected, not bytes (error at i18n.py:*) (glob)
> + hgext/largefiles/lfutil.py: error importing: <TypeError> str expected,
> not bytes (error at i18n.py:*) (glob)
> + hgext/largefiles/localstore.py: error importing: <TypeError> str
> expected, not bytes (error at i18n.py:*) (glob)
> + hgext/largefiles/overrides.py: error importing: <TypeError> str
> expected, not bytes (error at i18n.py:*) (glob)
> + hgext/largefiles/proto.py: error importing: <TypeError> str expected,
> not bytes (error at i18n.py:*) (glob)
> + hgext/largefiles/remotestore.py: error importing: <TypeError> str
> expected, not bytes (error at i18n.py:*) (glob)
> + hgext/largefiles/reposetup.py: error importing: <TypeError> str
> expected, not bytes (error at i18n.py:*) (glob)
> + hgext/largefiles/storefactory.py: error importing: <TypeError> str
> expected, not bytes (error at i18n.py:*) (glob)
> + hgext/largefiles/uisetup.py: error importing: <TypeError> str expected,
> not bytes (error at i18n.py:*) (glob)
> hgext/largefiles/wirestore.py: error importing module: <SystemError>
> Parent module 'hgext.largefiles' not loaded, cannot perform relative import
> (line *) (glob)
> - hgext/mq.py: error importing: <TypeError> str expected, not bytes
> (error at encoding.py:*) (glob)
> - hgext/notify.py: error importing: <TypeError> str expected, not bytes
> (error at encoding.py:*) (glob)
> - hgext/pager.py: error importing: <TypeError> str expected, not bytes
> (error at encoding.py:*) (glob)
> - hgext/patchbomb.py: error importing: <TypeError> str expected, not
> bytes (error at encoding.py:*) (glob)
> - hgext/purge.py: error importing: <TypeError> str expected, not bytes
> (error at encoding.py:*) (glob)
> - hgext/rebase.py: error importing: <TypeError> str expected, not bytes
> (error at encoding.py:*) (glob)
> - hgext/record.py: error importing: <TypeError> str expected, not bytes
> (error at encoding.py:*) (glob)
> - hgext/relink.py: error importing: <TypeError> str expected, not bytes
> (error at encoding.py:*) (glob)
> - hgext/schemes.py: error importing: <TypeError> str expected, not bytes
> (error at encoding.py:*) (glob)
> - hgext/share.py: error importing: <TypeError> str expected, not bytes
> (error at encoding.py:*) (glob)
> - hgext/shelve.py: error importing: <TypeError> str expected, not bytes
> (error at encoding.py:*) (glob)
> - hgext/strip.py: error importing: <TypeError> str expected, not bytes
> (error at encoding.py:*) (glob)
> - hgext/transplant.py: error importing: <TypeError> str expected, not
> bytes (error at encoding.py:*) (glob)
> - hgext/win32mbcs.py: error importing: <TypeError> str expected, not
> bytes (error at encoding.py:*) (glob)
> - hgext/win32text.py: error importing: <TypeError> str expected, not
> bytes (error at encoding.py:*) (glob)
> + hgext/mq.py: error importing: <TypeError> str expected, not bytes
> (error at i18n.py:*) (glob)
> + hgext/notify.py: error importing: <TypeError> str expected, not bytes
> (error at i18n.py:*) (glob)
> + hgext/pager.py: error importing: <TypeError> str expected, not bytes
> (error at i18n.py:*) (glob)
> + hgext/patchbomb.py: error importing: <TypeError> str expected, not
> bytes (error at i18n.py:*) (glob)
> + hgext/purge.py: error importing: <TypeError> str expected, not bytes
> (error at i18n.py:*) (glob)
> + hgext/rebase.py: error importing: <TypeError> str expected, not bytes
> (error at i18n.py:*) (glob)
> + hgext/record.py: error importing: <TypeError> str expected, not bytes
> (error at i18n.py:*) (glob)
> + hgext/relink.py: error importing: <TypeError> str expected, not bytes
> (error at i18n.py:*) (glob)
> + hgext/schemes.py: error importing: <TypeError> str expected, not bytes
> (error at i18n.py:*) (glob)
> + hgext/share.py: error importing: <TypeError> str expected, not bytes
> (error at i18n.py:*) (glob)
> + hgext/shelve.py: error importing: <TypeError> str expected, not bytes
> (error at i18n.py:*) (glob)
> + hgext/strip.py: error importing: <TypeError> str expected, not bytes
> (error at i18n.py:*) (glob)
> + hgext/transplant.py: error importing: <TypeError> str expected, not
> bytes (error at i18n.py:*) (glob)
> + hgext/win32mbcs.py: error importing: <TypeError> str expected, not
> bytes (error at i18n.py:*) (glob)
> + hgext/win32text.py: error importing: <TypeError> str expected, not
> bytes (error at i18n.py:*) (glob)
> mercurial/archival.py: invalid syntax: invalid syntax (<unknown>, line
> *) (glob)
> - mercurial/bookmarks.py: error importing: <TypeError> str expected, not
> bytes (error at encoding.py:*) (glob)
> - mercurial/branchmap.py: error importing: <TypeError> str expected, not
> bytes (error at encoding.py:*) (glob)
> + mercurial/bookmarks.py: error importing: <TypeError> str expected, not
> bytes (error at i18n.py:*) (glob)
> + mercurial/branchmap.py: error importing: <TypeError> str expected, not
> bytes (error at i18n.py:*) (glob)
> mercurial/bundle2.py: invalid syntax: invalid syntax (<unknown>, line
> *) (glob)
> - mercurial/bundlerepo.py: error importing: <TypeError> str expected, not
> bytes (error at encoding.py:*) (glob)
> - mercurial/byterange.py: error importing: <TypeError> str expected, not
> bytes (error at encoding.py:*) (glob)
> - mercurial/changegroup.py: error importing: <TypeError> str expected,
> not bytes (error at encoding.py:*) (glob)
> - mercurial/changelog.py: error importing: <TypeError> str expected, not
> bytes (error at encoding.py:*) (glob)
> - mercurial/cmdutil.py: error importing: <TypeError> str expected, not
> bytes (error at encoding.py:*) (glob)
> + mercurial/bundlerepo.py: error importing: <TypeError> str expected, not
> bytes (error at i18n.py:*) (glob)
> + mercurial/byterange.py: error importing: <TypeError> str expected, not
> bytes (error at i18n.py:*) (glob)
> + mercurial/changegroup.py: error importing: <TypeError> str expected,
> not bytes (error at i18n.py:*) (glob)
> + mercurial/changelog.py: error importing: <TypeError> str expected, not
> bytes (error at i18n.py:*) (glob)
> + mercurial/cmdutil.py: error importing: <TypeError> str expected, not
> bytes (error at i18n.py:*) (glob)
> mercurial/commands.py: invalid syntax: invalid syntax (<unknown>, line
> *) (glob)
> - mercurial/commandserver.py: error importing: <TypeError> str expected,
> not bytes (error at encoding.py:*) (glob)
> - mercurial/config.py: error importing: <TypeError> str expected, not
> bytes (error at encoding.py:*) (glob)
> - mercurial/context.py: error importing: <TypeError> str expected, not
> bytes (error at encoding.py:*) (glob)
> - mercurial/copies.py: error importing: <TypeError> str expected, not
> bytes (error at encoding.py:*) (glob)
> - mercurial/crecord.py: error importing: <TypeError> str expected, not
> bytes (error at encoding.py:*) (glob)
> - mercurial/dagparser.py: error importing: <TypeError> str expected, not
> bytes (error at encoding.py:*) (glob)
> - mercurial/dagutil.py: error importing: <TypeError> str expected, not
> bytes (error at encoding.py:*) (glob)
> - mercurial/destutil.py: error importing: <TypeError> str expected, not
> bytes (error at encoding.py:*) (glob)
> - mercurial/dirstate.py: error importing: <TypeError> str expected, not
> bytes (error at encoding.py:*) (glob)
> - mercurial/discovery.py: error importing: <TypeError> str expected, not
> bytes (error at encoding.py:*) (glob)
> - mercurial/dispatch.py: error importing: <TypeError> str expected, not
> bytes (error at encoding.py:*) (glob)
> + mercurial/commandserver.py: error importing: <TypeError> str expected,
> not bytes (error at i18n.py:*) (glob)
> + mercurial/config.py: error importing: <TypeError> str expected, not
> bytes (error at i18n.py:*) (glob)
> + mercurial/context.py: error importing: <TypeError> str expected, not
> bytes (error at i18n.py:*) (glob)
> + mercurial/copies.py: error importing: <TypeError> str expected, not
> bytes (error at i18n.py:*) (glob)
> + mercurial/crecord.py: error importing: <TypeError> str expected, not
> bytes (error at i18n.py:*) (glob)
> + mercurial/dagparser.py: error importing: <TypeError> str expected, not
> bytes (error at i18n.py:*) (glob)
> + mercurial/dagutil.py: error importing: <TypeError> str expected, not
> bytes (error at i18n.py:*) (glob)
> + mercurial/destutil.py: error importing: <TypeError> str expected, not
> bytes (error at i18n.py:*) (glob)
> + mercurial/dirstate.py: error importing: <TypeError> str expected, not
> bytes (error at i18n.py:*) (glob)
> + mercurial/discovery.py: error importing: <TypeError> str expected, not
> bytes (error at i18n.py:*) (glob)
> + mercurial/dispatch.py: error importing: <TypeError> str expected, not
> bytes (error at i18n.py:*) (glob)
> mercurial/exchange.py: error importing: <TypeError> str expected, not
> bytes (error at i18n.py:*) (glob)
> mercurial/extensions.py: error importing: <TypeError> str expected, not
> bytes (error at i18n.py:*) (glob)
> mercurial/fancyopts.py: error importing: <TypeError> str expected, not
> bytes (error at i18n.py:*) (glob)
> _______________________________________________
> Mercurial-devel mailing list
> Mercurial-devel at mercurial-scm.org
> https://www.mercurial-scm.org/mailman/listinfo/mercurial-devel
>
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://www.mercurial-scm.org/pipermail/mercurial-devel/attachments/20160804/f3270045/attachment-0001.html>
More information about the Mercurial-devel
mailing list