[PATCH 5 of 5 V2] py3: import builtin wrappers automagically by code transformer

Yuya Nishihara yuya at tcha.org
Tue Aug 16 04:50:46 EDT 2016


# HG changeset patch
# User Yuya Nishihara <yuya at tcha.org>
# Date 1471318515 -32400
#      Tue Aug 16 12:35:15 2016 +0900
# Node ID 30c0fc2f9d07b0251e24ff1a61a23416038022c4
# Parent  82f7a2639ea514b22a2cbb2000740dce513663e2
py3: import builtin wrappers automagically by code transformer

This should be less invasive than mucking builtins.

Since tokenize.untokenize() looks start/end positions of tokens, we calculates
them from the NEWLINE token of the future import.

diff --git a/mercurial/__init__.py b/mercurial/__init__.py
--- a/mercurial/__init__.py
+++ b/mercurial/__init__.py
@@ -170,7 +170,7 @@ if sys.version_info[0] >= 3:
             spec.loader = hgloader(spec.name, spec.origin)
             return spec
 
-    def replacetokens(tokens):
+    def replacetokens(tokens, fullname):
         """Transform a stream of tokens from raw to Python 3.
 
         It is called by the custom module loading machinery to rewrite
@@ -184,6 +184,7 @@ if sys.version_info[0] >= 3:
         REMEMBER TO CHANGE ``BYTECODEHEADER`` WHEN CHANGING THIS FUNCTION
         OR CACHED FILES WON'T GET INVALIDATED PROPERLY.
         """
+        futureimpline = False
         for i, t in enumerate(tokens):
             # Convert most string literals to byte literals. String literals
             # in Python 2 are bytes. String literals in Python 3 are unicode.
@@ -217,6 +218,29 @@ if sys.version_info[0] >= 3:
                                           t.line)
                 continue
 
+            # Insert compatibility imports at "from __future__ import" line.
+            # No '\n' should be added to preserve line numbers.
+            if (t.type == token.NAME and t.string == 'import' and
+                all(u.type == token.NAME for u in tokens[i - 2:i]) and
+                [u.string for u in tokens[i - 2:i]] == ['from', '__future__']):
+                futureimpline = True
+            if t.type == token.NEWLINE and futureimpline:
+                futureimpline = False
+                if fullname == 'mercurial.pycompat':
+                    yield t
+                    continue
+                r, c = t.start
+                l = (b'; from mercurial.pycompat import '
+                     b'delattr, getattr, hasattr, setattr, xrange\n')
+                for u in tokenize.tokenize(io.BytesIO(l).readline):
+                    if u.type in (tokenize.ENCODING, token.ENDMARKER):
+                        continue
+                    yield tokenize.TokenInfo(u.type, u.string,
+                                             (r, c + u.start[1]),
+                                             (r, c + u.end[1]),
+                                             '')
+                continue
+
             try:
                 nexttoken = tokens[i + 1]
             except IndexError:
@@ -279,7 +303,7 @@ if sys.version_info[0] >= 3:
     # ``replacetoken`` or any mechanism that changes semantics of module
     # loading is changed. Otherwise cached bytecode may get loaded without
     # the new transformation mechanisms applied.
-    BYTECODEHEADER = b'HG\x00\x01'
+    BYTECODEHEADER = b'HG\x00\x02'
 
     class hgloader(importlib.machinery.SourceFileLoader):
         """Custom module loader that transforms source code.
@@ -338,7 +362,7 @@ if sys.version_info[0] >= 3:
             """Perform token transformation before compilation."""
             buf = io.BytesIO(data)
             tokens = tokenize.tokenize(buf.readline)
-            data = tokenize.untokenize(replacetokens(list(tokens)))
+            data = tokenize.untokenize(replacetokens(list(tokens), self.name))
             # Python's built-in importer strips frames from exceptions raised
             # for this code. Unfortunately, that mechanism isn't extensible
             # and our frame will be blamed for the import failure. There
diff --git a/mercurial/pycompat.py b/mercurial/pycompat.py
--- a/mercurial/pycompat.py
+++ b/mercurial/pycompat.py
@@ -32,7 +32,6 @@ else:
 if sys.version_info[0] >= 3:
     import builtins
     import functools
-    builtins.xrange = range
 
     def _wrapattrfunc(f):
         @functools.wraps(f)
@@ -42,10 +41,12 @@ if sys.version_info[0] >= 3:
             return f(object, name, *args)
         return w
 
+    # these wrappers are automagically imported by hgloader
     delattr = _wrapattrfunc(builtins.delattr)
     getattr = _wrapattrfunc(builtins.getattr)
     hasattr = _wrapattrfunc(builtins.hasattr)
     setattr = _wrapattrfunc(builtins.setattr)
+    xrange = builtins.range
 
 stringio = io.StringIO
 empty = _queue.Empty
diff --git a/tests/test-check-py3-compat.t b/tests/test-check-py3-compat.t
--- a/tests/test-check-py3-compat.t
+++ b/tests/test-check-py3-compat.t
@@ -122,48 +122,49 @@
   mercurial/hook.py: error importing: <TypeError> str expected, not bytes (error at i18n.py:*) (glob)
   mercurial/httpconnection.py: error importing: <TypeError> str expected, not bytes (error at i18n.py:*) (glob)
   mercurial/httppeer.py: error importing: <TypeError> str expected, not bytes (error at i18n.py:*) (glob)
-  mercurial/keepalive.py: error importing: <TypeError> getattr(): attribute name must be string (error at util.py:*) (glob)
-  mercurial/localrepo.py: error importing: <TypeError> getattr(): attribute name must be string (error at util.py:*) (glob)
-  mercurial/lock.py: error importing: <TypeError> getattr(): attribute name must be string (error at util.py:*) (glob)
-  mercurial/mail.py: error importing: <TypeError> getattr(): attribute name must be string (error at util.py:*) (glob)
-  mercurial/manifest.py: error importing: <TypeError> getattr(): attribute name must be string (error at util.py:*) (glob)
-  mercurial/match.py: error importing: <TypeError> getattr(): attribute name must be string (error at util.py:*) (glob)
-  mercurial/mdiff.py: error importing: <TypeError> getattr(): attribute name must be string (error at util.py:*) (glob)
-  mercurial/merge.py: error importing: <TypeError> getattr(): attribute name must be string (error at util.py:*) (glob)
-  mercurial/minirst.py: error importing: <TypeError> getattr(): attribute name must be string (error at util.py:*) (glob)
-  mercurial/namespaces.py: error importing: <TypeError> getattr(): attribute name must be string (error at util.py:*) (glob)
-  mercurial/obsolete.py: error importing: <TypeError> getattr(): attribute name must be string (error at util.py:*) (glob)
-  mercurial/patch.py: error importing: <TypeError> getattr(): attribute name must be string (error at util.py:*) (glob)
-  mercurial/pathutil.py: error importing: <TypeError> getattr(): attribute name must be string (error at util.py:*) (glob)
-  mercurial/peer.py: error importing: <TypeError> getattr(): attribute name must be string (error at util.py:*) (glob)
-  mercurial/pushkey.py: error importing: <TypeError> getattr(): attribute name must be string (error at util.py:*) (glob)
-  mercurial/pvec.py: error importing: <TypeError> getattr(): attribute name must be string (error at util.py:*) (glob)
-  mercurial/registrar.py: error importing: <TypeError> getattr(): attribute name must be string (error at util.py:*) (glob)
-  mercurial/repair.py: error importing: <TypeError> getattr(): attribute name must be string (error at util.py:*) (glob)
-  mercurial/repoview.py: error importing: <TypeError> getattr(): attribute name must be string (error at util.py:*) (glob)
-  mercurial/revlog.py: error importing: <TypeError> getattr(): attribute name must be string (error at util.py:*) (glob)
-  mercurial/revset.py: error importing: <TypeError> getattr(): attribute name must be string (error at util.py:*) (glob)
-  mercurial/scmutil.py: error importing: <TypeError> getattr(): attribute name must be string (error at util.py:*) (glob)
-  mercurial/scmwindows.py: error importing: <TypeError> getattr(): attribute name must be string (error at util.py:*) (glob)
-  mercurial/similar.py: error importing: <TypeError> getattr(): attribute name must be string (error at util.py:*) (glob)
-  mercurial/simplemerge.py: error importing: <TypeError> getattr(): attribute name must be string (error at util.py:*) (glob)
-  mercurial/sshpeer.py: error importing: <TypeError> getattr(): attribute name must be string (error at util.py:*) (glob)
-  mercurial/sshserver.py: error importing: <TypeError> getattr(): attribute name must be string (error at util.py:*) (glob)
-  mercurial/sslutil.py: error importing: <TypeError> getattr(): attribute name must be string (error at util.py:*) (glob)
-  mercurial/statichttprepo.py: error importing: <TypeError> getattr(): attribute name must be string (error at util.py:*) (glob)
-  mercurial/store.py: error importing: <TypeError> getattr(): attribute name must be string (error at util.py:*) (glob)
-  mercurial/streamclone.py: error importing: <TypeError> getattr(): attribute name must be string (error at util.py:*) (glob)
-  mercurial/subrepo.py: error importing: <TypeError> getattr(): attribute name must be string (error at util.py:*) (glob)
-  mercurial/tagmerge.py: error importing: <TypeError> getattr(): attribute name must be string (error at util.py:*) (glob)
-  mercurial/tags.py: error importing: <TypeError> getattr(): attribute name must be string (error at util.py:*) (glob)
-  mercurial/templatefilters.py: error importing: <TypeError> getattr(): attribute name must be string (error at util.py:*) (glob)
-  mercurial/templatekw.py: error importing: <TypeError> getattr(): attribute name must be string (error at util.py:*) (glob)
-  mercurial/templater.py: error importing: <TypeError> getattr(): attribute name must be string (error at util.py:*) (glob)
-  mercurial/transaction.py: error importing: <TypeError> getattr(): attribute name must be string (error at util.py:*) (glob)
-  mercurial/ui.py: error importing: <TypeError> getattr(): attribute name must be string (error at util.py:*) (glob)
-  mercurial/unionrepo.py: error importing: <TypeError> getattr(): attribute name must be string (error at util.py:*) (glob)
-  mercurial/url.py: error importing: <TypeError> getattr(): attribute name must be string (error at util.py:*) (glob)
-  mercurial/verify.py: error importing: <TypeError> attribute name must be string, not 'bytes' (error at mdiff.py:*) (glob)
+  mercurial/keepalive.py: error importing: <TypeError> __slots__ items must be strings, not 'bytes' (error at util.py:*) (glob)
+  mercurial/localrepo.py: error importing: <TypeError> __slots__ items must be strings, not 'bytes' (error at util.py:*) (glob)
+  mercurial/lock.py: error importing: <TypeError> __slots__ items must be strings, not 'bytes' (error at util.py:*) (glob)
+  mercurial/mail.py: error importing: <TypeError> __slots__ items must be strings, not 'bytes' (error at util.py:*) (glob)
+  mercurial/manifest.py: error importing: <TypeError> __slots__ items must be strings, not 'bytes' (error at util.py:*) (glob)
+  mercurial/match.py: error importing: <TypeError> __slots__ items must be strings, not 'bytes' (error at util.py:*) (glob)
+  mercurial/mdiff.py: error importing: <TypeError> __slots__ items must be strings, not 'bytes' (error at util.py:*) (glob)
+  mercurial/merge.py: error importing: <TypeError> __slots__ items must be strings, not 'bytes' (error at util.py:*) (glob)
+  mercurial/minirst.py: error importing: <TypeError> __slots__ items must be strings, not 'bytes' (error at util.py:*) (glob)
+  mercurial/namespaces.py: error importing: <TypeError> __slots__ items must be strings, not 'bytes' (error at util.py:*) (glob)
+  mercurial/obsolete.py: error importing: <TypeError> __slots__ items must be strings, not 'bytes' (error at util.py:*) (glob)
+  mercurial/patch.py: error importing: <TypeError> __slots__ items must be strings, not 'bytes' (error at util.py:*) (glob)
+  mercurial/pathutil.py: error importing: <TypeError> __slots__ items must be strings, not 'bytes' (error at util.py:*) (glob)
+  mercurial/peer.py: error importing: <TypeError> __slots__ items must be strings, not 'bytes' (error at util.py:*) (glob)
+  mercurial/profiling.py: error importing: <TypeError> __slots__ items must be strings, not 'bytes' (error at util.py:*) (glob)
+  mercurial/pushkey.py: error importing: <TypeError> __slots__ items must be strings, not 'bytes' (error at util.py:*) (glob)
+  mercurial/pvec.py: error importing: <TypeError> __slots__ items must be strings, not 'bytes' (error at util.py:*) (glob)
+  mercurial/registrar.py: error importing: <TypeError> __slots__ items must be strings, not 'bytes' (error at util.py:*) (glob)
+  mercurial/repair.py: error importing: <TypeError> __slots__ items must be strings, not 'bytes' (error at util.py:*) (glob)
+  mercurial/repoview.py: error importing: <TypeError> __slots__ items must be strings, not 'bytes' (error at util.py:*) (glob)
+  mercurial/revlog.py: error importing: <TypeError> __slots__ items must be strings, not 'bytes' (error at util.py:*) (glob)
+  mercurial/revset.py: error importing: <TypeError> __slots__ items must be strings, not 'bytes' (error at util.py:*) (glob)
+  mercurial/scmutil.py: error importing: <TypeError> __slots__ items must be strings, not 'bytes' (error at util.py:*) (glob)
+  mercurial/scmwindows.py: error importing: <TypeError> __slots__ items must be strings, not 'bytes' (error at util.py:*) (glob)
+  mercurial/similar.py: error importing: <TypeError> __slots__ items must be strings, not 'bytes' (error at util.py:*) (glob)
+  mercurial/simplemerge.py: error importing: <TypeError> __slots__ items must be strings, not 'bytes' (error at util.py:*) (glob)
+  mercurial/sshpeer.py: error importing: <TypeError> __slots__ items must be strings, not 'bytes' (error at util.py:*) (glob)
+  mercurial/sshserver.py: error importing: <TypeError> __slots__ items must be strings, not 'bytes' (error at util.py:*) (glob)
+  mercurial/sslutil.py: error importing: <TypeError> __slots__ items must be strings, not 'bytes' (error at util.py:*) (glob)
+  mercurial/statichttprepo.py: error importing: <TypeError> __slots__ items must be strings, not 'bytes' (error at util.py:*) (glob)
+  mercurial/store.py: error importing: <TypeError> __slots__ items must be strings, not 'bytes' (error at util.py:*) (glob)
+  mercurial/streamclone.py: error importing: <TypeError> __slots__ items must be strings, not 'bytes' (error at util.py:*) (glob)
+  mercurial/subrepo.py: error importing: <TypeError> __slots__ items must be strings, not 'bytes' (error at util.py:*) (glob)
+  mercurial/tagmerge.py: error importing: <TypeError> __slots__ items must be strings, not 'bytes' (error at util.py:*) (glob)
+  mercurial/tags.py: error importing: <TypeError> __slots__ items must be strings, not 'bytes' (error at util.py:*) (glob)
+  mercurial/templatefilters.py: error importing: <TypeError> __slots__ items must be strings, not 'bytes' (error at util.py:*) (glob)
+  mercurial/templatekw.py: error importing: <TypeError> __slots__ items must be strings, not 'bytes' (error at util.py:*) (glob)
+  mercurial/templater.py: error importing: <TypeError> __slots__ items must be strings, not 'bytes' (error at util.py:*) (glob)
+  mercurial/transaction.py: error importing: <TypeError> __slots__ items must be strings, not 'bytes' (error at util.py:*) (glob)
+  mercurial/ui.py: error importing: <TypeError> __slots__ items must be strings, not 'bytes' (error at util.py:*) (glob)
+  mercurial/unionrepo.py: error importing: <TypeError> __slots__ items must be strings, not 'bytes' (error at util.py:*) (glob)
+  mercurial/url.py: error importing: <TypeError> __slots__ items must be strings, not 'bytes' (error at util.py:*) (glob)
+  mercurial/verify.py: error importing module: <TypeError> unorderable types: str() >= tuple() (line *) (glob)
   mercurial/win32.py: error importing module: <ImportError> No module named 'msvcrt' (line *) (glob)
   mercurial/windows.py: error importing module: <ImportError> No module named 'msvcrt' (line *) (glob)
   mercurial/wireproto.py: error importing module: <TypeError> unorderable types: str() >= tuple() (line *) (glob)


More information about the Mercurial-devel mailing list