[PATCH 4 of 4] py3: use bytestr wrapper in revsetlang.tokenize()

[Yuya Nishihara]

# HG changeset patch
# User Yuya Nishihara <yuya at tcha.org>
# Date 1489667781 -32400
#      Thu Mar 16 21:36:21 2017 +0900
# Node ID d21fd508342b1d43ebefaac65f1d6e2e5ca60704
# Parent  649866574c86bae35ab8143c50af72bee7fef810
py3: use bytestr wrapper in revsetlang.tokenize()

This backs out 77270ec0cdd9 and wraps program by bytestr() instead.

diff --git a/mercurial/revsetlang.py b/mercurial/revsetlang.py
--- a/mercurial/revsetlang.py
+++ b/mercurial/revsetlang.py
@@ -78,6 +78,7 @@ def tokenize(program, lookup=None, symin
     [('symbol', '@', 0), ('::', None, 1), ('end', None, 3)]
 
     '''
+    program = pycompat.bytestr(program)
     if syminitletters is None:
         syminitletters = _syminitletters
     if symletters is None:
@@ -100,7 +101,7 @@ def tokenize(program, lookup=None, symin
 
     pos, l = 0, len(program)
     while pos < l:
-        c = program[pos:pos + 1]
+        c = program[pos]
         if c.isspace(): # skip inter-token whitespace
             pass
         elif c == ':' and program[pos:pos + 2] == '::': # look ahead carefully
@@ -118,14 +119,14 @@ def tokenize(program, lookup=None, symin
               program[pos:pos + 2] in ("r'", 'r"')): # handle quoted strings
             if c == 'r':
                 pos += 1
-                c = program[pos:pos + 1]
+                c = program[pos]
                 decode = lambda x: x
             else:
                 decode = parser.unescapestr
             pos += 1
             s = pos
             while pos < l: # find closing quote
-                d = program[pos:pos + 1]
+                d = program[pos]
                 if d == '\\': # skip over escaped characters
                     pos += 2
                     continue
@@ -140,11 +141,10 @@ def tokenize(program, lookup=None, symin
             s = pos
             pos += 1
             while pos < l: # find end of symbol
-                d = program[pos:pos + 1]
+                d = program[pos]
                 if d not in symletters:
                     break
-                if (d == '.'
-                    and program[pos - 1:pos] == '.'): # special case for ..
+                if d == '.' and program[pos - 1] == '.': # special case for ..
                     pos -= 1
                     break
                 pos += 1