[3.12] gh-105324: Fix tokenize module main function for stdin (GH-105325

) (#105330)
python · Jun 5, 2023 · 3aa3be4 · 3aa3be4
1 parent ab660e9
commit 3aa3be4
Show file tree

Hide file tree

Showing 2 changed files with 3 additions and 2 deletions.
diff --git a/Lib/tokenize.py b/Lib/tokenize.py
@@ -490,8 +490,7 @@ def error(message, filename=None, location=None):
         else:
             filename = "<stdin>"
             tokens = _generate_tokens_from_c_tokenizer(
-                (x.encode('utf-8') for x in iter(sys.stdin.readline, "")
-            ), "utf-8", extra_tokens=True)
+                sys.stdin.readline, extra_tokens=True)
 
 
         # Output the tokenization

diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-06-05-17-35-50.gh-issue-105324.BqhiJJ.rst b/Misc/NEWS.d/next/Core and Builtins/2023-06-05-17-35-50.gh-issue-105324.BqhiJJ.rst
@@ -0,0 +1,2 @@
+Fix the main function of the :mod:`tokenize` module when reading from
+``sys.stdin``. Patch by Pablo Galindo