Skip to content

Commit

Permalink
pythongh-104719: IDLE - test existence of all tokenize references. (p…
Browse files Browse the repository at this point in the history
…ythonGH-104767)

Class editor.IndentSearcher contains all editor references to tokenize module.
Module io tokenize reference cover those other modules.

(cherry picked from commit e561c09)

Co-authored-by: Terry Jan Reedy <tjreedy@udel.edu>
Co-authored-by: Jelle Zijlstra <jelle.zijlstra@gmail.com>
  • Loading branch information
2 people authored and miss-islington committed May 24, 2023
1 parent b719dd8 commit 1061d30
Show file tree
Hide file tree
Showing 5 changed files with 54 additions and 12 deletions.
3 changes: 3 additions & 0 deletions Lib/idlelib/NEWS.txt
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,9 @@ Released on 2023-10-02
=========================


gh-104719: Remove IDLE's modification of tokenize.tabsize and test
other uses of tokenize data and methods.

gh-104499: Fix completions for Tk Aqua 8.7 (currently blank).

gh-104486: Make About print both tcl and tk versions if different,
Expand Down
19 changes: 10 additions & 9 deletions Lib/idlelib/editor.py
Original file line number Diff line number Diff line change
Expand Up @@ -1571,7 +1571,7 @@ def reindent_to(self, column):
# blocks are found).

def guess_indent(self):
opener, indented = IndentSearcher(self.text, self.tabwidth).run()
opener, indented = IndentSearcher(self.text).run()
if opener and indented:
raw, indentsmall = get_line_indent(opener, self.tabwidth)
raw, indentlarge = get_line_indent(indented, self.tabwidth)
Expand Down Expand Up @@ -1609,15 +1609,10 @@ def get_line_indent(line, tabwidth):


class IndentSearcher:
"Manage initial indent guess, returned by run method."

# .run() chews over the Text widget, looking for a block opener
# and the stmt following it. Returns a pair,
# (line containing block opener, line containing stmt)
# Either or both may be None.

def __init__(self, text, tabwidth):
def __init__(self, text):
self.text = text
self.tabwidth = tabwidth
self.i = self.finished = 0
self.blkopenline = self.indentedline = None

Expand All @@ -1633,7 +1628,8 @@ def readline(self):
def tokeneater(self, type, token, start, end, line,
INDENT=tokenize.INDENT,
NAME=tokenize.NAME,
OPENERS=('class', 'def', 'for', 'if', 'try', 'while')):
OPENERS=('class', 'def', 'for', 'if', 'match', 'try',
'while', 'with')):
if self.finished:
pass
elif type == NAME and token in OPENERS:
Expand All @@ -1643,6 +1639,10 @@ def tokeneater(self, type, token, start, end, line,
self.finished = 1

def run(self):
"""Return 2 lines containing block opener and and indent.
Either the indent line or both may be None.
"""
try:
tokens = tokenize.generate_tokens(self.readline)
for token in tokens:
Expand All @@ -1654,6 +1654,7 @@ def run(self):

### end autoindent code ###


def prepstr(s):
"""Extract the underscore from a string.
Expand Down
36 changes: 33 additions & 3 deletions Lib/idlelib/idle_test/test_editor.py
Original file line number Diff line number Diff line change
@@ -1,10 +1,10 @@
"Test editor, coverage 35%."
"Test editor, coverage 53%."

from idlelib import editor
import unittest
from collections import namedtuple
from test.support import requires
from tkinter import Tk
from tkinter import Tk, Text

Editor = editor.EditorWindow

Expand All @@ -31,7 +31,7 @@ def test_init(self):
e._close()


class TestGetLineIndent(unittest.TestCase):
class GetLineIndentTest(unittest.TestCase):
def test_empty_lines(self):
for tabwidth in [1, 2, 4, 6, 8]:
for line in ['', '\n']:
Expand Down Expand Up @@ -181,6 +181,36 @@ def test_indent_and_newline_event(self):
eq(get('1.0', 'end'), ' def f1(self, a,\n \n return a + b\n')


class IndentSearcherTest(unittest.TestCase):

@classmethod
def setUpClass(cls):
requires('gui')
cls.root = Tk()
cls.root.withdraw()
cls.text = Text(cls.root)

@classmethod
def tearDownClass(cls):
cls.root.destroy()
del cls.root

def test_searcher(self):
text = self.text
searcher = (self.text)
test_info = (# text, (block, indent))
("", (None, None)),
("[1,", (None, None)), # TokenError
("if 1:\n", ('if 1:\n', None)),
("if 1:\n 2\n 3\n", ('if 1:\n', ' 2\n')),
)
for code, expected_pair in test_info:
with self.subTest(code=code):
insert(text, code)
actual_pair = editor.IndentSearcher(text).run()
self.assertEqual(actual_pair, expected_pair)


class RMenuTest(unittest.TestCase):

@classmethod
Expand Down
6 changes: 6 additions & 0 deletions Lib/idlelib/idle_test/test_iomenu.py
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,12 @@
from idlelib import util
from idlelib.idle_test.mock_idle import Func

# Fail if either tokenize.open and t.detect_encoding does not exist.
# These are used in loadfile and encode.
# Also used in pyshell.MI.execfile and runscript.tabnanny.
from tokenize import open, detect_encoding
# Remove when we have proper tests that use both.


class IOBindingTest(unittest.TestCase):

Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
Remove IDLE's modification of tokenize.tabsize and test other uses of
tokenize data and methods.

0 comments on commit 1061d30

Please sign in to comment.