Skip to content

Commit

Permalink
Restrict the number of errors shown when there are missing stubs (#10579
Browse files Browse the repository at this point in the history
)

When upgrading to mypy 0.900, most projects will see some errors about
missing stubs. Projects with strict settings could see thousands of errors,
since missing stubs will generate many additional Any types.

After 200 errors (only if some of them are about imports) we will now 
only show errors about unresolved imports or missing stubs, so that the 
likely root causes won't be obscured in a high volume of errors.

Fixes #10529.
  • Loading branch information
JukkaL committed Jun 7, 2021
1 parent 56d5b3a commit e056774
Show file tree
Hide file tree
Showing 7 changed files with 165 additions and 4 deletions.
8 changes: 8 additions & 0 deletions docs/source/command_line.rst
Original file line number Diff line number Diff line change
Expand Up @@ -687,6 +687,14 @@ in error messages.

Show absolute paths to files.

.. option:: --soft-error-limit N

This flag will adjust the limit after which mypy will (sometimes)
disable reporting most additional errors. The limit only applies
if it seems likely that most of the remaining errors will not be
useful or they may be overly noisy. If ``N`` is negative, there is
no limit. The default limit is 200.


.. _incremental:

Expand Down
3 changes: 2 additions & 1 deletion mypy/build.py
Original file line number Diff line number Diff line change
Expand Up @@ -224,7 +224,8 @@ def _build(sources: List[BuildSource],
lambda path: read_py_file(path, cached_read, options.python_version),
options.show_absolute_path,
options.enabled_error_codes,
options.disabled_error_codes)
options.disabled_error_codes,
options.many_errors_threshold)
plugin, snapshot = load_plugins(options, errors, stdout, extra_plugins)

# Add catch-all .gitignore to cache dir if we created it
Expand Down
4 changes: 4 additions & 0 deletions mypy/defaults.py
Original file line number Diff line number Diff line change
Expand Up @@ -30,3 +30,7 @@
'html',
'txt',
'lineprecision'] # type: Final

# Threshold after which we sometimes filter out most errors to avoid very
# verbose output
MANY_ERRORS_THRESHOLD = 200 # type: Final
66 changes: 63 additions & 3 deletions mypy/errors.py
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@
from mypy.scope import Scope
from mypy.options import Options
from mypy.version import __version__ as mypy_version
from mypy.errorcodes import ErrorCode
from mypy.errorcodes import ErrorCode, IMPORT
from mypy import errorcodes as codes
from mypy.util import DEFAULT_SOURCE_OFFSET, is_typeshed_file

Expand Down Expand Up @@ -65,6 +65,10 @@ class ErrorInfo:
# Fine-grained incremental target where this was reported
target = None # type: Optional[str]

# If True, don't show this message in output, but still record the error (needed
# by mypy daemon)
hidden = False

def __init__(self,
import_ctx: List[Tuple[str, int]],
file: str,
Expand Down Expand Up @@ -158,6 +162,10 @@ class Errors:
target_module = None # type: Optional[str]
scope = None # type: Optional[Scope]

# Have we seen an import-related error so far? If yes, we filter out other messages
# in some cases to avoid reporting huge numbers of errors.
seen_import_error = False

def __init__(self,
show_error_context: bool = False,
show_column_numbers: bool = False,
Expand All @@ -166,7 +174,8 @@ def __init__(self,
read_source: Optional[Callable[[str], Optional[List[str]]]] = None,
show_absolute_path: bool = False,
enabled_error_codes: Optional[Set[ErrorCode]] = None,
disabled_error_codes: Optional[Set[ErrorCode]] = None) -> None:
disabled_error_codes: Optional[Set[ErrorCode]] = None,
many_errors_threshold: int = -1) -> None:
self.show_error_context = show_error_context
self.show_column_numbers = show_column_numbers
self.show_error_codes = show_error_codes
Expand All @@ -176,6 +185,7 @@ def __init__(self,
self.read_source = read_source
self.enabled_error_codes = enabled_error_codes or set()
self.disabled_error_codes = disabled_error_codes or set()
self.many_errors_threshold = many_errors_threshold
self.initialize()

def initialize(self) -> None:
Expand All @@ -189,6 +199,7 @@ def initialize(self) -> None:
self.only_once_messages = set()
self.scope = None
self.target_module = None
self.seen_import_error = False

def reset(self) -> None:
self.initialize()
Expand All @@ -201,12 +212,14 @@ def copy(self) -> 'Errors':
self.read_source,
self.show_absolute_path,
self.enabled_error_codes,
self.disabled_error_codes)
self.disabled_error_codes,
self.many_errors_threshold)
new.file = self.file
new.import_ctx = self.import_ctx[:]
new.function_or_member = self.function_or_member[:]
new.target_module = self.target_module
new.scope = self.scope
new.seen_import_error = self.seen_import_error
return new

def total_errors(self) -> int:
Expand Down Expand Up @@ -330,6 +343,8 @@ def _add_error_info(self, file: str, info: ErrorInfo) -> None:
if file not in self.error_info_map:
self.error_info_map[file] = []
self.error_info_map[file].append(info)
if info.code is IMPORT:
self.seen_import_error = True

def add_error_info(self, info: ErrorInfo) -> None:
file, line, end_line = info.origin
Expand All @@ -354,8 +369,52 @@ def add_error_info(self, info: ErrorInfo) -> None:
if info.message in self.only_once_messages:
return
self.only_once_messages.add(info.message)
if self.seen_import_error and info.code is not IMPORT and self.has_many_errors():
# Missing stubs can easily cause thousands of errors about
# Any types, especially when upgrading to mypy 0.900,
# which no longer bundles third-party library stubs. Avoid
# showing too many errors to make it easier to see
# import-related errors.
info.hidden = True
self.report_hidden_errors(info)
self._add_error_info(file, info)

def has_many_errors(self) -> bool:
if self.many_errors_threshold < 0:
return False
if len(self.error_info_map) >= self.many_errors_threshold:
return True
if sum(len(errors)
for errors in self.error_info_map.values()) >= self.many_errors_threshold:
return True
return False

def report_hidden_errors(self, info: ErrorInfo) -> None:
message = (
'(Skipping most remaining errors due to unresolved imports or missing stubs; ' +
'fix these first)'
)
if message in self.only_once_messages:
return
self.only_once_messages.add(message)
new_info = ErrorInfo(
import_ctx=info.import_ctx,
file=info.file,
module=info.module,
typ=None,
function_or_member=None,
line=info.line,
column=info.line,
severity='note',
message=message,
code=None,
blocker=False,
only_once=True,
origin=info.origin,
target=info.target,
)
self._add_error_info(info.origin[0], new_info)

def is_ignored_error(self, line: int, info: ErrorInfo, ignores: Dict[int, List[str]]) -> bool:
if info.blocker:
# Blocking errors can never be ignored
Expand Down Expand Up @@ -453,6 +512,7 @@ def format_messages(self, error_info: List[ErrorInfo],
severity 'error').
"""
a = [] # type: List[str]
error_info = [info for info in error_info if not info.hidden]
errors = self.render_messages(self.sort_messages(error_info))
errors = self.remove_duplicates(errors)
for file, line, column, severity, message, code in errors:
Expand Down
2 changes: 2 additions & 0 deletions mypy/main.py
Original file line number Diff line number Diff line change
Expand Up @@ -663,6 +663,8 @@ def add_invertible_flag(flag: str,
add_invertible_flag('--show-absolute-path', default=False,
help="Show absolute paths to files",
group=error_group)
error_group.add_argument('--soft-error-limit', default=defaults.MANY_ERRORS_THRESHOLD,
type=int, dest="many_errors_threshold", help=argparse.SUPPRESS)

incremental_group = parser.add_argument_group(
title='Incremental mode',
Expand Down
4 changes: 4 additions & 0 deletions mypy/options.py
Original file line number Diff line number Diff line change
Expand Up @@ -295,6 +295,10 @@ def __init__(self) -> None:
self.show_absolute_path = False # type: bool
# Install missing stub packages if True
self.install_types = False
# When we encounter errors that may cause many additional errors,
# skip most errors after this many messages have been reported.
# -1 means unlimited.
self.many_errors_threshold = defaults.MANY_ERRORS_THRESHOLD

# To avoid breaking plugin compatibility, keep providing new_semantic_analyzer
@property
Expand Down
82 changes: 82 additions & 0 deletions test-data/unit/check-modules.test
Original file line number Diff line number Diff line change
Expand Up @@ -2982,3 +2982,85 @@ T = TypeVar("T")
class F(M):
x: C
class C: ...

[case testLimitLegacyStubErrorVolume]
# flags: --disallow-any-expr --soft-error-limit=5
import certifi # E: Cannot find implementation or library stub for module named "certifi" \
# N: See https://mypy.readthedocs.io/en/stable/running_mypy.html#missing-imports
certifi.x # E: Expression has type "Any"
certifi.x # E: Expression has type "Any"
certifi.x # E: Expression has type "Any"
certifi.x # N: (Skipping most remaining errors due to unresolved imports or missing stubs; fix these first)
certifi.x
certifi.x
certifi.x
certifi.x

[case testDoNotLimitErrorVolumeIfNotImportErrors]
# flags: --disallow-any-expr --soft-error-limit=5
def f(): pass
certifi = f() # E: Expression has type "Any"
1() # E: "int" not callable
certifi.x # E: Expression has type "Any"
certifi.x # E: Expression has type "Any"
certifi.x # E: Expression has type "Any"
certifi.x # E: Expression has type "Any"
certifi.x # E: Expression has type "Any"
certifi.x # E: Expression has type "Any"
certifi.x # E: Expression has type "Any"
certifi.x # E: Expression has type "Any"
1() # E: "int" not callable


[case testDoNotLimitImportErrorVolume]
# flags: --disallow-any-expr --soft-error-limit=3
import xyz1 # E: Cannot find implementation or library stub for module named "xyz1" \
# N: See https://mypy.readthedocs.io/en/stable/running_mypy.html#missing-imports
import xyz2 # E: Cannot find implementation or library stub for module named "xyz2"
import xyz3 # E: Cannot find implementation or library stub for module named "xyz3"
import xyz4 # E: Cannot find implementation or library stub for module named "xyz4"

[case testUnlimitedStubErrorVolume]
# flags: --disallow-any-expr --soft-error-limit=-1
import certifi # E: Cannot find implementation or library stub for module named "certifi" \
# N: See https://mypy.readthedocs.io/en/stable/running_mypy.html#missing-imports
certifi.x # E: Expression has type "Any"
certifi.x # E: Expression has type "Any"
certifi.x # E: Expression has type "Any"
certifi.x # E: Expression has type "Any"
certifi.x # E: Expression has type "Any"
certifi.x # E: Expression has type "Any"
certifi.x # E: Expression has type "Any"
certifi.x # E: Expression has type "Any"
certifi.x # E: Expression has type "Any"
certifi.x # E: Expression has type "Any"
certifi.x # E: Expression has type "Any"
certifi.x # E: Expression has type "Any"
certifi.x # E: Expression has type "Any"
certifi.x # E: Expression has type "Any"
certifi.x # E: Expression has type "Any"
certifi.x # E: Expression has type "Any"
certifi.x # E: Expression has type "Any"
certifi.x # E: Expression has type "Any"
certifi.x # E: Expression has type "Any"
certifi.x # E: Expression has type "Any"
certifi.x # E: Expression has type "Any"
certifi.x # E: Expression has type "Any"
certifi.x # E: Expression has type "Any"
certifi.x # E: Expression has type "Any"
certifi.x # E: Expression has type "Any"
certifi.x # E: Expression has type "Any"
certifi.x # E: Expression has type "Any"
certifi.x # E: Expression has type "Any"
certifi.x # E: Expression has type "Any"
certifi.x # E: Expression has type "Any"
certifi.x # E: Expression has type "Any"
certifi.x # E: Expression has type "Any"
certifi.x # E: Expression has type "Any"
certifi.x # E: Expression has type "Any"
certifi.x # E: Expression has type "Any"
certifi.x # E: Expression has type "Any"
certifi.x # E: Expression has type "Any"
certifi.x # E: Expression has type "Any"
certifi.x # E: Expression has type "Any"
certifi.x # E: Expression has type "Any"

0 comments on commit e056774

Please sign in to comment.