Merge remote-tracking branch 'upstream/master' into conditional-overloads

Author: cdce8p

.github/workflows/test.yml

@@ -75,11 +75,16 @@ jobs:
           toxenv: py36
           tox_extra_args: "-n 2 mypyc/test/test_run.py mypyc/test/test_external.py"
           debug_build: true
-        - name: Type check our own code
+        - name: Type check our own code (py37-ubuntu)
           python: '3.7'
           arch: x64
           os: ubuntu-latest
           toxenv: type
+        - name: Type check our own code (py37-windows-64)
+          python: '3.7'
+          arch: x64
+          os: windows-latest
+          toxenv: type
         - name: Code style with flake8
           python: '3.7'
           arch: x64

docs/source/command_line.rst

@@ -524,7 +524,16 @@ of the above sections.
            # 'items' has type list[str]
            items = [item.split() for item in items]
            # 'items' now has type list[list[str]]
-           ...
+
+    The variable must be used before it can be redefined:
+
+    .. code-block:: python
+
+        def process(items: list[str]) -> None:
+           items = "mypy"  # invalid redefinition to str because the variable hasn't been used yet
+           print(items)
+           items = "100"  # valid, items now has type str
+           items = int(items)  # valid, items now has type int
 
 .. option:: --local-partial-types
 

docs/source/config_file.rst

@@ -237,8 +237,9 @@ section of the command line docs.
 
           [tool.mypy]
           exclude = [
-            "^file1\\.py$",  # TOML's double-quoted strings require escaping backslashes
-            '^file2\.py$',  # but TOML's single-quoted strings do not
+              "^one\.py$",  # TOML's double-quoted strings require escaping backslashes
+              'two\.pyi$',  # but TOML's single-quoted strings do not
+              '^three\.',
           ]
 
        A single, multi-line string:
@@ -247,9 +248,10 @@ section of the command line docs.
 
           [tool.mypy]
           exclude = '''(?x)(
-              ^file1\.py$
-              |^file2\.py$,
-          )'''
+              ^one\.py$    # files named "one.py"
+              | two\.pyi$  # or files ending with "two.pyi"
+              | ^three\.   # or files starting with "three."
+          )'''  # TOML's single-quoted strings do not require escaping backslashes
 
        See :ref:`using-a-pyproject-toml`.
 
@@ -613,6 +615,24 @@ section of the command line docs.
 
     Allows variables to be redefined with an arbitrary type, as long as the redefinition
     is in the same block and nesting level as the original definition.
+    Example where this can be useful:
+
+    .. code-block:: python
+
+       def process(items: list[str]) -> None:
+           # 'items' has type list[str]
+           items = [item.split() for item in items]
+           # 'items' now has type list[list[str]]
+
+    The variable must be used before it can be redefined:
+
+    .. code-block:: python
+
+        def process(items: list[str]) -> None:
+           items = "mypy"  # invalid redefinition to str because the variable hasn't been used yet
+           print(items)
+           items = "100"  # valid, items now has type str
+           items = int(items)  # valid, items now has type int
 
 .. confval:: local_partial_types
 

docs/source/generics.rst

@@ -673,6 +673,10 @@ protocols mostly follow the normal rules for generic classes. Example:
    y: Box[int] = ...
    x = y  # Error -- Box is invariant
 
+Per :pep:`PEP 544: Generic protocols <544#generic-protocols>`, ``class
+ClassName(Protocol[T])`` is allowed as a shorthand for ``class
+ClassName(Protocol, Generic[T])``.
+
 The main difference between generic protocols and ordinary generic
 classes is that mypy checks that the declared variances of generic
 type variables in a protocol match how they are used in the protocol

mypy/checker.py

@@ -24,9 +24,9 @@
     Context, Decorator, PrintStmt, BreakStmt, PassStmt, ContinueStmt,
     ComparisonExpr, StarExpr, EllipsisExpr, RefExpr, PromoteExpr,
     Import, ImportFrom, ImportAll, ImportBase, TypeAlias,
-    ARG_POS, ARG_STAR, LITERAL_TYPE, LDEF, MDEF, GDEF,
+    ARG_POS, ARG_STAR, ARG_NAMED, LITERAL_TYPE, LDEF, MDEF, GDEF,
     CONTRAVARIANT, COVARIANT, INVARIANT, TypeVarExpr, AssignmentExpr,
-    is_final_node, ARG_NAMED, MatchStmt)
+    is_final_node, MatchStmt)
 from mypy import nodes
 from mypy import operators
 from mypy.literals import literal, literal_hash, Key
@@ -86,7 +86,7 @@
 from mypy import state, errorcodes as codes
 from mypy.traverser import has_return_statement, all_return_statements
 from mypy.errorcodes import ErrorCode
-from mypy.util import is_typeshed_file
+from mypy.util import is_typeshed_file, is_dunder, is_sunder
 
 T = TypeVar('T')
 
@@ -1833,6 +1833,10 @@ def visit_class_def(self, defn: ClassDef) -> None:
                 # that completely swap out the type.  (e.g. Callable[[Type[A]], Type[B]])
         if typ.is_protocol and typ.defn.type_vars:
             self.check_protocol_variance(defn)
+        if not defn.has_incompatible_baseclass and defn.info.is_enum:
+            for base in defn.info.mro[1:-1]:  # we don't need self and `object`
+                if base.is_enum and base.fullname not in ENUM_BASES:
+                    self.check_final_enum(defn, base)
 
     def check_final_deletable(self, typ: TypeInfo) -> None:
         # These checks are only for mypyc. Only perform some checks that are easier
@@ -1890,6 +1894,43 @@ def check_init_subclass(self, defn: ClassDef) -> None:
             # all other bases have already been checked.
             break
 
+    def check_final_enum(self, defn: ClassDef, base: TypeInfo) -> None:
+        for sym in base.names.values():
+            if self.is_final_enum_value(sym):
+                self.fail(
+                    'Cannot extend enum with existing members: "{}"'.format(
+                        base.name,
+                    ),
+                    defn,
+                )
+                break
+
+    def is_final_enum_value(self, sym: SymbolTableNode) -> bool:
+        if isinstance(sym.node, (FuncBase, Decorator)):
+            return False  # A method is fine
+        if not isinstance(sym.node, Var):
+            return True  # Can be a class or anything else
+
+        # Now, only `Var` is left, we need to check:
+        # 1. Private name like in `__prop = 1`
+        # 2. Dunder name like `__hash__ = some_hasher`
+        # 3. Sunder name like `_order_ = 'a, b, c'`
+        # 4. If it is a method / descriptor like in `method = classmethod(func)`
+        if (
+            is_private(sym.node.name)
+            or is_dunder(sym.node.name)
+            or is_sunder(sym.node.name)
+            # TODO: make sure that `x = @class/staticmethod(func)`
+            # and `x = property(prop)` both work correctly.
+            # Now they are incorrectly counted as enum members.
+            or isinstance(get_proper_type(sym.node.type), FunctionLike)
+        ):
+            return False
+
+        if self.is_stub or sym.node.has_explicit_value:
+            return True
+        return False
+
     def check_protocol_variance(self, defn: ClassDef) -> None:
         """Check that protocol definition is compatible with declared
         variances of type variables.

mypy/checkexpr.py

@@ -4,7 +4,7 @@
 from contextlib import contextmanager
 import itertools
 from typing import (
-    Any, cast, Dict, Set, List, Tuple, Callable, Union, Optional, Sequence, Iterator
+    cast, Dict, Set, List, Tuple, Callable, Union, Optional, Sequence, Iterator
 )
 from typing_extensions import ClassVar, Final, overload, TypeAlias as _TypeAlias
 
@@ -378,6 +378,8 @@ def visit_call_expr_inner(self, e: CallExpr, allow_none_return: bool = False) ->
         if isinstance(e.callee, MemberExpr) and e.callee.name == 'format':
             self.check_str_format_call(e)
         ret_type = get_proper_type(ret_type)
+        if isinstance(ret_type, UnionType):
+            ret_type = make_simplified_union(ret_type.items)
         if isinstance(ret_type, UninhabitedType) and not ret_type.ambiguous:
             self.chk.binder.unreachable()
         # Warn on calls to functions that always return None. The check
@@ -1605,8 +1607,7 @@ def check_overload_call(self,
                 # Record if we succeeded. Next we need to see if maybe normal procedure
                 # gives a narrower type.
                 if unioned_return:
-                    # TODO: fix signature of zip() in typeshed.
-                    returns, inferred_types = cast(Any, zip)(*unioned_return)
+                    returns, inferred_types = zip(*unioned_return)
                     # Note that we use `combine_function_signatures` instead of just returning
                     # a union of inferred callables because for example a call
                     # Union[int -> int, str -> str](Union[int, str]) is invalid and

mypy/erasetype.py

@@ -1,10 +1,10 @@
-from typing import Optional, Container, Callable
+from typing import Optional, Container, Callable, List, Dict, cast
 
 from mypy.types import (
     Type, TypeVisitor, UnboundType, AnyType, NoneType, TypeVarId, Instance, TypeVarType,
     CallableType, TupleType, TypedDictType, UnionType, Overloaded, ErasedType, PartialType,
     DeletedType, TypeTranslator, UninhabitedType, TypeType, TypeOfAny, LiteralType, ProperType,
-    get_proper_type, TypeAliasType, ParamSpecType
+    get_proper_type, get_proper_types, TypeAliasType, ParamSpecType
 )
 from mypy.nodes import ARG_STAR, ARG_STAR2
 
@@ -161,3 +161,34 @@ def visit_type_alias_type(self, t: TypeAliasType) -> Type:
         # Type aliases can't contain literal values, because they are
         # always constructed as explicit types.
         return t
+
+    def visit_union_type(self, t: UnionType) -> Type:
+        new = cast(UnionType, super().visit_union_type(t))
+        # Erasure can result in many duplicate items; merge them.
+        # Call make_simplified_union only on lists of instance types
+        # that all have the same fullname, to avoid simplifying too
+        # much.
+        instances = [item for item in new.items
+                     if isinstance(get_proper_type(item), Instance)]
+        # Avoid merge in simple cases such as optional types.
+        if len(instances) > 1:
+            instances_by_name: Dict[str, List[Instance]] = {}
+            new_items = get_proper_types(new.items)
+            for item in new_items:
+                if isinstance(item, Instance) and not item.args:
+                    instances_by_name.setdefault(item.type.fullname, []).append(item)
+            merged: List[Type] = []
+            for item in new_items:
+                if isinstance(item, Instance) and not item.args:
+                    types = instances_by_name.get(item.type.fullname)
+                    if types is not None:
+                        if len(types) == 1:
+                            merged.append(item)
+                        else:
+                            from mypy.typeops import make_simplified_union
+                            merged.append(make_simplified_union(types))
+                            del instances_by_name[item.type.fullname]
+                else:
+                    merged.append(item)
+            return UnionType.make_union(merged)
+        return new

mypy/errors.py

@@ -16,8 +16,13 @@
 from mypy.util import DEFAULT_SOURCE_OFFSET, is_typeshed_file
 
 T = TypeVar("T")
+
 allowed_duplicates: Final = ["@overload", "Got:", "Expected:"]
 
+# Keep track of the original error code when the error code of a message is changed.
+# This is used to give notes about out-of-date "type: ignore" comments.
+original_error_codes: Final = {codes.LITERAL_REQ: codes.MISC}
+
 
 class ErrorInfo:
     """Representation of a single error message."""
@@ -388,6 +393,24 @@ def add_error_info(self, info: ErrorInfo) -> None:
             info.hidden = True
             self.report_hidden_errors(info)
         self._add_error_info(file, info)
+        ignored_codes = self.ignored_lines.get(file, {}).get(info.line, [])
+        if ignored_codes and info.code:
+            # Something is ignored on the line, but not this error, so maybe the error
+            # code is incorrect.
+            msg = f'Error code "{info.code.code}" not covered by "type: ignore" comment'
+            if info.code in original_error_codes:
+                # If there seems to be a "type: ignore" with a stale error
+                # code, report a more specific note.
+                old_code = original_error_codes[info.code].code
+                if old_code in ignored_codes:
+                    msg = (f'Error code changed to {info.code.code}; "type: ignore" comment ' +
+                           'may be out of date')
+            note = ErrorInfo(
+                info.import_ctx, info.file, info.module, info.type, info.function_or_member,
+                info.line, info.column, 'note', msg,
+                code=None, blocker=False, only_once=False, allow_dups=False
+            )
+            self._add_error_info(file, note)
 
     def has_many_errors(self) -> bool:
         if self.many_errors_threshold < 0:

mypy/fastparse.py

@@ -123,6 +123,7 @@ def ast3_parse(source: Union[str, bytes], filename: str, mode: str,
         MatchClass = ast3.MatchClass
         MatchAs = ast3.MatchAs
         MatchOr = ast3.MatchOr
+        AstNode = Union[ast3.expr, ast3.stmt, ast3.pattern, ast3.ExceptHandler]
     else:
         Match = Any
         MatchValue = Any
@@ -133,6 +134,7 @@ def ast3_parse(source: Union[str, bytes], filename: str, mode: str,
         MatchClass = Any
         MatchAs = Any
         MatchOr = Any
+        AstNode = Union[ast3.expr, ast3.stmt, ast3.ExceptHandler]
 except ImportError:
     try:
         from typed_ast import ast35  # type: ignore[attr-defined]  # noqa: F401
@@ -357,7 +359,7 @@ def visit(self, node: Optional[AST]) -> Any:
             self.visitor_cache[typeobj] = visitor
         return visitor(node)
 
-    def set_line(self, node: N, n: Union[ast3.expr, ast3.stmt, ast3.ExceptHandler]) -> N:
+    def set_line(self, node: N, n: AstNode) -> N:
         node.line = n.lineno
         node.column = n.col_offset
         node.end_line = getattr(n, "end_lineno", None) if isinstance(n, ast3.expr) else None

mypy/fscache.py

@@ -143,16 +143,13 @@ def _fake_init(self, path: str) -> os.stat_result:
         assert not os.path.exists(path), path  # Not cached!
         dirname = os.path.normpath(dirname)
         st = self.stat(dirname)  # May raise OSError
-        # Get stat result as a sequence so we can modify it.
-        # (Alas, typeshed's os.stat_result is not a sequence yet.)
-        tpl = tuple(st)  # type: ignore[arg-type, var-annotated]
-        seq: List[float] = list(tpl)
+        # Get stat result as a list so we can modify it.
+        seq: List[float] = list(st)
         seq[stat.ST_MODE] = stat.S_IFREG | 0o444
         seq[stat.ST_INO] = 1
         seq[stat.ST_NLINK] = 1
         seq[stat.ST_SIZE] = 0
-        tpl = tuple(seq)
-        st = os.stat_result(tpl)
+        st = os.stat_result(seq)
         self.stat_cache[path] = st
         # Make listdir() and read() also pretend this file exists.
         self.fake_package_cache.add(dirname)

mypy/ipc.py

@@ -54,9 +54,6 @@ def read(self, size: int = 100000) -> bytes:
         if sys.platform == 'win32':
             while True:
                 ov, err = _winapi.ReadFile(self.connection, size, overlapped=True)
-                # TODO: remove once typeshed supports Literal types
-                assert isinstance(ov, _winapi.Overlapped)
-                assert isinstance(err, int)
                 try:
                     if err == _winapi.ERROR_IO_PENDING:
                         timeout = int(self.timeout * 1000) if self.timeout else _winapi.INFINITE

mypy/join.py

@@ -175,15 +175,15 @@ def join_types(s: Type, t: Type, instance_joiner: Optional[InstanceJoiner] = Non
         s = mypy.typeops.true_or_false(s)
         t = mypy.typeops.true_or_false(t)
 
+    if isinstance(s, UnionType) and not isinstance(t, UnionType):
+        s, t = t, s
+
     if isinstance(s, AnyType):
         return s
 
     if isinstance(s, ErasedType):
         return t
 
-    if isinstance(s, UnionType) and not isinstance(t, UnionType):
-        s, t = t, s
-
     if isinstance(s, NoneType) and not isinstance(t, NoneType):
         s, t = t, s
 

mypy/meet.py

@@ -508,15 +508,14 @@ def visit_param_spec(self, t: ParamSpecType) -> ProperType:
 
     def visit_instance(self, t: Instance) -> ProperType:
         if isinstance(self.s, Instance):
-            si = self.s
-            if t.type == si.type:
+            if t.type == self.s.type:
                 if is_subtype(t, self.s) or is_subtype(self.s, t):
                     # Combine type arguments. We could have used join below
                     # equivalently.
                     args: List[Type] = []
                     # N.B: We use zip instead of indexing because the lengths might have
                     # mismatches during daemon reprocessing.
-                    for ta, sia in zip(t.args, si.args):
+                    for ta, sia in zip(t.args, self.s.args):
                         args.append(self.meet(ta, sia))
                     return Instance(t.type, args)
                 else: