Merge pull request #49 from python/master

Sync Fork from Upstream Repo

Author: sthagen

MANIFEST.in

@@ -29,6 +29,7 @@ graft mypyc/doc
 
 # files necessary for testing sdist
 include mypy-requirements.txt
+include build-requirements.txt
 include test-requirements.txt
 include mypy_self_check.ini
 prune misc

build-requirements.txt

@@ -0,0 +1,4 @@
+-r mypy-requirements.txt
+types-typed-ast>=1.4.0,<1.5.0
+types-toml>=0.0
+types-enum34>=0.0; python_version == '3.5'

docs/source/command_line.rst

@@ -49,6 +49,30 @@ for full details, see :ref:`running-mypy`.
     Asks mypy to type check the provided string as a program.
 
 
+.. option:: --exclude
+
+    A regular expression that matches file names, directory names and paths
+    which mypy should ignore while recursively discovering files to check.
+    Use forward slashes on all platforms.
+
+    For instance, to avoid discovering any files named `setup.py` you could
+    pass ``--exclude '/setup\.py$'``. Similarly, you can ignore discovering
+    directories with a given name by e.g. ``--exclude /build/`` or
+    those matching a subpath with ``--exclude /project/vendor/``.
+
+    Note that this flag only affects recursive discovery, that is, when mypy is
+    discovering files within a directory tree or submodules of a package to
+    check. If you pass a file or module explicitly it will still be checked. For
+    instance, ``mypy --exclude '/setup.py$' but_still_check/setup.py``.
+
+    Note that mypy will never recursively discover files and directories named
+    "site-packages", "node_modules" or "__pycache__", or those whose name starts
+    with a period, exactly as ``--exclude
+    '/(site-packages|node_modules|__pycache__|\..*)/$'`` would. Mypy will also
+    never recursively discover files with extensions other than ``.py`` or
+    ``.pyi``.
+
+
 Optional arguments
 ******************
 
@@ -435,7 +459,7 @@ potentially problematic or redundant in some way.
     .. code-block:: python
 
         def process(x: int) -> None:
-            # Error: Right operand of 'or' is never evaluated
+            # Error: Right operand of "or" is never evaluated
             if isinstance(x, int) or x > 7:
                 # Error: Unsupported operand types for + ("int" and "str")
                 print(x + "bad")

docs/source/common_issues.rst

@@ -772,3 +772,30 @@ False:
 
 If you use the :option:`--warn-unreachable <mypy --warn-unreachable>` flag, mypy will generate
 an error about each unreachable code block.
+
+
+Narrowing and inner functions
+-----------------------------
+
+Because closures in Python are late-binding (https://docs.python-guide.org/writing/gotchas/#late-binding-closures),
+mypy will not narrow the type of a captured variable in an inner function.
+This is best understood via an example:
+
+.. code-block:: python
+
+    def foo(x: Optional[int]) -> Callable[[], int]:
+        if x is None:
+            x = 5
+        print(x + 1)  # mypy correctly deduces x must be an int here
+        def inner() -> int:
+            return x + 1  # but (correctly) complains about this line
+
+        x = None  # because x could later be assigned None
+        return inner
+
+    inner = foo(5)
+    inner()  # this will raise an error when called
+
+To get this code to type check, you could assign ``y = x`` after ``x`` has been
+narrowed, and use ``y`` in the inner function, or add an assert in the inner
+function.

docs/source/config_file.rst

@@ -197,6 +197,18 @@ section of the command line docs.
 
     This option may only be set in the global section (``[mypy]``).
 
+.. confval:: exclude
+
+    :type: regular expression
+
+    A regular expression that matches file names, directory names and paths
+    which mypy should ignore while recursively discovering files to check.
+    Use forward slashes on all platforms.
+
+    For more details, see :option:`--exclude <mypy --exclude>`.
+
+    This option may only be set in the global section (``[mypy]``).
+
 .. confval:: namespace_packages
 
     :type: boolean
@@ -207,6 +219,19 @@ section of the command line docs.
 
     This option may only be set in the global section (``[mypy]``).
 
+.. confval:: explicit_package_bases
+
+    :type: boolean
+    :default: False
+
+    This flag tells mypy that top-level packages will be based in either the
+    current directory, or a member of the ``MYPYPATH`` environment variable or
+    :confval:`mypy_path` config option. This option is only useful in
+    conjunction with :confval:`namespace_packages`. See :ref:`Mapping file
+    paths to modules <mapping-paths-to-modules>` for details.
+
+    This option may only be set in the global section (``[mypy]``).
+
 .. confval:: ignore_missing_imports
 
     :type: boolean
@@ -250,6 +275,10 @@ section of the command line docs.
     Used in conjunction with :confval:`follow_imports=error <follow_imports>`, this can be used
     to make any use of a particular ``typeshed`` module an error.
 
+    .. note::
+
+         This is not supported by the mypy daemon.
+
 .. confval:: python_executable
 
     :type: string

docs/source/error_code_list.rst

@@ -208,7 +208,7 @@ Example with an error:
 
    class Bundle:
        def __init__(self) -> None:
-           # Error: Need type annotation for 'items'
+           # Error: Need type annotation for "items"
            #        (hint: "items: List[<type>] = ...")  [var-annotated]
            self.items = []
 

docs/source/error_code_list2.rst

@@ -187,7 +187,7 @@ incorrect control flow or conditional checks that are accidentally always true o
     # mypy: warn-unreachable
 
     def example(x: int) -> None:
-        # Error: Right operand of 'or' is never evaluated  [unreachable]
+        # Error: Right operand of "or" is never evaluated  [unreachable]
         assert isinstance(x, int) or x == 'unused'
 
         return
@@ -205,7 +205,7 @@ mypy generates an error if it thinks that an expression is redundant.
     # mypy: enable-error-code redundant-expr
 
     def example(x: int) -> None:
-        # Error: Left operand of 'and' is always true  [redundant-expr]
+        # Error: Left operand of "and" is always true  [redundant-expr]
         if isinstance(x, int) and x > 0:
             pass
 

docs/source/error_codes.rst

@@ -24,7 +24,7 @@ Displaying error codes
 ----------------------
 
 Error codes are not displayed by default.  Use :option:`--show-error-codes <mypy --show-error-codes>`
-to display error codes. Error codes are shown inside square brackets:
+or config `show_error_codes = True` to display error codes. Error codes are shown inside square brackets:
 
 .. code-block:: text
 

docs/source/getting_started.rst

@@ -302,7 +302,7 @@ for example, when assigning an empty dictionary to some global value:
 
 .. code-block:: python
 
-    my_global_dict = {}  # Error: Need type annotation for 'my_global_dict'
+    my_global_dict = {}  # Error: Need type annotation for "my_global_dict"
 
 You can teach mypy what type ``my_global_dict`` is meant to have by giving it
 a type hint. For example, if you knew this variable is supposed to be a dict

docs/source/protocols.rst

@@ -429,6 +429,8 @@ support for basic runtime structural checks:
        def __init__(self) -> None:
            self.handles = 1
 
+   def use(handles: int) -> None: ...
+
    mug = Mug()
    if isinstance(mug, Portable):
       use(mug.handles)  # Works statically and at runtime

docs/source/running_mypy.rst

@@ -390,7 +390,8 @@ to modules to type check.
 - Mypy will check all paths provided that correspond to files.
 
 - Mypy will recursively discover and check all files ending in ``.py`` or
-  ``.pyi`` in directory paths provided.
+  ``.pyi`` in directory paths provided, after accounting for
+  :option:`--exclude <mypy --exclude>`.
 
 - For each file to be checked, mypy will attempt to associate the file (e.g.
   ``project/foo/bar/baz.py``) with a fully qualified module name (e.g.

docs/source/type_inference_and_annotations.rst

@@ -78,7 +78,7 @@ without some help:
 
 .. code-block:: python
 
-   l = []  # Error: Need type annotation for 'l'
+   l = []  # Error: Need type annotation for "l"
 
 In these cases you can give the type explicitly using a type annotation:
 
@@ -162,7 +162,7 @@ in the following statement:
     def foo(arg: List[int]) -> None:
         print('Items:', ', '.join(arg))
 
-    a = []  # Error: Need type annotation for 'a'
+    a = []  # Error: Need type annotation for "a"
     foo(a)
 
 Working around the issue is easy by adding a type annotation:

misc/build_wheel.py

@@ -0,0 +1,131 @@
+"""Script to build compiled binary wheels that can be uploaded to PyPI.
+
+The main GitHub workflow where this script is used:
+https://github.com/mypyc/mypy_mypyc-wheels/blob/master/.github/workflows/build.yml
+
+This uses cibuildwheel (https://github.com/joerick/cibuildwheel) to
+build the wheels.
+
+Usage:
+
+  build_wheel_ci.py --python-version <python-version> \
+                    --output-dir <dir>
+
+Wheels for the given Python version will be created in the given directory.
+Python version is in form "39".
+
+This works on macOS, Windows and Linux.
+
+You can test locally by using --extra-opts. macOS example:
+
+  mypy/misc/build_wheel_ci.py --python-version 39 --output-dir out --extra-opts="--platform macos"
+
+Other supported values for platform: linux, windows
+"""
+
+import argparse
+import os
+import subprocess
+import sys
+from typing import Dict
+
+# Clang package we use on Linux
+LLVM_URL = 'https://github.com/mypyc/mypy_mypyc-wheels/releases/download/llvm/llvm-centos-5.tar.gz'
+
+# Mypy repository root
+ROOT_DIR = os.path.dirname(os.path.dirname(__file__))
+
+
+def create_environ(python_version: str) -> Dict[str, str]:
+    """Set up environment variables for cibuildwheel."""
+    env = os.environ.copy()
+
+    env['CIBW_BUILD'] = "cp{}-*".format(python_version)
+
+    # Don't build 32-bit wheels
+    env['CIBW_SKIP'] = "*-manylinux_i686 *-win32"
+
+    env['CIBW_BUILD_VERBOSITY'] = '1'
+
+    # mypy's isolated builds don't specify the requirements mypyc needs, so install
+    # requirements and don't use isolated builds. we need to use build-requirements.txt
+    # with recent mypy commits to get stub packages needed for compilation.
+    #
+    # TODO: remove use of mypy-requirements.txt once we no longer need to support
+    #       building pre modular typeshed releases
+    env['CIBW_BEFORE_BUILD'] = """
+      pip install -r {package}/mypy-requirements.txt &&
+      (pip install -r {package}/build-requirements.txt || true)
+    """.replace('\n', ' ')
+
+    # download a copy of clang to use to compile on linux. this was probably built in 2018,
+    # speeds up compilation 2x
+    env['CIBW_BEFORE_BUILD_LINUX'] = """
+      (cd / && curl -L %s | tar xzf -) &&
+      pip install -r {package}/mypy-requirements.txt &&
+      (pip install -r {package}/build-requirements.txt || true)
+    """.replace('\n', ' ') % LLVM_URL
+
+    # the double negative is counterintuitive, https://github.com/pypa/pip/issues/5735
+    env['CIBW_ENVIRONMENT'] = 'MYPY_USE_MYPYC=1 MYPYC_OPT_LEVEL=3 PIP_NO_BUILD_ISOLATION=no'
+    env['CIBW_ENVIRONMENT_LINUX'] = (
+        'MYPY_USE_MYPYC=1 MYPYC_OPT_LEVEL=3 PIP_NO_BUILD_ISOLATION=no ' +
+        'CC=/opt/llvm/bin/clang'
+    )
+    env['CIBW_ENVIRONMENT_WINDOWS'] = (
+        'MYPY_USE_MYPYC=1 MYPYC_OPT_LEVEL=2 PIP_NO_BUILD_ISOLATION=no'
+    )
+
+    # lxml is slow to build wheels for new releases, so allow installing reqs to fail
+    # if we failed to install lxml, we'll skip tests, but allow the build to succeed
+    env['CIBW_BEFORE_TEST'] = (
+        'pip install -r {project}/mypy/test-requirements.txt || true'
+    )
+
+    # pytest looks for configuration files in the parent directories of where the tests live.
+    # since we are trying to run the tests from their installed location, we copy those into
+    # the venv. Ew ew ew.
+    env['CIBW_TEST_COMMAND'] = """
+      ( ! pip list | grep lxml ) || (
+      DIR=$(python -c 'import mypy, os; dn = os.path.dirname; print(dn(dn(mypy.__path__[0])))')
+      && TEST_DIR=$(python -c 'import mypy.test; print(mypy.test.__path__[0])')
+      && cp '{project}/mypy/pytest.ini' '{project}/mypy/conftest.py' $DIR
+      && MYPY_TEST_PREFIX='{project}/mypy' pytest $TEST_DIR
+      )
+    """.replace('\n', ' ')
+
+    # i ran into some flaky tests on windows, so only run testcheck. it looks like we
+    # previously didn't run any tests on windows wheels, so this is a net win.
+    env['CIBW_TEST_COMMAND_WINDOWS'] = """
+      bash -c "
+      ( ! pip list | grep lxml ) || (
+      DIR=$(python -c 'import mypy, os; dn = os.path.dirname; print(dn(dn(mypy.__path__[0])))')
+      && TEST_DIR=$(python -c 'import mypy.test; print(mypy.test.__path__[0])')
+      && cp '{project}/mypy/pytest.ini' '{project}/mypy/conftest.py' $DIR
+      && MYPY_TEST_PREFIX='{project}/mypy' pytest $TEST_DIR/testcheck.py
+      )
+      "
+    """.replace('\n', ' ')
+    return env
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser()
+    parser.add_argument('--python-version', required=True, metavar='XY',
+                        help='Python version (e.g. 38 or 39)')
+    parser.add_argument('--output-dir', required=True, metavar='DIR',
+                        help='Output directory for created wheels')
+    parser.add_argument('--extra-opts', default='', metavar='OPTIONS',
+                        help='Extra options passed to cibuildwheel verbatim')
+    args = parser.parse_args()
+    python_version = args.python_version
+    output_dir = args.output_dir
+    extra_opts = args.extra_opts
+    environ = create_environ(python_version)
+    script = 'python -m cibuildwheel {} --output-dir {} {}'.format(extra_opts, output_dir,
+                                                                   ROOT_DIR)
+    subprocess.check_call(script, shell=True, env=environ)
+
+
+if __name__ == '__main__':
+    main()