Suggest additional types-* packages from typeshed (#13698)

Now we have suggestions for almost all the currently available
third-party packages in typeshed that don't have a release that includes
PEP 561 type information. `sqlalchemy` is not included, since it has a
few alternatives (some outside typeshed), and I don't know which we
should suggest.

These stubs were never bundled with mypy, so `--ignore-missing-imports`
works with these, unlike the existing packages for which we have
suggestions.

Here's an example mypy output:
```
t.py:1: error: Library stubs not installed for "tree_sitter"
t.py:1: note: Hint: "python3 -m pip install types-tree-sitter"
t.py:1: note: (or run "mypy --install-types" to install all missing stub packages)
t.py:1: note: See https://mypy.readthedocs.io/en/stable/running_mypy.html#missing-imports
```

I manually tested that `--install-types` works for these.

Author: JukkaL

mypy/build.py

@@ -92,7 +92,12 @@
 from mypy.plugins.default import DefaultPlugin
 from mypy.renaming import LimitedVariableRenameVisitor, VariableRenameVisitor
 from mypy.stats import dump_type_stats
-from mypy.stubinfo import is_legacy_bundled_package, legacy_bundled_packages
+from mypy.stubinfo import (
+    is_legacy_bundled_package,
+    legacy_bundled_packages,
+    non_bundled_packages,
+    stub_package_name,
+)
 from mypy.types import Type
 from mypy.typestate import TypeState, reset_global_state
 from mypy.version import __version__
@@ -2740,14 +2745,14 @@ def module_not_found(
         msg, notes = reason.error_message_templates(daemon)
         errors.report(line, 0, msg.format(module=target), code=codes.IMPORT)
         top_level, second_level = get_top_two_prefixes(target)
-        if second_level in legacy_bundled_packages:
+        if second_level in legacy_bundled_packages or second_level in non_bundled_packages:
             top_level = second_level
         for note in notes:
             if "{stub_dist}" in note:
-                note = note.format(stub_dist=legacy_bundled_packages[top_level])
+                note = note.format(stub_dist=stub_package_name(top_level))
             errors.report(line, 0, note, severity="note", only_once=True, code=codes.IMPORT)
         if reason is ModuleNotFoundReason.APPROVED_STUBS_NOT_INSTALLED:
-            manager.missing_stub_packages.add(legacy_bundled_packages[top_level])
+            manager.missing_stub_packages.add(stub_package_name(top_level))
     errors.set_import_context(save_import_context)
 
 

mypy/modulefinder.py

@@ -28,7 +28,7 @@
 from mypy.fscache import FileSystemCache
 from mypy.nodes import MypyFile
 from mypy.options import Options
-from mypy.stubinfo import is_legacy_bundled_package
+from mypy.stubinfo import approved_stub_package_exists
 
 
 # Paths to be searched in find_module().
@@ -336,13 +336,13 @@ def _find_module_non_stub_helper(
             # If this is not a directory then we can't traverse further into it
             if not self.fscache.isdir(dir_path):
                 break
-        if is_legacy_bundled_package(components[0]):
+        if approved_stub_package_exists(components[0]):
             if len(components) == 1 or (
                 self.find_module(components[0])
                 is ModuleNotFoundReason.APPROVED_STUBS_NOT_INSTALLED
             ):
                 return ModuleNotFoundReason.APPROVED_STUBS_NOT_INSTALLED
-        if is_legacy_bundled_package(".".join(components[:2])):
+        if approved_stub_package_exists(".".join(components[:2])):
             return ModuleNotFoundReason.APPROVED_STUBS_NOT_INSTALLED
         if plausible_match:
             return ModuleNotFoundReason.FOUND_WITHOUT_TYPE_HINTS

mypy/stubinfo.py

@@ -5,6 +5,14 @@ def is_legacy_bundled_package(prefix: str) -> bool:
     return prefix in legacy_bundled_packages
 
 
+def approved_stub_package_exists(prefix: str) -> bool:
+    return is_legacy_bundled_package(prefix) or prefix in non_bundled_packages
+
+
+def stub_package_name(prefix: str) -> str:
+    return legacy_bundled_packages.get(prefix) or non_bundled_packages[prefix]
+
+
 # Stubs for these third-party packages used to be shipped with mypy.
 #
 # Map package name to PyPI stub distribution name.
@@ -64,3 +72,98 @@ def is_legacy_bundled_package(prefix: str) -> bool:
     "waitress": "types-waitress",
     "yaml": "types-PyYAML",
 }
+
+# Map package name to PyPI stub distribution name from typeshed.
+# Stubs for these packages were never bundled with mypy. Don't
+# include packages that have a release that includes PEP 561 type
+# information.
+#
+# Package name can have one or two components ('a' or 'a.b').
+#
+# Note that these packages are omitted for now:
+#   sqlalchemy: It's unclear which stub package to suggest. There's also
+#               a mypy plugin available.
+non_bundled_packages = {
+    "MySQLdb": "types-mysqlclient",
+    "PIL": "types-Pillow",
+    "PyInstaller": "types-pyinstaller",
+    "annoy": "types-annoy",
+    "appdirs": "types-appdirs",
+    "aws_xray_sdk": "types-aws-xray-sdk",
+    "babel": "types-babel",
+    "backports.ssl_match_hostname": "types-backports.ssl_match_hostname",
+    "braintree": "types-braintree",
+    "bs4": "types-beautifulsoup4",
+    "bugbear": "types-flake8-bugbear",
+    "caldav": "types-caldav",
+    "cffi": "types-cffi",
+    "chevron": "types-chevron",
+    "colorama": "types-colorama",
+    "commonmark": "types-commonmark",
+    "cryptography": "types-cryptography",
+    "d3dshot": "types-D3DShot",
+    "dj_database_url": "types-dj-database-url",
+    "docopt": "types-docopt",
+    "editdistance": "types-editdistance",
+    "entrypoints": "types-entrypoints",
+    "farmhash": "types-pyfarmhash",
+    "flake8_2020": "types-flake8-2020",
+    "flake8_builtins": "types-flake8-builtins",
+    "flake8_docstrings": "types-flake8-docstrings",
+    "flake8_plugin_utils": "types-flake8-plugin-utils",
+    "flake8_rst_docstrings": "types-flake8-rst-docstrings",
+    "flake8_simplify": "types-flake8-simplify",
+    "flake8_typing_imports": "types-flake8-typing-imports",
+    "flask_cors": "types-Flask-Cors",
+    "flask_sqlalchemy": "types-Flask-SQLAlchemy",
+    "fpdf": "types-fpdf2",
+    "gdb": "types-gdb",
+    "google.cloud": "types-google-cloud-ndb",
+    "hdbcli": "types-hdbcli",
+    "html5lib": "types-html5lib",
+    "httplib2": "types-httplib2",
+    "humanfriendly": "types-humanfriendly",
+    "invoke": "types-invoke",
+    "jack": "types-JACK-Client",
+    "jmespath": "types-jmespath",
+    "jose": "types-python-jose",
+    "jsonschema": "types-jsonschema",
+    "keyboard": "types-keyboard",
+    "ldap3": "types-ldap3",
+    "nmap": "types-python-nmap",
+    "oauthlib": "types-oauthlib",
+    "openpyxl": "types-openpyxl",
+    "opentracing": "types-opentracing",
+    "parsimonious": "types-parsimonious",
+    "passlib": "types-passlib",
+    "passpy": "types-passpy",
+    "pep8ext_naming": "types-pep8-naming",
+    "playsound": "types-playsound",
+    "prettytable": "types-prettytable",
+    "psutil": "types-psutil",
+    "psycopg2": "types-psycopg2",
+    "pyaudio": "types-pyaudio",
+    "pyautogui": "types-PyAutoGUI",
+    "pyflakes": "types-pyflakes",
+    "pygments": "types-Pygments",
+    "pyi_splash": "types-pyinstaller",
+    "pynput": "types-pynput",
+    "pysftp": "types-pysftp",
+    "pytest_lazyfixture": "types-pytest-lazy-fixture",
+    "regex": "types-regex",
+    "send2trash": "types-Send2Trash",
+    "slumber": "types-slumber",
+    "stdlib_list": "types-stdlib-list",
+    "stripe": "types-stripe",
+    "toposort": "types-toposort",
+    "tqdm": "types-tqdm",
+    "tree_sitter": "types-tree-sitter",
+    "tree_sitter_languages": "types-tree-sitter-languages",
+    "ttkthemes": "types-ttkthemes",
+    "urllib3": "types-urllib3",
+    "vobject": "types-vobject",
+    "whatthepatch": "types-whatthepatch",
+    "xmltodict": "types-xmltodict",
+    "xxhash": "types-xxhash",
+    "zxcvbn": "types-zxcvbn",
+}

test-data/unit/check-modules.test

@@ -3111,10 +3111,15 @@ from google.cloud import x
 [case testErrorFromGoogleCloud]
 import google.cloud
 from google.cloud import x
+import google.non_existent
+from google.non_existent import x
 [out]
-main:1: error: Cannot find implementation or library stub for module named "google.cloud"
+main:1: error: Library stubs not installed for "google.cloud"
+main:1: note: Hint: "python3 -m pip install types-google-cloud-ndb"
+main:1: note: (or run "mypy --install-types" to install all missing stub packages)
 main:1: note: See https://mypy.readthedocs.io/en/stable/running_mypy.html#missing-imports
 main:1: error: Cannot find implementation or library stub for module named "google"
+main:3: error: Cannot find implementation or library stub for module named "google.non_existent"
 
 [case testMissingSubmoduleOfInstalledStubPackage]
 import bleach.xyz

test-data/unit/fine-grained-modules.test

@@ -2192,17 +2192,17 @@ x = 'x'
 [case testLibraryStubsNotInstalled]
 import a
 [file a.py]
-import waitress
+import requests
 [file a.py.2]
 # nothing
 [file a.py.3]
-import requests
+import jack
 [out]
-a.py:1: error: Library stubs not installed for "waitress"
-a.py:1: note: Hint: "python3 -m pip install types-waitress"
+a.py:1: error: Library stubs not installed for "requests"
+a.py:1: note: Hint: "python3 -m pip install types-requests"
 a.py:1: note: See https://mypy.readthedocs.io/en/stable/running_mypy.html#missing-imports
 ==
 ==
-a.py:1: error: Library stubs not installed for "requests"
-a.py:1: note: Hint: "python3 -m pip install types-requests"
+a.py:1: error: Library stubs not installed for "jack"
+a.py:1: note: Hint: "python3 -m pip install types-JACK-Client"
 a.py:1: note: See https://mypy.readthedocs.io/en/stable/running_mypy.html#missing-imports

test-data/unit/pythoneval.test

@@ -1561,6 +1561,7 @@ import scribe  # No Python 3 stubs available for scribe
 from scribe import x
 import maxminddb  # Python 3 stubs available for maxminddb
 import foobar_asdf
+import jack  # This has a stubs package but was never bundled with mypy, so ignoring works
 [out]
 _testIgnoreImportIfNoPython3StubAvailable.py:4: error: Library stubs not installed for "maxminddb"
 _testIgnoreImportIfNoPython3StubAvailable.py:4: note: Hint: "python3 -m pip install types-maxminddb"