diff --git a/docs/source/running_mypy.rst b/docs/source/running_mypy.rst index 9f7461d24f72..d3393bbcf071 100644 --- a/docs/source/running_mypy.rst +++ b/docs/source/running_mypy.rst @@ -403,6 +403,11 @@ By default, :option:`--install-types ` shows a confirmatio Use :option:`--non-interactive ` to install all suggested stub packages without asking for confirmation *and* type check your code: +If you maintain a lock file (for example ``pylock.toml``), you can combine +:option:`--install-types ` with +``--install-types-from-pylock FILE`` to install known stub packages for locked +runtime dependencies while avoiding upgrades or downgrades of runtime packages. + If you've already installed the relevant third-party libraries in an environment other than the one mypy is running in, you can use :option:`--python-executable ` flag to point to the Python executable for that diff --git a/mypy/installtypes.py b/mypy/installtypes.py new file mode 100644 index 000000000000..f77df93e70df --- /dev/null +++ b/mypy/installtypes.py @@ -0,0 +1,85 @@ +from __future__ import annotations + +import re +import sys +from collections.abc import Mapping + +if sys.version_info >= (3, 11): + import tomllib +else: + import tomli as tomllib + +from mypy.stubinfo import ( + non_bundled_packages_flat, + non_bundled_packages_namespace, + stub_distribution_name, +) + +_DIST_NORMALIZE_RE = re.compile(r"[-_.]+") + + +def normalize_distribution_name(name: str) -> str: + return _DIST_NORMALIZE_RE.sub("-", name).lower() + + +def read_locked_packages(path: str) -> dict[str, str | None]: + """Read package name/version pairs from a pylock-like TOML file. + + Supports common lockfile layouts that use either [[package]] or + [[packages]] tables with "name" and optional "version" keys. + """ + with open(path, "rb") as f: + data = tomllib.load(f) + + entries: list[object] = [] + for key in ("package", "packages"): + value = data.get(key) + if isinstance(value, list): + entries.extend(value) + + locked: dict[str, str | None] = {} + for entry in entries: + if not isinstance(entry, Mapping): + continue + name = entry.get("name") + if not isinstance(name, str) or not name.strip(): + continue + version_obj = entry.get("version") + version = version_obj if isinstance(version_obj, str) and version_obj.strip() else None + locked[normalize_distribution_name(name)] = version + + return locked + + +def resolve_stub_packages_from_lock(locked: Mapping[str, str | None]) -> list[str]: + """Map runtime packages from a lock file to known stubs packages. + + This uses mypy's existing known typeshed mapping and intentionally skips + heuristics that could cause accidental installation of unrelated packages. + """ + known_stubs = set(non_bundled_packages_flat.values()) + for namespace_packages in non_bundled_packages_namespace.values(): + known_stubs.update(namespace_packages.values()) + + stubs: set[str] = set() + for dist_name in locked: + if dist_name.startswith("types-"): + continue + candidates = {dist_name, dist_name.replace("-", "_")} + for module_name in candidates: + stub = stub_distribution_name(module_name) + if stub: + stubs.add(stub) + typeshed_name = f"types-{dist_name}" + if typeshed_name in known_stubs: + stubs.add(typeshed_name) + return sorted(stubs) + + +def make_runtime_constraints(locked: Mapping[str, str | None]) -> list[str]: + """Create pip constraints that pin runtime packages to locked versions.""" + constraints: list[str] = [] + for name, version in sorted(locked.items()): + if version: + constraints.append(f"{name}=={version}") + return constraints diff --git a/mypy/main.py b/mypy/main.py index e90ee961fc70..723cd17e9f74 100644 --- a/mypy/main.py +++ b/mypy/main.py @@ -7,6 +7,7 @@ import platform import subprocess import sys +import tempfile import time from collections import defaultdict from collections.abc import Sequence @@ -29,6 +30,11 @@ from mypy.errors import CompileError from mypy.find_sources import InvalidSourceList, create_source_list from mypy.fscache import FileSystemCache +from mypy.installtypes import ( + make_runtime_constraints, + read_locked_packages, + resolve_stub_packages_from_lock, +) from mypy.modulefinder import ( BuildSource, FindModuleCache, @@ -124,6 +130,18 @@ def main( if options.non_interactive and not options.install_types: fail("error: --non-interactive is only supported with --install-types", stderr, options) + if options.install_types_from_pylock is not None and not options.install_types: + fail( + "error: --install-types-from-pylock is only supported with --install-types", + stderr, + options, + ) + + if options.install_types_from_pylock is not None and not os.path.isfile( + options.install_types_from_pylock + ): + fail(f"error: Can't find lock file '{options.install_types_from_pylock}'", stderr, options) + if options.install_types and not options.incremental: fail( "error: --install-types not supported with incremental mode disabled", stderr, options @@ -137,9 +155,22 @@ def main( ) if options.install_types and not sources: - install_types(formatter, options, non_interactive=options.non_interactive) + install_types( + formatter, + options, + non_interactive=options.non_interactive, + pylock_path=options.install_types_from_pylock, + ) return + if options.install_types and options.install_types_from_pylock: + install_types( + formatter, + options, + non_interactive=options.non_interactive, + pylock_path=options.install_types_from_pylock, + ) + res, messages, blockers = run_build(sources, options, fscache, t0, stdout, stderr) if options.non_interactive: @@ -1204,6 +1235,15 @@ def add_invertible_flag( dest="special-opts:find_occurrences", help="Print out all usages of a class member (experimental)", ) + misc_group.add_argument( + "--install-types-from-pylock", + metavar="FILE", + dest="install_types_from_pylock", + help=( + "With --install-types, read packages from a pylock TOML file and " + "install matching known stub packages" + ), + ) misc_group.add_argument( "--scripts-are-modules", action="store_true", @@ -1711,9 +1751,17 @@ def install_types( *, after_run: bool = False, non_interactive: bool = False, + pylock_path: str | None = None, ) -> bool: """Install stub packages using pip if some missing stubs were detected.""" - packages = read_types_packages_to_install(options.cache_dir, after_run) + constraints: list[str] = [] + if pylock_path is None: + packages = read_types_packages_to_install(options.cache_dir, after_run) + else: + locked = read_locked_packages(pylock_path) + packages = resolve_stub_packages_from_lock(locked) + constraints = make_runtime_constraints(locked) + if not packages: # If there are no missing stubs, generate no output. return False @@ -1721,14 +1769,30 @@ def install_types( print() print("Installing missing stub packages:") assert options.python_executable, "Python executable required to install types" - cmd = [options.python_executable, "-m", "pip", "install"] + packages + cmd = [options.python_executable, "-m", "pip", "install"] + constraints_file = None + if pylock_path is not None: + cmd.append("--no-deps") + if pylock_path is not None and constraints: + constraints_file = tempfile.NamedTemporaryFile("w", delete=False, encoding="utf-8") + with constraints_file as f: + f.write("\n".join(constraints)) + f.write("\n") + cmd += ["--constraint", constraints_file.name] + cmd += packages print(formatter.style(" ".join(cmd), "none", bold=True)) print() if not non_interactive: x = input("Install? [yN] ") if not x.strip() or not x.lower().startswith("y"): print(formatter.style("mypy: Skipping installation", "red", bold=True)) + if constraints_file is not None: + os.unlink(constraints_file.name) sys.exit(2) print() - subprocess.run(cmd) + try: + subprocess.run(cmd) + finally: + if constraints_file is not None: + os.unlink(constraints_file.name) return True diff --git a/mypy/options.py b/mypy/options.py index 81fd88345a43..3e616ba5b971 100644 --- a/mypy/options.py +++ b/mypy/options.py @@ -409,6 +409,8 @@ def __init__(self) -> None: # Install missing stub packages in non-interactive mode (don't prompt for # confirmation, and don't show any errors) self.non_interactive = False + # Lock file used for --install-types to discover installable stub packages. + self.install_types_from_pylock: str | None = None # When we encounter errors that may cause many additional errors, # skip most errors after this many messages have been reported. # -1 means unlimited. diff --git a/mypy/test/testinstalltypes.py b/mypy/test/testinstalltypes.py new file mode 100644 index 000000000000..b1fe2bbb34cc --- /dev/null +++ b/mypy/test/testinstalltypes.py @@ -0,0 +1,51 @@ +from __future__ import annotations + +import os +import tempfile +import textwrap +import unittest + +from mypy.installtypes import ( + make_runtime_constraints, + read_locked_packages, + resolve_stub_packages_from_lock, +) + + +class TestInstallTypesFromPylock(unittest.TestCase): + def test_read_locked_packages(self) -> None: + content = textwrap.dedent(""" + [[package]] + name = "requests" + version = "2.32.3" + + [[packages]] + name = "python-dateutil" + version = "2.9.0" + + [[package]] + name = "types-requests" + version = "2.32.0" + """) + with tempfile.NamedTemporaryFile("w", suffix=".toml", delete=False, encoding="utf-8") as f: + f.write(content) + path = f.name + try: + locked = read_locked_packages(path) + finally: + os.unlink(path) + + assert locked["requests"] == "2.32.3" + assert locked["python-dateutil"] == "2.9.0" + assert locked["types-requests"] == "2.32.0" + + def test_resolve_stub_packages_from_lock(self) -> None: + locked = {"requests": "2.32.3", "python-dateutil": "2.9.0", "types-requests": "2.32.0"} + stubs = resolve_stub_packages_from_lock(locked) + assert "types-requests" in stubs + assert "types-python-dateutil" in stubs + + def test_make_runtime_constraints(self) -> None: + locked = {"requests": "2.32.3", "python-dateutil": "2.9.0", "no-version": None} + constraints = make_runtime_constraints(locked) + assert constraints == ["python-dateutil==2.9.0", "requests==2.32.3"] diff --git a/test-data/unit/cmdline.test b/test-data/unit/cmdline.test index c3440fda74a0..595c7665192c 100644 --- a/test-data/unit/cmdline.test +++ b/test-data/unit/cmdline.test @@ -915,6 +915,12 @@ pkg.py:1: error: Incompatible types in assignment (expression has type "int", va error: --non-interactive is only supported with --install-types == Return code: 2 +[case testCmdlineInstallTypesFromPylockWithoutInstallTypes] +# cmd: mypy --install-types-from-pylock pylock.toml -m pkg +[out] +error: --install-types-from-pylock is only supported with --install-types +== Return code: 2 + [case testCmdlineNonInteractiveInstallTypesNothingToDo] # cmd: mypy --install-types --non-interactive -m pkg [file pkg.py]