xref: /qemu/python/scripts/mkvenv.py (revision 0f1ec0705b92b79e5e5f69bed236639dae67d312)
1dd84028fSJohn Snow"""
2dd84028fSJohn Snowmkvenv - QEMU pyvenv bootstrapping utility
3dd84028fSJohn Snow
4dd84028fSJohn Snowusage: mkvenv [-h] command ...
5dd84028fSJohn Snow
6dd84028fSJohn SnowQEMU pyvenv bootstrapping utility
7dd84028fSJohn Snow
8dd84028fSJohn Snowoptions:
9dd84028fSJohn Snow  -h, --help  show this help message and exit
10dd84028fSJohn Snow
11dd84028fSJohn SnowCommands:
12dd84028fSJohn Snow  command     Description
13dd84028fSJohn Snow    create    create a venv
14f1ad527fSJohn Snow    post_init
15f1ad527fSJohn Snow              post-venv initialization
16c5538eedSJohn Snow    ensure    Ensure that the specified package is installed.
17dd84028fSJohn Snow
18dd84028fSJohn Snow--------------------------------------------------
19dd84028fSJohn Snow
20dd84028fSJohn Snowusage: mkvenv create [-h] target
21dd84028fSJohn Snow
22dd84028fSJohn Snowpositional arguments:
23dd84028fSJohn Snow  target      Target directory to install virtual environment into.
24dd84028fSJohn Snow
25dd84028fSJohn Snowoptions:
26dd84028fSJohn Snow  -h, --help  show this help message and exit
27dd84028fSJohn Snow
28c5538eedSJohn Snow--------------------------------------------------
29c5538eedSJohn Snow
30f1ad527fSJohn Snowusage: mkvenv post_init [-h]
31f1ad527fSJohn Snow
32f1ad527fSJohn Snowoptions:
33f1ad527fSJohn Snow  -h, --help         show this help message and exit
34f1ad527fSJohn Snow
35f1ad527fSJohn Snow--------------------------------------------------
36f1ad527fSJohn Snow
37c5538eedSJohn Snowusage: mkvenv ensure [-h] [--online] [--dir DIR] dep_spec...
38c5538eedSJohn Snow
39c5538eedSJohn Snowpositional arguments:
40c5538eedSJohn Snow  dep_spec    PEP 508 Dependency specification, e.g. 'meson>=0.61.5'
41c5538eedSJohn Snow
42c5538eedSJohn Snowoptions:
43c5538eedSJohn Snow  -h, --help  show this help message and exit
44c5538eedSJohn Snow  --online    Install packages from PyPI, if necessary.
45c5538eedSJohn Snow  --dir DIR   Path to vendored packages where we may install from.
46c5538eedSJohn Snow
47dd84028fSJohn Snow"""
48dd84028fSJohn Snow
49*0f1ec070SPaolo Bonzini# The duplication between importlib and pkg_resources does not help
50*0f1ec070SPaolo Bonzini# pylint: disable=too-many-lines
51*0f1ec070SPaolo Bonzini
52dd84028fSJohn Snow# Copyright (C) 2022-2023 Red Hat, Inc.
53dd84028fSJohn Snow#
54dd84028fSJohn Snow# Authors:
55dd84028fSJohn Snow#  John Snow <jsnow@redhat.com>
56dd84028fSJohn Snow#  Paolo Bonzini <pbonzini@redhat.com>
57dd84028fSJohn Snow#
58dd84028fSJohn Snow# This work is licensed under the terms of the GNU GPL, version 2 or
59dd84028fSJohn Snow# later. See the COPYING file in the top-level directory.
60dd84028fSJohn Snow
61dd84028fSJohn Snowimport argparse
62a9dbde71SJohn Snowfrom importlib.util import find_spec
63dd84028fSJohn Snowimport logging
64dd84028fSJohn Snowimport os
65dd84028fSJohn Snowfrom pathlib import Path
664695a22eSJohn Snowimport re
674695a22eSJohn Snowimport shutil
68dee01b82SJohn Snowimport site
69dd84028fSJohn Snowimport subprocess
70dd84028fSJohn Snowimport sys
71dee01b82SJohn Snowimport sysconfig
72dd84028fSJohn Snowfrom types import SimpleNamespace
73c5538eedSJohn Snowfrom typing import (
74c5538eedSJohn Snow    Any,
75*0f1ec070SPaolo Bonzini    Dict,
7692834894SJohn Snow    Iterator,
77c5538eedSJohn Snow    Optional,
78c5538eedSJohn Snow    Sequence,
794695a22eSJohn Snow    Tuple,
80c5538eedSJohn Snow    Union,
81c5538eedSJohn Snow)
82dd84028fSJohn Snowimport venv
83c5538eedSJohn Snow
8468ea6d17SJohn Snow
8568ea6d17SJohn Snow# Try to load distlib, with a fallback to pip's vendored version.
8668ea6d17SJohn Snow# HAVE_DISTLIB is checked below, just-in-time, so that mkvenv does not fail
8768ea6d17SJohn Snow# outside the venv or before a potential call to ensurepip in checkpip().
8868ea6d17SJohn SnowHAVE_DISTLIB = True
8968ea6d17SJohn Snowtry:
9092834894SJohn Snow    import distlib.scripts
91c5538eedSJohn Snow    import distlib.version
9268ea6d17SJohn Snowexcept ImportError:
9368ea6d17SJohn Snow    try:
9468ea6d17SJohn Snow        # Reach into pip's cookie jar.  pylint and flake8 don't understand
9568ea6d17SJohn Snow        # that these imports will be used via distlib.xxx.
9668ea6d17SJohn Snow        from pip._vendor import distlib
9768ea6d17SJohn Snow        import pip._vendor.distlib.scripts  # noqa, pylint: disable=unused-import
9868ea6d17SJohn Snow        import pip._vendor.distlib.version  # noqa, pylint: disable=unused-import
9968ea6d17SJohn Snow    except ImportError:
10068ea6d17SJohn Snow        HAVE_DISTLIB = False
101dd84028fSJohn Snow
102dd84028fSJohn Snow# Do not add any mandatory dependencies from outside the stdlib:
103dd84028fSJohn Snow# This script *must* be usable standalone!
104dd84028fSJohn Snow
105dd84028fSJohn SnowDirType = Union[str, bytes, "os.PathLike[str]", "os.PathLike[bytes]"]
106dd84028fSJohn Snowlogger = logging.getLogger("mkvenv")
107dd84028fSJohn Snow
108dd84028fSJohn Snow
109dee01b82SJohn Snowdef inside_a_venv() -> bool:
110dee01b82SJohn Snow    """Returns True if it is executed inside of a virtual environment."""
111dee01b82SJohn Snow    return sys.prefix != sys.base_prefix
112dee01b82SJohn Snow
113dee01b82SJohn Snow
114dd84028fSJohn Snowclass Ouch(RuntimeError):
115dd84028fSJohn Snow    """An Exception class we can't confuse with a builtin."""
116dd84028fSJohn Snow
117dd84028fSJohn Snow
118dd84028fSJohn Snowclass QemuEnvBuilder(venv.EnvBuilder):
119dd84028fSJohn Snow    """
120dd84028fSJohn Snow    An extension of venv.EnvBuilder for building QEMU's configure-time venv.
121dd84028fSJohn Snow
122dee01b82SJohn Snow    The primary difference is that it emulates a "nested" virtual
123dee01b82SJohn Snow    environment when invoked from inside of an existing virtual
124f1ad527fSJohn Snow    environment by including packages from the parent.  Also,
125f1ad527fSJohn Snow    "ensurepip" is replaced if possible with just recreating pip's
126f1ad527fSJohn Snow    console_scripts inside the virtual environment.
127dd84028fSJohn Snow
128dd84028fSJohn Snow    Parameters for base class init:
129dd84028fSJohn Snow      - system_site_packages: bool = False
130dd84028fSJohn Snow      - clear: bool = False
131dd84028fSJohn Snow      - symlinks: bool = False
132dd84028fSJohn Snow      - upgrade: bool = False
133dd84028fSJohn Snow      - with_pip: bool = False
134dd84028fSJohn Snow      - prompt: Optional[str] = None
135dd84028fSJohn Snow      - upgrade_deps: bool = False             (Since 3.9)
136dd84028fSJohn Snow    """
137dd84028fSJohn Snow
138dd84028fSJohn Snow    def __init__(self, *args: Any, **kwargs: Any) -> None:
139dd84028fSJohn Snow        logger.debug("QemuEnvBuilder.__init__(...)")
140a9dbde71SJohn Snow
141dee01b82SJohn Snow        # For nested venv emulation:
142dee01b82SJohn Snow        self.use_parent_packages = False
143dee01b82SJohn Snow        if inside_a_venv():
144dee01b82SJohn Snow            # Include parent packages only if we're in a venv and
145dee01b82SJohn Snow            # system_site_packages was True.
146dee01b82SJohn Snow            self.use_parent_packages = kwargs.pop(
147dee01b82SJohn Snow                "system_site_packages", False
148dee01b82SJohn Snow            )
149dee01b82SJohn Snow            # Include system_site_packages only when the parent,
150dee01b82SJohn Snow            # The venv we are currently in, also does so.
151dee01b82SJohn Snow            kwargs["system_site_packages"] = sys.base_prefix in site.PREFIXES
152dee01b82SJohn Snow
153f1ad527fSJohn Snow        # ensurepip is slow: venv creation can be very fast for cases where
154f1ad527fSJohn Snow        # we allow the use of system_site_packages. Therefore, ensurepip is
155f1ad527fSJohn Snow        # replaced with our own script generation once the virtual environment
156f1ad527fSJohn Snow        # is setup.
157f1ad527fSJohn Snow        self.want_pip = kwargs.get("with_pip", False)
158f1ad527fSJohn Snow        if self.want_pip:
159f1ad527fSJohn Snow            if (
160f1ad527fSJohn Snow                kwargs.get("system_site_packages", False)
161f1ad527fSJohn Snow                and not need_ensurepip()
162f1ad527fSJohn Snow            ):
163f1ad527fSJohn Snow                kwargs["with_pip"] = False
164f1ad527fSJohn Snow            else:
165c8049626SJohn Snow                check_ensurepip(suggest_remedy=True)
166a9dbde71SJohn Snow
167dd84028fSJohn Snow        super().__init__(*args, **kwargs)
168dd84028fSJohn Snow
169dd84028fSJohn Snow        # Make the context available post-creation:
170dd84028fSJohn Snow        self._context: Optional[SimpleNamespace] = None
171dd84028fSJohn Snow
172dee01b82SJohn Snow    def get_parent_libpath(self) -> Optional[str]:
173dee01b82SJohn Snow        """Return the libpath of the parent venv, if applicable."""
174dee01b82SJohn Snow        if self.use_parent_packages:
175dee01b82SJohn Snow            return sysconfig.get_path("purelib")
176dee01b82SJohn Snow        return None
177dee01b82SJohn Snow
178dee01b82SJohn Snow    @staticmethod
179dee01b82SJohn Snow    def compute_venv_libpath(context: SimpleNamespace) -> str:
180dee01b82SJohn Snow        """
181dee01b82SJohn Snow        Compatibility wrapper for context.lib_path for Python < 3.12
182dee01b82SJohn Snow        """
183dee01b82SJohn Snow        # Python 3.12+, not strictly necessary because it's documented
184dee01b82SJohn Snow        # to be the same as 3.10 code below:
185dee01b82SJohn Snow        if sys.version_info >= (3, 12):
186dee01b82SJohn Snow            return context.lib_path
187dee01b82SJohn Snow
188dee01b82SJohn Snow        # Python 3.10+
189dee01b82SJohn Snow        if "venv" in sysconfig.get_scheme_names():
190dee01b82SJohn Snow            lib_path = sysconfig.get_path(
191dee01b82SJohn Snow                "purelib", scheme="venv", vars={"base": context.env_dir}
192dee01b82SJohn Snow            )
193dee01b82SJohn Snow            assert lib_path is not None
194dee01b82SJohn Snow            return lib_path
195dee01b82SJohn Snow
196dee01b82SJohn Snow        # For Python <= 3.9 we need to hardcode this. Fortunately the
197dee01b82SJohn Snow        # code below was the same in Python 3.6-3.10, so there is only
198dee01b82SJohn Snow        # one case.
199dee01b82SJohn Snow        if sys.platform == "win32":
200dee01b82SJohn Snow            return os.path.join(context.env_dir, "Lib", "site-packages")
201dee01b82SJohn Snow        return os.path.join(
202dee01b82SJohn Snow            context.env_dir,
203dee01b82SJohn Snow            "lib",
204dee01b82SJohn Snow            "python%d.%d" % sys.version_info[:2],
205dee01b82SJohn Snow            "site-packages",
206dee01b82SJohn Snow        )
207dee01b82SJohn Snow
208dd84028fSJohn Snow    def ensure_directories(self, env_dir: DirType) -> SimpleNamespace:
209dd84028fSJohn Snow        logger.debug("ensure_directories(env_dir=%s)", env_dir)
210dd84028fSJohn Snow        self._context = super().ensure_directories(env_dir)
211dd84028fSJohn Snow        return self._context
212dd84028fSJohn Snow
213dee01b82SJohn Snow    def create(self, env_dir: DirType) -> None:
214dee01b82SJohn Snow        logger.debug("create(env_dir=%s)", env_dir)
215dee01b82SJohn Snow        super().create(env_dir)
216dee01b82SJohn Snow        assert self._context is not None
217dee01b82SJohn Snow        self.post_post_setup(self._context)
218dee01b82SJohn Snow
219dee01b82SJohn Snow    def post_post_setup(self, context: SimpleNamespace) -> None:
220dee01b82SJohn Snow        """
221dee01b82SJohn Snow        The final, final hook. Enter the venv and run commands inside of it.
222dee01b82SJohn Snow        """
223dee01b82SJohn Snow        if self.use_parent_packages:
224dee01b82SJohn Snow            # We're inside of a venv and we want to include the parent
225dee01b82SJohn Snow            # venv's packages.
226dee01b82SJohn Snow            parent_libpath = self.get_parent_libpath()
227dee01b82SJohn Snow            assert parent_libpath is not None
228dee01b82SJohn Snow            logger.debug("parent_libpath: %s", parent_libpath)
229dee01b82SJohn Snow
230dee01b82SJohn Snow            our_libpath = self.compute_venv_libpath(context)
231dee01b82SJohn Snow            logger.debug("our_libpath: %s", our_libpath)
232dee01b82SJohn Snow
233dee01b82SJohn Snow            pth_file = os.path.join(our_libpath, "nested.pth")
234dee01b82SJohn Snow            with open(pth_file, "w", encoding="UTF-8") as file:
235dee01b82SJohn Snow                file.write(parent_libpath + os.linesep)
236dee01b82SJohn Snow
237f1ad527fSJohn Snow        if self.want_pip:
238f1ad527fSJohn Snow            args = [
239f1ad527fSJohn Snow                context.env_exe,
240f1ad527fSJohn Snow                __file__,
241f1ad527fSJohn Snow                "post_init",
242f1ad527fSJohn Snow            ]
243f1ad527fSJohn Snow            subprocess.run(args, check=True)
244f1ad527fSJohn Snow
245dd84028fSJohn Snow    def get_value(self, field: str) -> str:
246dd84028fSJohn Snow        """
247dd84028fSJohn Snow        Get a string value from the context namespace after a call to build.
248dd84028fSJohn Snow
249dd84028fSJohn Snow        For valid field names, see:
250dd84028fSJohn Snow        https://docs.python.org/3/library/venv.html#venv.EnvBuilder.ensure_directories
251dd84028fSJohn Snow        """
252dd84028fSJohn Snow        ret = getattr(self._context, field)
253dd84028fSJohn Snow        assert isinstance(ret, str)
254dd84028fSJohn Snow        return ret
255dd84028fSJohn Snow
256dd84028fSJohn Snow
257f1ad527fSJohn Snowdef need_ensurepip() -> bool:
258f1ad527fSJohn Snow    """
259f1ad527fSJohn Snow    Tests for the presence of setuptools and pip.
260f1ad527fSJohn Snow
261f1ad527fSJohn Snow    :return: `True` if we do not detect both packages.
262f1ad527fSJohn Snow    """
263f1ad527fSJohn Snow    # Don't try to actually import them, it's fraught with danger:
264f1ad527fSJohn Snow    # https://github.com/pypa/setuptools/issues/2993
265f1ad527fSJohn Snow    if find_spec("setuptools") and find_spec("pip"):
266f1ad527fSJohn Snow        return False
267f1ad527fSJohn Snow    return True
268f1ad527fSJohn Snow
269f1ad527fSJohn Snow
270c8049626SJohn Snowdef check_ensurepip(prefix: str = "", suggest_remedy: bool = False) -> None:
271a9dbde71SJohn Snow    """
272a9dbde71SJohn Snow    Check that we have ensurepip.
273a9dbde71SJohn Snow
274a9dbde71SJohn Snow    Raise a fatal exception with a helpful hint if it isn't available.
275a9dbde71SJohn Snow    """
276a9dbde71SJohn Snow    if not find_spec("ensurepip"):
277a9dbde71SJohn Snow        msg = (
278a9dbde71SJohn Snow            "Python's ensurepip module is not found.\n"
279a9dbde71SJohn Snow            "It's normally part of the Python standard library, "
280a9dbde71SJohn Snow            "maybe your distribution packages it separately?\n"
281c8049626SJohn Snow            "(Debian puts ensurepip in its python3-venv package.)\n"
282c8049626SJohn Snow        )
283c8049626SJohn Snow        if suggest_remedy:
284c8049626SJohn Snow            msg += (
285a9dbde71SJohn Snow                "Either install ensurepip, or alleviate the need for it in the"
286a9dbde71SJohn Snow                " first place by installing pip and setuptools for "
287a9dbde71SJohn Snow                f"'{sys.executable}'.\n"
288a9dbde71SJohn Snow            )
289c8049626SJohn Snow        raise Ouch(prefix + msg)
290a9dbde71SJohn Snow
291a9dbde71SJohn Snow    # ensurepip uses pyexpat, which can also go missing on us:
292a9dbde71SJohn Snow    if not find_spec("pyexpat"):
293a9dbde71SJohn Snow        msg = (
294a9dbde71SJohn Snow            "Python's pyexpat module is not found.\n"
295a9dbde71SJohn Snow            "It's normally part of the Python standard library, "
296a9dbde71SJohn Snow            "maybe your distribution packages it separately?\n"
297c8049626SJohn Snow            "(NetBSD's pkgsrc debundles this to e.g. 'py310-expat'.)\n"
298c8049626SJohn Snow        )
299c8049626SJohn Snow        if suggest_remedy:
300c8049626SJohn Snow            msg += (
301a9dbde71SJohn Snow                "Either install pyexpat, or alleviate the need for it in the "
302a9dbde71SJohn Snow                "first place by installing pip and setuptools for "
303c8049626SJohn Snow                f"'{sys.executable}'.\n"
304a9dbde71SJohn Snow            )
305c8049626SJohn Snow        raise Ouch(prefix + msg)
306a9dbde71SJohn Snow
307a9dbde71SJohn Snow
308dd84028fSJohn Snowdef make_venv(  # pylint: disable=too-many-arguments
309dd84028fSJohn Snow    env_dir: Union[str, Path],
310dd84028fSJohn Snow    system_site_packages: bool = False,
311dd84028fSJohn Snow    clear: bool = True,
312dd84028fSJohn Snow    symlinks: Optional[bool] = None,
313dd84028fSJohn Snow    with_pip: bool = True,
314dd84028fSJohn Snow) -> None:
315dd84028fSJohn Snow    """
316dd84028fSJohn Snow    Create a venv using `QemuEnvBuilder`.
317dd84028fSJohn Snow
318dd84028fSJohn Snow    This is analogous to the `venv.create` module-level convenience
319dd84028fSJohn Snow    function that is part of the Python stdblib, except it uses
320dd84028fSJohn Snow    `QemuEnvBuilder` instead.
321dd84028fSJohn Snow
322dd84028fSJohn Snow    :param env_dir: The directory to create/install to.
323dd84028fSJohn Snow    :param system_site_packages:
324dd84028fSJohn Snow        Allow inheriting packages from the system installation.
325dd84028fSJohn Snow    :param clear: When True, fully remove any prior venv and files.
326dd84028fSJohn Snow    :param symlinks:
327dd84028fSJohn Snow        Whether to use symlinks to the target interpreter or not. If
328dd84028fSJohn Snow        left unspecified, it will use symlinks except on Windows to
329dd84028fSJohn Snow        match behavior with the "venv" CLI tool.
330dd84028fSJohn Snow    :param with_pip:
331dd84028fSJohn Snow        Whether to install "pip" binaries or not.
332dd84028fSJohn Snow    """
333dd84028fSJohn Snow    logger.debug(
334dd84028fSJohn Snow        "%s: make_venv(env_dir=%s, system_site_packages=%s, "
335dd84028fSJohn Snow        "clear=%s, symlinks=%s, with_pip=%s)",
336dd84028fSJohn Snow        __file__,
337dd84028fSJohn Snow        str(env_dir),
338dd84028fSJohn Snow        system_site_packages,
339dd84028fSJohn Snow        clear,
340dd84028fSJohn Snow        symlinks,
341dd84028fSJohn Snow        with_pip,
342dd84028fSJohn Snow    )
343dd84028fSJohn Snow
344dd84028fSJohn Snow    if symlinks is None:
345dd84028fSJohn Snow        # Default behavior of standard venv CLI
346dd84028fSJohn Snow        symlinks = os.name != "nt"
347dd84028fSJohn Snow
348dd84028fSJohn Snow    builder = QemuEnvBuilder(
349dd84028fSJohn Snow        system_site_packages=system_site_packages,
350dd84028fSJohn Snow        clear=clear,
351dd84028fSJohn Snow        symlinks=symlinks,
352dd84028fSJohn Snow        with_pip=with_pip,
353dd84028fSJohn Snow    )
354dd84028fSJohn Snow
355dd84028fSJohn Snow    style = "non-isolated" if builder.system_site_packages else "isolated"
356dee01b82SJohn Snow    nested = ""
357dee01b82SJohn Snow    if builder.use_parent_packages:
358dee01b82SJohn Snow        nested = f"(with packages from '{builder.get_parent_libpath()}') "
359dd84028fSJohn Snow    print(
360dd84028fSJohn Snow        f"mkvenv: Creating {style} virtual environment"
361dee01b82SJohn Snow        f" {nested}at '{str(env_dir)}'",
362dd84028fSJohn Snow        file=sys.stderr,
363dd84028fSJohn Snow    )
364dd84028fSJohn Snow
365dd84028fSJohn Snow    try:
366dd84028fSJohn Snow        logger.debug("Invoking builder.create()")
367dd84028fSJohn Snow        try:
368dd84028fSJohn Snow            builder.create(str(env_dir))
369dd84028fSJohn Snow        except SystemExit as exc:
370dd84028fSJohn Snow            # Some versions of the venv module raise SystemExit; *nasty*!
371dd84028fSJohn Snow            # We want the exception that prompted it. It might be a subprocess
372dd84028fSJohn Snow            # error that has output we *really* want to see.
373dd84028fSJohn Snow            logger.debug("Intercepted SystemExit from EnvBuilder.create()")
374dd84028fSJohn Snow            raise exc.__cause__ or exc.__context__ or exc
375dd84028fSJohn Snow        logger.debug("builder.create() finished")
376dd84028fSJohn Snow    except subprocess.CalledProcessError as exc:
377dd84028fSJohn Snow        logger.error("mkvenv subprocess failed:")
378dd84028fSJohn Snow        logger.error("cmd: %s", exc.cmd)
379dd84028fSJohn Snow        logger.error("returncode: %d", exc.returncode)
380dd84028fSJohn Snow
381dd84028fSJohn Snow        def _stringify(data: Union[str, bytes]) -> str:
382dd84028fSJohn Snow            if isinstance(data, bytes):
383dd84028fSJohn Snow                return data.decode()
384dd84028fSJohn Snow            return data
385dd84028fSJohn Snow
386dd84028fSJohn Snow        lines = []
387dd84028fSJohn Snow        if exc.stdout:
388dd84028fSJohn Snow            lines.append("========== stdout ==========")
389dd84028fSJohn Snow            lines.append(_stringify(exc.stdout))
390dd84028fSJohn Snow            lines.append("============================")
391dd84028fSJohn Snow        if exc.stderr:
392dd84028fSJohn Snow            lines.append("========== stderr ==========")
393dd84028fSJohn Snow            lines.append(_stringify(exc.stderr))
394dd84028fSJohn Snow            lines.append("============================")
395dd84028fSJohn Snow        if lines:
396dd84028fSJohn Snow            logger.error(os.linesep.join(lines))
397dd84028fSJohn Snow
398dd84028fSJohn Snow        raise Ouch("VENV creation subprocess failed.") from exc
399dd84028fSJohn Snow
400dd84028fSJohn Snow    # print the python executable to stdout for configure.
401dd84028fSJohn Snow    print(builder.get_value("env_exe"))
402dd84028fSJohn Snow
403dd84028fSJohn Snow
40492834894SJohn Snowdef _gen_importlib(packages: Sequence[str]) -> Iterator[str]:
40592834894SJohn Snow    # pylint: disable=import-outside-toplevel
40692834894SJohn Snow    # pylint: disable=no-name-in-module
40792834894SJohn Snow    # pylint: disable=import-error
40892834894SJohn Snow    try:
40992834894SJohn Snow        # First preference: Python 3.8+ stdlib
41092834894SJohn Snow        from importlib.metadata import (  # type: ignore
41192834894SJohn Snow            PackageNotFoundError,
41292834894SJohn Snow            distribution,
41392834894SJohn Snow        )
41492834894SJohn Snow    except ImportError as exc:
41592834894SJohn Snow        logger.debug("%s", str(exc))
41692834894SJohn Snow        # Second preference: Commonly available PyPI backport
41792834894SJohn Snow        from importlib_metadata import (  # type: ignore
41892834894SJohn Snow            PackageNotFoundError,
41992834894SJohn Snow            distribution,
42092834894SJohn Snow        )
42192834894SJohn Snow
42292834894SJohn Snow    def _generator() -> Iterator[str]:
42392834894SJohn Snow        for package in packages:
42492834894SJohn Snow            try:
42592834894SJohn Snow                entry_points = distribution(package).entry_points
42692834894SJohn Snow            except PackageNotFoundError:
42792834894SJohn Snow                continue
42892834894SJohn Snow
42992834894SJohn Snow            # The EntryPoints type is only available in 3.10+,
43092834894SJohn Snow            # treat this as a vanilla list and filter it ourselves.
43192834894SJohn Snow            entry_points = filter(
43292834894SJohn Snow                lambda ep: ep.group == "console_scripts", entry_points
43392834894SJohn Snow            )
43492834894SJohn Snow
43592834894SJohn Snow            for entry_point in entry_points:
43692834894SJohn Snow                yield f"{entry_point.name} = {entry_point.value}"
43792834894SJohn Snow
43892834894SJohn Snow    return _generator()
43992834894SJohn Snow
44092834894SJohn Snow
44192834894SJohn Snowdef _gen_pkg_resources(packages: Sequence[str]) -> Iterator[str]:
44292834894SJohn Snow    # pylint: disable=import-outside-toplevel
44392834894SJohn Snow    # Bundled with setuptools; has a good chance of being available.
44492834894SJohn Snow    import pkg_resources
44592834894SJohn Snow
44692834894SJohn Snow    def _generator() -> Iterator[str]:
44792834894SJohn Snow        for package in packages:
44892834894SJohn Snow            try:
44992834894SJohn Snow                eps = pkg_resources.get_entry_map(package, "console_scripts")
45092834894SJohn Snow            except pkg_resources.DistributionNotFound:
45192834894SJohn Snow                continue
45292834894SJohn Snow
45392834894SJohn Snow            for entry_point in eps.values():
45492834894SJohn Snow                yield str(entry_point)
45592834894SJohn Snow
45692834894SJohn Snow    return _generator()
45792834894SJohn Snow
45892834894SJohn Snow
45992834894SJohn Snowdef generate_console_scripts(
46092834894SJohn Snow    packages: Sequence[str],
46192834894SJohn Snow    python_path: Optional[str] = None,
46292834894SJohn Snow    bin_path: Optional[str] = None,
46392834894SJohn Snow) -> None:
46492834894SJohn Snow    """
46592834894SJohn Snow    Generate script shims for console_script entry points in @packages.
46692834894SJohn Snow    """
46792834894SJohn Snow    if python_path is None:
46892834894SJohn Snow        python_path = sys.executable
46992834894SJohn Snow    if bin_path is None:
47092834894SJohn Snow        bin_path = sysconfig.get_path("scripts")
47192834894SJohn Snow        assert bin_path is not None
47292834894SJohn Snow
47392834894SJohn Snow    logger.debug(
47492834894SJohn Snow        "generate_console_scripts(packages=%s, python_path=%s, bin_path=%s)",
47592834894SJohn Snow        packages,
47692834894SJohn Snow        python_path,
47792834894SJohn Snow        bin_path,
47892834894SJohn Snow    )
47992834894SJohn Snow
48092834894SJohn Snow    if not packages:
48192834894SJohn Snow        return
48292834894SJohn Snow
48392834894SJohn Snow    def _get_entry_points() -> Iterator[str]:
48492834894SJohn Snow        """Python 3.7 compatibility shim for iterating entry points."""
48592834894SJohn Snow        # Python 3.8+, or Python 3.7 with importlib_metadata installed.
48692834894SJohn Snow        try:
48792834894SJohn Snow            return _gen_importlib(packages)
48892834894SJohn Snow        except ImportError as exc:
48992834894SJohn Snow            logger.debug("%s", str(exc))
49092834894SJohn Snow
49192834894SJohn Snow        # Python 3.7 with setuptools installed.
49292834894SJohn Snow        try:
49392834894SJohn Snow            return _gen_pkg_resources(packages)
49492834894SJohn Snow        except ImportError as exc:
49592834894SJohn Snow            logger.debug("%s", str(exc))
49692834894SJohn Snow            raise Ouch(
49792834894SJohn Snow                "Neither importlib.metadata nor pkg_resources found, "
49892834894SJohn Snow                "can't generate console script shims.\n"
49992834894SJohn Snow                "Use Python 3.8+, or install importlib-metadata or setuptools."
50092834894SJohn Snow            ) from exc
50192834894SJohn Snow
50292834894SJohn Snow    maker = distlib.scripts.ScriptMaker(None, bin_path)
50392834894SJohn Snow    maker.variants = {""}
50492834894SJohn Snow    maker.clobber = False
50592834894SJohn Snow
50692834894SJohn Snow    for entry_point in _get_entry_points():
50792834894SJohn Snow        for filename in maker.make(entry_point):
50892834894SJohn Snow            logger.debug("wrote console_script '%s'", filename)
50992834894SJohn Snow
51092834894SJohn Snow
511c8049626SJohn Snowdef checkpip() -> bool:
512c8049626SJohn Snow    """
513c8049626SJohn Snow    Debian10 has a pip that's broken when used inside of a virtual environment.
514c8049626SJohn Snow
515c8049626SJohn Snow    We try to detect and correct that case here.
516c8049626SJohn Snow    """
517c8049626SJohn Snow    try:
518c8049626SJohn Snow        # pylint: disable=import-outside-toplevel,unused-import,import-error
519c8049626SJohn Snow        # pylint: disable=redefined-outer-name
520c8049626SJohn Snow        import pip._internal  # type: ignore  # noqa: F401
521c8049626SJohn Snow
522c8049626SJohn Snow        logger.debug("pip appears to be working correctly.")
523c8049626SJohn Snow        return False
524c8049626SJohn Snow    except ModuleNotFoundError as exc:
525c8049626SJohn Snow        if exc.name == "pip._internal":
526c8049626SJohn Snow            # Uh, fair enough. They did say "internal".
527c8049626SJohn Snow            # Let's just assume it's fine.
528c8049626SJohn Snow            return False
529c8049626SJohn Snow        logger.warning("pip appears to be malfunctioning: %s", str(exc))
530c8049626SJohn Snow
531c8049626SJohn Snow    check_ensurepip("pip appears to be non-functional, and ")
532c8049626SJohn Snow
533c8049626SJohn Snow    logger.debug("Attempting to repair pip ...")
534c8049626SJohn Snow    subprocess.run(
535c8049626SJohn Snow        (sys.executable, "-m", "ensurepip"),
536c8049626SJohn Snow        stdout=subprocess.DEVNULL,
537c8049626SJohn Snow        check=True,
538c8049626SJohn Snow    )
539c8049626SJohn Snow    logger.debug("Pip is now (hopefully) repaired!")
540c8049626SJohn Snow    return True
541c8049626SJohn Snow
542c8049626SJohn Snow
5434695a22eSJohn Snowdef pkgname_from_depspec(dep_spec: str) -> str:
5444695a22eSJohn Snow    """
5454695a22eSJohn Snow    Parse package name out of a PEP-508 depspec.
5464695a22eSJohn Snow
5474695a22eSJohn Snow    See https://peps.python.org/pep-0508/#names
5484695a22eSJohn Snow    """
5494695a22eSJohn Snow    match = re.match(
5504695a22eSJohn Snow        r"^([A-Z0-9]([A-Z0-9._-]*[A-Z0-9])?)", dep_spec, re.IGNORECASE
5514695a22eSJohn Snow    )
5524695a22eSJohn Snow    if not match:
5534695a22eSJohn Snow        raise ValueError(
5544695a22eSJohn Snow            f"dep_spec '{dep_spec}'"
5554695a22eSJohn Snow            " does not appear to contain a valid package name"
5564695a22eSJohn Snow        )
5574695a22eSJohn Snow    return match.group(0)
5584695a22eSJohn Snow
5594695a22eSJohn Snow
56047a90a51SPaolo Bonzinidef _get_path_importlib(package: str) -> Optional[str]:
56147a90a51SPaolo Bonzini    # pylint: disable=import-outside-toplevel
56247a90a51SPaolo Bonzini    # pylint: disable=no-name-in-module
56347a90a51SPaolo Bonzini    # pylint: disable=import-error
56447a90a51SPaolo Bonzini    try:
56547a90a51SPaolo Bonzini        # First preference: Python 3.8+ stdlib
56647a90a51SPaolo Bonzini        from importlib.metadata import (  # type: ignore
56747a90a51SPaolo Bonzini            PackageNotFoundError,
56847a90a51SPaolo Bonzini            distribution,
56947a90a51SPaolo Bonzini        )
57047a90a51SPaolo Bonzini    except ImportError as exc:
57147a90a51SPaolo Bonzini        logger.debug("%s", str(exc))
57247a90a51SPaolo Bonzini        # Second preference: Commonly available PyPI backport
57347a90a51SPaolo Bonzini        from importlib_metadata import (  # type: ignore
57447a90a51SPaolo Bonzini            PackageNotFoundError,
57547a90a51SPaolo Bonzini            distribution,
57647a90a51SPaolo Bonzini        )
57747a90a51SPaolo Bonzini
57847a90a51SPaolo Bonzini    try:
57947a90a51SPaolo Bonzini        return str(distribution(package).locate_file("."))
58047a90a51SPaolo Bonzini    except PackageNotFoundError:
58147a90a51SPaolo Bonzini        return None
58247a90a51SPaolo Bonzini
58347a90a51SPaolo Bonzini
58447a90a51SPaolo Bonzinidef _get_path_pkg_resources(package: str) -> Optional[str]:
58547a90a51SPaolo Bonzini    # pylint: disable=import-outside-toplevel
58647a90a51SPaolo Bonzini    # Bundled with setuptools; has a good chance of being available.
58747a90a51SPaolo Bonzini    import pkg_resources
58847a90a51SPaolo Bonzini
58947a90a51SPaolo Bonzini    try:
59047a90a51SPaolo Bonzini        return str(pkg_resources.get_distribution(package).location)
59147a90a51SPaolo Bonzini    except pkg_resources.DistributionNotFound:
59247a90a51SPaolo Bonzini        return None
59347a90a51SPaolo Bonzini
59447a90a51SPaolo Bonzini
59547a90a51SPaolo Bonzinidef _get_path(package: str) -> Optional[str]:
59647a90a51SPaolo Bonzini    try:
59747a90a51SPaolo Bonzini        return _get_path_importlib(package)
59847a90a51SPaolo Bonzini    except ImportError as exc:
59947a90a51SPaolo Bonzini        logger.debug("%s", str(exc))
60047a90a51SPaolo Bonzini
60147a90a51SPaolo Bonzini    try:
60247a90a51SPaolo Bonzini        return _get_path_pkg_resources(package)
60347a90a51SPaolo Bonzini    except ImportError as exc:
60447a90a51SPaolo Bonzini        logger.debug("%s", str(exc))
60547a90a51SPaolo Bonzini        raise Ouch(
60647a90a51SPaolo Bonzini            "Neither importlib.metadata nor pkg_resources found. "
60747a90a51SPaolo Bonzini            "Use Python 3.8+, or install importlib-metadata or setuptools."
60847a90a51SPaolo Bonzini        ) from exc
60947a90a51SPaolo Bonzini
61047a90a51SPaolo Bonzini
61147a90a51SPaolo Bonzinidef _path_is_prefix(prefix: Optional[str], path: str) -> bool:
61247a90a51SPaolo Bonzini    try:
61347a90a51SPaolo Bonzini        return (
61447a90a51SPaolo Bonzini            prefix is not None and os.path.commonpath([prefix, path]) == prefix
61547a90a51SPaolo Bonzini        )
61647a90a51SPaolo Bonzini    except ValueError:
61747a90a51SPaolo Bonzini        return False
61847a90a51SPaolo Bonzini
61947a90a51SPaolo Bonzini
62047a90a51SPaolo Bonzinidef _is_system_package(package: str) -> bool:
62147a90a51SPaolo Bonzini    path = _get_path(package)
62247a90a51SPaolo Bonzini    return path is not None and not (
62347a90a51SPaolo Bonzini        _path_is_prefix(sysconfig.get_path("purelib"), path)
62447a90a51SPaolo Bonzini        or _path_is_prefix(sysconfig.get_path("platlib"), path)
62547a90a51SPaolo Bonzini    )
62647a90a51SPaolo Bonzini
62747a90a51SPaolo Bonzini
628c673f3d0SPaolo Bonzinidef _get_version_importlib(package: str) -> Optional[str]:
629c673f3d0SPaolo Bonzini    # pylint: disable=import-outside-toplevel
630c673f3d0SPaolo Bonzini    # pylint: disable=no-name-in-module
631c673f3d0SPaolo Bonzini    # pylint: disable=import-error
632c673f3d0SPaolo Bonzini    try:
633c673f3d0SPaolo Bonzini        # First preference: Python 3.8+ stdlib
634c673f3d0SPaolo Bonzini        from importlib.metadata import (  # type: ignore
635c673f3d0SPaolo Bonzini            PackageNotFoundError,
636c673f3d0SPaolo Bonzini            distribution,
637c673f3d0SPaolo Bonzini        )
638c673f3d0SPaolo Bonzini    except ImportError as exc:
639c673f3d0SPaolo Bonzini        logger.debug("%s", str(exc))
640c673f3d0SPaolo Bonzini        # Second preference: Commonly available PyPI backport
641c673f3d0SPaolo Bonzini        from importlib_metadata import (  # type: ignore
642c673f3d0SPaolo Bonzini            PackageNotFoundError,
643c673f3d0SPaolo Bonzini            distribution,
644c673f3d0SPaolo Bonzini        )
645c673f3d0SPaolo Bonzini
646c673f3d0SPaolo Bonzini    try:
647c673f3d0SPaolo Bonzini        return str(distribution(package).version)
648c673f3d0SPaolo Bonzini    except PackageNotFoundError:
649c673f3d0SPaolo Bonzini        return None
650c673f3d0SPaolo Bonzini
651c673f3d0SPaolo Bonzini
652c673f3d0SPaolo Bonzinidef _get_version_pkg_resources(package: str) -> Optional[str]:
653c673f3d0SPaolo Bonzini    # pylint: disable=import-outside-toplevel
654c673f3d0SPaolo Bonzini    # Bundled with setuptools; has a good chance of being available.
655c673f3d0SPaolo Bonzini    import pkg_resources
656c673f3d0SPaolo Bonzini
657c673f3d0SPaolo Bonzini    try:
658c673f3d0SPaolo Bonzini        return str(pkg_resources.get_distribution(package).version)
659c673f3d0SPaolo Bonzini    except pkg_resources.DistributionNotFound:
660c673f3d0SPaolo Bonzini        return None
661c673f3d0SPaolo Bonzini
662c673f3d0SPaolo Bonzini
663c673f3d0SPaolo Bonzinidef _get_version(package: str) -> Optional[str]:
664c673f3d0SPaolo Bonzini    try:
665c673f3d0SPaolo Bonzini        return _get_version_importlib(package)
666c673f3d0SPaolo Bonzini    except ImportError as exc:
667c673f3d0SPaolo Bonzini        logger.debug("%s", str(exc))
668c673f3d0SPaolo Bonzini
669c673f3d0SPaolo Bonzini    try:
670c673f3d0SPaolo Bonzini        return _get_version_pkg_resources(package)
671c673f3d0SPaolo Bonzini    except ImportError as exc:
672c673f3d0SPaolo Bonzini        logger.debug("%s", str(exc))
673c673f3d0SPaolo Bonzini        raise Ouch(
674c673f3d0SPaolo Bonzini            "Neither importlib.metadata nor pkg_resources found. "
675c673f3d0SPaolo Bonzini            "Use Python 3.8+, or install importlib-metadata or setuptools."
676c673f3d0SPaolo Bonzini        ) from exc
677c673f3d0SPaolo Bonzini
678c673f3d0SPaolo Bonzini
6794695a22eSJohn Snowdef diagnose(
6804695a22eSJohn Snow    dep_spec: str,
6814695a22eSJohn Snow    online: bool,
6824695a22eSJohn Snow    wheels_dir: Optional[Union[str, Path]],
6834695a22eSJohn Snow    prog: Optional[str],
6844695a22eSJohn Snow) -> Tuple[str, bool]:
6854695a22eSJohn Snow    """
6864695a22eSJohn Snow    Offer a summary to the user as to why a package failed to be installed.
6874695a22eSJohn Snow
6884695a22eSJohn Snow    :param dep_spec: The package we tried to ensure, e.g. 'meson>=0.61.5'
6894695a22eSJohn Snow    :param online: Did we allow PyPI access?
6904695a22eSJohn Snow    :param prog:
6914695a22eSJohn Snow        Optionally, a shell program name that can be used as a
6924695a22eSJohn Snow        bellwether to detect if this program is installed elsewhere on
6934695a22eSJohn Snow        the system. This is used to offer advice when a program is
6944695a22eSJohn Snow        detected for a different python version.
6954695a22eSJohn Snow    :param wheels_dir:
6964695a22eSJohn Snow        Optionally, a directory that was searched for vendored packages.
6974695a22eSJohn Snow    """
6984695a22eSJohn Snow    # pylint: disable=too-many-branches
6994695a22eSJohn Snow
7004695a22eSJohn Snow    # Some errors are not particularly serious
7014695a22eSJohn Snow    bad = False
7024695a22eSJohn Snow
7034695a22eSJohn Snow    pkg_name = pkgname_from_depspec(dep_spec)
704c673f3d0SPaolo Bonzini    pkg_version = _get_version(pkg_name)
7054695a22eSJohn Snow
7064695a22eSJohn Snow    lines = []
7074695a22eSJohn Snow
7084695a22eSJohn Snow    if pkg_version:
7094695a22eSJohn Snow        lines.append(
7104695a22eSJohn Snow            f"Python package '{pkg_name}' version '{pkg_version}' was found,"
7114695a22eSJohn Snow            " but isn't suitable."
7124695a22eSJohn Snow        )
7134695a22eSJohn Snow    else:
7144695a22eSJohn Snow        lines.append(
715c673f3d0SPaolo Bonzini            f"Python package '{pkg_name}' was not found nor installed."
7164695a22eSJohn Snow        )
7174695a22eSJohn Snow
7184695a22eSJohn Snow    if wheels_dir:
7194695a22eSJohn Snow        lines.append(
7204695a22eSJohn Snow            "No suitable version found in, or failed to install from"
7214695a22eSJohn Snow            f" '{wheels_dir}'."
7224695a22eSJohn Snow        )
7234695a22eSJohn Snow        bad = True
7244695a22eSJohn Snow
7254695a22eSJohn Snow    if online:
7264695a22eSJohn Snow        lines.append("A suitable version could not be obtained from PyPI.")
7274695a22eSJohn Snow        bad = True
7284695a22eSJohn Snow    else:
7294695a22eSJohn Snow        lines.append(
7304695a22eSJohn Snow            "mkvenv was configured to operate offline and did not check PyPI."
7314695a22eSJohn Snow        )
7324695a22eSJohn Snow
7334695a22eSJohn Snow    if prog and not pkg_version:
7344695a22eSJohn Snow        which = shutil.which(prog)
7354695a22eSJohn Snow        if which:
7364695a22eSJohn Snow            if sys.base_prefix in site.PREFIXES:
7374695a22eSJohn Snow                pypath = Path(sys.executable).resolve()
7384695a22eSJohn Snow                lines.append(
7394695a22eSJohn Snow                    f"'{prog}' was detected on your system at '{which}', "
7404695a22eSJohn Snow                    f"but the Python package '{pkg_name}' was not found by "
7414695a22eSJohn Snow                    f"this Python interpreter ('{pypath}'). "
7424695a22eSJohn Snow                    f"Typically this means that '{prog}' has been installed "
7434695a22eSJohn Snow                    "against a different Python interpreter on your system."
7444695a22eSJohn Snow                )
7454695a22eSJohn Snow            else:
7464695a22eSJohn Snow                lines.append(
7474695a22eSJohn Snow                    f"'{prog}' was detected on your system at '{which}', "
7484695a22eSJohn Snow                    "but the build is using an isolated virtual environment."
7494695a22eSJohn Snow                )
7504695a22eSJohn Snow            bad = True
7514695a22eSJohn Snow
7524695a22eSJohn Snow    lines = [f" • {line}" for line in lines]
7534695a22eSJohn Snow    if bad:
7544695a22eSJohn Snow        lines.insert(0, f"Could not provide build dependency '{dep_spec}':")
7554695a22eSJohn Snow    else:
7564695a22eSJohn Snow        lines.insert(0, f"'{dep_spec}' not found:")
7574695a22eSJohn Snow    return os.linesep.join(lines), bad
7584695a22eSJohn Snow
7594695a22eSJohn Snow
760c5538eedSJohn Snowdef pip_install(
761c5538eedSJohn Snow    args: Sequence[str],
762c5538eedSJohn Snow    online: bool = False,
763c5538eedSJohn Snow    wheels_dir: Optional[Union[str, Path]] = None,
764c5538eedSJohn Snow) -> None:
765c5538eedSJohn Snow    """
766c5538eedSJohn Snow    Use pip to install a package or package(s) as specified in @args.
767c5538eedSJohn Snow    """
768c5538eedSJohn Snow    loud = bool(
769c5538eedSJohn Snow        os.environ.get("DEBUG")
770c5538eedSJohn Snow        or os.environ.get("GITLAB_CI")
771c5538eedSJohn Snow        or os.environ.get("V")
772c5538eedSJohn Snow    )
773c5538eedSJohn Snow
774c5538eedSJohn Snow    full_args = [
775c5538eedSJohn Snow        sys.executable,
776c5538eedSJohn Snow        "-m",
777c5538eedSJohn Snow        "pip",
778c5538eedSJohn Snow        "install",
779c5538eedSJohn Snow        "--disable-pip-version-check",
780c5538eedSJohn Snow        "-v" if loud else "-q",
781c5538eedSJohn Snow    ]
782c5538eedSJohn Snow    if not online:
783c5538eedSJohn Snow        full_args += ["--no-index"]
784c5538eedSJohn Snow    if wheels_dir:
785c5538eedSJohn Snow        full_args += ["--find-links", f"file://{str(wheels_dir)}"]
786c5538eedSJohn Snow    full_args += list(args)
787c5538eedSJohn Snow    subprocess.run(
788c5538eedSJohn Snow        full_args,
789c5538eedSJohn Snow        check=True,
790c5538eedSJohn Snow    )
791c5538eedSJohn Snow
792c5538eedSJohn Snow
793*0f1ec070SPaolo Bonzinidef _make_version_constraint(info: Dict[str, str], install: bool) -> str:
794*0f1ec070SPaolo Bonzini    """
795*0f1ec070SPaolo Bonzini    Construct the version constraint part of a PEP 508 dependency
796*0f1ec070SPaolo Bonzini    specification (for example '>=0.61.5') from the accepted and
797*0f1ec070SPaolo Bonzini    installed keys of the provided dictionary.
798*0f1ec070SPaolo Bonzini
799*0f1ec070SPaolo Bonzini    :param info: A dictionary corresponding to a TOML key-value list.
800*0f1ec070SPaolo Bonzini    :param install: True generates install constraints, False generates
801*0f1ec070SPaolo Bonzini        presence constraints
802*0f1ec070SPaolo Bonzini    """
803*0f1ec070SPaolo Bonzini    if install and "installed" in info:
804*0f1ec070SPaolo Bonzini        return "==" + info["installed"]
805*0f1ec070SPaolo Bonzini
806*0f1ec070SPaolo Bonzini    dep_spec = info.get("accepted", "")
807*0f1ec070SPaolo Bonzini    dep_spec = dep_spec.strip()
808*0f1ec070SPaolo Bonzini    # Double check that they didn't just use a version number
809*0f1ec070SPaolo Bonzini    if dep_spec and dep_spec[0] not in "!~><=(":
810*0f1ec070SPaolo Bonzini        raise Ouch(
811*0f1ec070SPaolo Bonzini            "invalid dependency specifier " + dep_spec + " in dependency file"
812*0f1ec070SPaolo Bonzini        )
813*0f1ec070SPaolo Bonzini
814*0f1ec070SPaolo Bonzini    return dep_spec
815*0f1ec070SPaolo Bonzini
816*0f1ec070SPaolo Bonzini
8174695a22eSJohn Snowdef _do_ensure(
818*0f1ec070SPaolo Bonzini    group: Dict[str, Dict[str, str]],
819c5538eedSJohn Snow    online: bool = False,
820c5538eedSJohn Snow    wheels_dir: Optional[Union[str, Path]] = None,
821d37c21b5SPaolo Bonzini) -> Optional[Tuple[str, bool]]:
822c5538eedSJohn Snow    """
823*0f1ec070SPaolo Bonzini    Use pip to ensure we have the packages specified in @group.
824c5538eedSJohn Snow
825*0f1ec070SPaolo Bonzini    If the packages are already installed, do nothing. If online and
826c5538eedSJohn Snow    wheels_dir are both provided, prefer packages found in wheels_dir
827c5538eedSJohn Snow    first before connecting to PyPI.
828c5538eedSJohn Snow
829*0f1ec070SPaolo Bonzini    :param group: A dictionary of dictionaries, corresponding to a
830*0f1ec070SPaolo Bonzini        section in a pythondeps.toml file.
831c5538eedSJohn Snow    :param online: If True, fall back to PyPI.
832c5538eedSJohn Snow    :param wheels_dir: If specified, search this path for packages.
833c5538eedSJohn Snow    """
834c5538eedSJohn Snow    absent = []
83592834894SJohn Snow    present = []
83667b9a83dSPaolo Bonzini    canary = None
837*0f1ec070SPaolo Bonzini    for name, info in group.items():
838*0f1ec070SPaolo Bonzini        constraint = _make_version_constraint(info, False)
839*0f1ec070SPaolo Bonzini        matcher = distlib.version.LegacyMatcher(name + constraint)
840*0f1ec070SPaolo Bonzini        ver = _get_version(name)
84147a90a51SPaolo Bonzini        if (
84247a90a51SPaolo Bonzini            ver is None
84347a90a51SPaolo Bonzini            # Always pass installed package to pip, so that they can be
84447a90a51SPaolo Bonzini            # updated if the requested version changes
845*0f1ec070SPaolo Bonzini            or not _is_system_package(name)
84647a90a51SPaolo Bonzini            or not matcher.match(distlib.version.LegacyVersion(ver))
847c673f3d0SPaolo Bonzini        ):
848*0f1ec070SPaolo Bonzini            absent.append(name + _make_version_constraint(info, True))
849*0f1ec070SPaolo Bonzini            if len(absent) == 1:
850*0f1ec070SPaolo Bonzini                canary = info.get("canary", None)
851c5538eedSJohn Snow        else:
852*0f1ec070SPaolo Bonzini            logger.info("found %s %s", name, ver)
853*0f1ec070SPaolo Bonzini            present.append(name)
85492834894SJohn Snow
85592834894SJohn Snow    if present:
85692834894SJohn Snow        generate_console_scripts(present)
857c5538eedSJohn Snow
858c5538eedSJohn Snow    if absent:
859d37c21b5SPaolo Bonzini        if online or wheels_dir:
860c5538eedSJohn Snow            # Some packages are missing or aren't a suitable version,
861c5538eedSJohn Snow            # install a suitable (possibly vendored) package.
862c5538eedSJohn Snow            print(f"mkvenv: installing {', '.join(absent)}", file=sys.stderr)
863d37c21b5SPaolo Bonzini            try:
864c5538eedSJohn Snow                pip_install(args=absent, online=online, wheels_dir=wheels_dir)
865d37c21b5SPaolo Bonzini                return None
866d37c21b5SPaolo Bonzini            except subprocess.CalledProcessError:
867d37c21b5SPaolo Bonzini                pass
868d37c21b5SPaolo Bonzini
869d37c21b5SPaolo Bonzini        return diagnose(
870d37c21b5SPaolo Bonzini            absent[0],
871d37c21b5SPaolo Bonzini            online,
872d37c21b5SPaolo Bonzini            wheels_dir,
87367b9a83dSPaolo Bonzini            canary,
874d37c21b5SPaolo Bonzini        )
875d37c21b5SPaolo Bonzini
876d37c21b5SPaolo Bonzini    return None
877c5538eedSJohn Snow
878c5538eedSJohn Snow
8794695a22eSJohn Snowdef ensure(
8804695a22eSJohn Snow    dep_specs: Sequence[str],
8814695a22eSJohn Snow    online: bool = False,
8824695a22eSJohn Snow    wheels_dir: Optional[Union[str, Path]] = None,
8834695a22eSJohn Snow    prog: Optional[str] = None,
8844695a22eSJohn Snow) -> None:
8854695a22eSJohn Snow    """
8864695a22eSJohn Snow    Use pip to ensure we have the package specified by @dep_specs.
8874695a22eSJohn Snow
8884695a22eSJohn Snow    If the package is already installed, do nothing. If online and
8894695a22eSJohn Snow    wheels_dir are both provided, prefer packages found in wheels_dir
8904695a22eSJohn Snow    first before connecting to PyPI.
8914695a22eSJohn Snow
8924695a22eSJohn Snow    :param dep_specs:
8934695a22eSJohn Snow        PEP 508 dependency specifications. e.g. ['meson>=0.61.5'].
8944695a22eSJohn Snow    :param online: If True, fall back to PyPI.
8954695a22eSJohn Snow    :param wheels_dir: If specified, search this path for packages.
8964695a22eSJohn Snow    :param prog:
8974695a22eSJohn Snow        If specified, use this program name for error diagnostics that will
8984695a22eSJohn Snow        be presented to the user. e.g., 'sphinx-build' can be used as a
8994695a22eSJohn Snow        bellwether for the presence of 'sphinx'.
9004695a22eSJohn Snow    """
9014695a22eSJohn Snow    print(f"mkvenv: checking for {', '.join(dep_specs)}", file=sys.stderr)
90268ea6d17SJohn Snow
90368ea6d17SJohn Snow    if not HAVE_DISTLIB:
90468ea6d17SJohn Snow        raise Ouch("a usable distlib could not be found, please install it")
90568ea6d17SJohn Snow
906*0f1ec070SPaolo Bonzini    # Convert the depspecs to a dictionary, as if they came
907*0f1ec070SPaolo Bonzini    # from a section in a pythondeps.toml file
908*0f1ec070SPaolo Bonzini    group: Dict[str, Dict[str, str]] = {}
909*0f1ec070SPaolo Bonzini    for spec in dep_specs:
910*0f1ec070SPaolo Bonzini        name = distlib.version.LegacyMatcher(spec).name
911*0f1ec070SPaolo Bonzini        group[name] = {}
912*0f1ec070SPaolo Bonzini
913*0f1ec070SPaolo Bonzini        spec = spec.strip()
914*0f1ec070SPaolo Bonzini        pos = len(name)
915*0f1ec070SPaolo Bonzini        ver = spec[pos:].strip()
916*0f1ec070SPaolo Bonzini        if ver:
917*0f1ec070SPaolo Bonzini            group[name]["accepted"] = ver
918*0f1ec070SPaolo Bonzini
919*0f1ec070SPaolo Bonzini        if prog:
920*0f1ec070SPaolo Bonzini            group[name]["canary"] = prog
921*0f1ec070SPaolo Bonzini            prog = None
922*0f1ec070SPaolo Bonzini
923*0f1ec070SPaolo Bonzini    result = _do_ensure(group, online, wheels_dir)
924d37c21b5SPaolo Bonzini    if result:
9254695a22eSJohn Snow        # Well, that's not good.
926d37c21b5SPaolo Bonzini        if result[1]:
927d37c21b5SPaolo Bonzini            raise Ouch(result[0])
928d37c21b5SPaolo Bonzini        raise SystemExit(f"\n{result[0]}\n\n")
9294695a22eSJohn Snow
9304695a22eSJohn Snow
931f1ad527fSJohn Snowdef post_venv_setup() -> None:
932f1ad527fSJohn Snow    """
933f1ad527fSJohn Snow    This is intended to be run *inside the venv* after it is created.
934f1ad527fSJohn Snow    """
935f1ad527fSJohn Snow    logger.debug("post_venv_setup()")
936c8049626SJohn Snow    # Test for a broken pip (Debian 10 or derivative?) and fix it if needed
937c8049626SJohn Snow    if not checkpip():
938c8049626SJohn Snow        # Finally, generate a 'pip' script so the venv is usable in a normal
939f1ad527fSJohn Snow        # way from the CLI. This only happens when we inherited pip from a
940f1ad527fSJohn Snow        # parent/system-site and haven't run ensurepip in some way.
941f1ad527fSJohn Snow        generate_console_scripts(["pip"])
942f1ad527fSJohn Snow
943f1ad527fSJohn Snow
944dd84028fSJohn Snowdef _add_create_subcommand(subparsers: Any) -> None:
945dd84028fSJohn Snow    subparser = subparsers.add_parser("create", help="create a venv")
946dd84028fSJohn Snow    subparser.add_argument(
947dd84028fSJohn Snow        "target",
948dd84028fSJohn Snow        type=str,
949dd84028fSJohn Snow        action="store",
950dd84028fSJohn Snow        help="Target directory to install virtual environment into.",
951dd84028fSJohn Snow    )
952dd84028fSJohn Snow
953dd84028fSJohn Snow
954f1ad527fSJohn Snowdef _add_post_init_subcommand(subparsers: Any) -> None:
955f1ad527fSJohn Snow    subparsers.add_parser("post_init", help="post-venv initialization")
956f1ad527fSJohn Snow
957f1ad527fSJohn Snow
958c5538eedSJohn Snowdef _add_ensure_subcommand(subparsers: Any) -> None:
959c5538eedSJohn Snow    subparser = subparsers.add_parser(
960c5538eedSJohn Snow        "ensure", help="Ensure that the specified package is installed."
961c5538eedSJohn Snow    )
962c5538eedSJohn Snow    subparser.add_argument(
963c5538eedSJohn Snow        "--online",
964c5538eedSJohn Snow        action="store_true",
965c5538eedSJohn Snow        help="Install packages from PyPI, if necessary.",
966c5538eedSJohn Snow    )
967c5538eedSJohn Snow    subparser.add_argument(
968c5538eedSJohn Snow        "--dir",
969c5538eedSJohn Snow        type=str,
970c5538eedSJohn Snow        action="store",
971c5538eedSJohn Snow        help="Path to vendored packages where we may install from.",
972c5538eedSJohn Snow    )
973c5538eedSJohn Snow    subparser.add_argument(
9744695a22eSJohn Snow        "--diagnose",
9754695a22eSJohn Snow        type=str,
9764695a22eSJohn Snow        action="store",
9774695a22eSJohn Snow        help=(
9784695a22eSJohn Snow            "Name of a shell utility to use for "
9794695a22eSJohn Snow            "diagnostics if this command fails."
9804695a22eSJohn Snow        ),
9814695a22eSJohn Snow    )
9824695a22eSJohn Snow    subparser.add_argument(
983c5538eedSJohn Snow        "dep_specs",
984c5538eedSJohn Snow        type=str,
985c5538eedSJohn Snow        action="store",
986c5538eedSJohn Snow        help="PEP 508 Dependency specification, e.g. 'meson>=0.61.5'",
987c5538eedSJohn Snow        nargs="+",
988c5538eedSJohn Snow    )
989c5538eedSJohn Snow
990c5538eedSJohn Snow
991dd84028fSJohn Snowdef main() -> int:
992dd84028fSJohn Snow    """CLI interface to make_qemu_venv. See module docstring."""
993dd84028fSJohn Snow    if os.environ.get("DEBUG") or os.environ.get("GITLAB_CI"):
994dd84028fSJohn Snow        # You're welcome.
995dd84028fSJohn Snow        logging.basicConfig(level=logging.DEBUG)
996c5538eedSJohn Snow    else:
997c5538eedSJohn Snow        if os.environ.get("V"):
998dd84028fSJohn Snow            logging.basicConfig(level=logging.INFO)
999dd84028fSJohn Snow
1000dd84028fSJohn Snow    parser = argparse.ArgumentParser(
1001dd84028fSJohn Snow        prog="mkvenv",
1002dd84028fSJohn Snow        description="QEMU pyvenv bootstrapping utility",
1003dd84028fSJohn Snow    )
1004dd84028fSJohn Snow    subparsers = parser.add_subparsers(
1005dd84028fSJohn Snow        title="Commands",
1006dd84028fSJohn Snow        dest="command",
100702312f1aSPaolo Bonzini        required=True,
1008dd84028fSJohn Snow        metavar="command",
1009dd84028fSJohn Snow        help="Description",
1010dd84028fSJohn Snow    )
1011dd84028fSJohn Snow
1012dd84028fSJohn Snow    _add_create_subcommand(subparsers)
1013f1ad527fSJohn Snow    _add_post_init_subcommand(subparsers)
1014c5538eedSJohn Snow    _add_ensure_subcommand(subparsers)
1015dd84028fSJohn Snow
1016dd84028fSJohn Snow    args = parser.parse_args()
1017dd84028fSJohn Snow    try:
1018dd84028fSJohn Snow        if args.command == "create":
1019dd84028fSJohn Snow            make_venv(
1020dd84028fSJohn Snow                args.target,
1021dd84028fSJohn Snow                system_site_packages=True,
1022dd84028fSJohn Snow                clear=True,
1023dd84028fSJohn Snow            )
1024f1ad527fSJohn Snow        if args.command == "post_init":
1025f1ad527fSJohn Snow            post_venv_setup()
1026c5538eedSJohn Snow        if args.command == "ensure":
1027c5538eedSJohn Snow            ensure(
1028c5538eedSJohn Snow                dep_specs=args.dep_specs,
1029c5538eedSJohn Snow                online=args.online,
1030c5538eedSJohn Snow                wheels_dir=args.dir,
10314695a22eSJohn Snow                prog=args.diagnose,
1032c5538eedSJohn Snow            )
1033dd84028fSJohn Snow        logger.debug("mkvenv.py %s: exiting", args.command)
1034dd84028fSJohn Snow    except Ouch as exc:
1035dd84028fSJohn Snow        print("\n*** Ouch! ***\n", file=sys.stderr)
1036dd84028fSJohn Snow        print(str(exc), "\n\n", file=sys.stderr)
1037dd84028fSJohn Snow        return 1
1038dd84028fSJohn Snow    except SystemExit:
1039dd84028fSJohn Snow        raise
1040dd84028fSJohn Snow    except:  # pylint: disable=bare-except
1041dd84028fSJohn Snow        logger.exception("mkvenv did not complete successfully:")
1042dd84028fSJohn Snow        return 2
1043dd84028fSJohn Snow    return 0
1044dd84028fSJohn Snow
1045dd84028fSJohn Snow
1046dd84028fSJohn Snowif __name__ == "__main__":
1047dd84028fSJohn Snow    sys.exit(main())
1048