xref: /qemu/python/scripts/mkvenv.py (revision 0a88ac9662950cecac74b5de3056071a964e4fc4)
1dd84028fSJohn Snow"""
2dd84028fSJohn Snowmkvenv - QEMU pyvenv bootstrapping utility
3dd84028fSJohn Snow
4dd84028fSJohn Snowusage: mkvenv [-h] command ...
5dd84028fSJohn Snow
6dd84028fSJohn SnowQEMU pyvenv bootstrapping utility
7dd84028fSJohn Snow
8dd84028fSJohn Snowoptions:
9dd84028fSJohn Snow  -h, --help  show this help message and exit
10dd84028fSJohn Snow
11dd84028fSJohn SnowCommands:
12dd84028fSJohn Snow  command     Description
13dd84028fSJohn Snow    create    create a venv
14f1ad527fSJohn Snow    post_init
15f1ad527fSJohn Snow              post-venv initialization
16c5538eedSJohn Snow    ensure    Ensure that the specified package is installed.
1771ed611cSPaolo Bonzini    ensuregroup
1871ed611cSPaolo Bonzini              Ensure that the specified package group is installed.
19dd84028fSJohn Snow
20dd84028fSJohn Snow--------------------------------------------------
21dd84028fSJohn Snow
22dd84028fSJohn Snowusage: mkvenv create [-h] target
23dd84028fSJohn Snow
24dd84028fSJohn Snowpositional arguments:
25dd84028fSJohn Snow  target      Target directory to install virtual environment into.
26dd84028fSJohn Snow
27dd84028fSJohn Snowoptions:
28dd84028fSJohn Snow  -h, --help  show this help message and exit
29dd84028fSJohn Snow
30c5538eedSJohn Snow--------------------------------------------------
31c5538eedSJohn Snow
32f1ad527fSJohn Snowusage: mkvenv post_init [-h]
33f1ad527fSJohn Snow
34f1ad527fSJohn Snowoptions:
35f1ad527fSJohn Snow  -h, --help         show this help message and exit
36f1ad527fSJohn Snow
37f1ad527fSJohn Snow--------------------------------------------------
38f1ad527fSJohn Snow
39c5538eedSJohn Snowusage: mkvenv ensure [-h] [--online] [--dir DIR] dep_spec...
40c5538eedSJohn Snow
41c5538eedSJohn Snowpositional arguments:
42c5538eedSJohn Snow  dep_spec    PEP 508 Dependency specification, e.g. 'meson>=0.61.5'
43c5538eedSJohn Snow
44c5538eedSJohn Snowoptions:
45c5538eedSJohn Snow  -h, --help  show this help message and exit
46c5538eedSJohn Snow  --online    Install packages from PyPI, if necessary.
47c5538eedSJohn Snow  --dir DIR   Path to vendored packages where we may install from.
48c5538eedSJohn Snow
4971ed611cSPaolo Bonzini--------------------------------------------------
5071ed611cSPaolo Bonzini
5171ed611cSPaolo Bonziniusage: mkvenv ensuregroup [-h] [--online] [--dir DIR] file group...
5271ed611cSPaolo Bonzini
5371ed611cSPaolo Bonzinipositional arguments:
5471ed611cSPaolo Bonzini  file        pointer to a TOML file
5571ed611cSPaolo Bonzini  group       section name in the TOML file
5671ed611cSPaolo Bonzini
5771ed611cSPaolo Bonzinioptions:
5871ed611cSPaolo Bonzini  -h, --help  show this help message and exit
5971ed611cSPaolo Bonzini  --online    Install packages from PyPI, if necessary.
6071ed611cSPaolo Bonzini  --dir DIR   Path to vendored packages where we may install from.
6171ed611cSPaolo Bonzini
62dd84028fSJohn Snow"""
63dd84028fSJohn Snow
64dd84028fSJohn Snow# Copyright (C) 2022-2023 Red Hat, Inc.
65dd84028fSJohn Snow#
66dd84028fSJohn Snow# Authors:
67dd84028fSJohn Snow#  John Snow <jsnow@redhat.com>
68dd84028fSJohn Snow#  Paolo Bonzini <pbonzini@redhat.com>
69dd84028fSJohn Snow#
70dd84028fSJohn Snow# This work is licensed under the terms of the GNU GPL, version 2 or
71dd84028fSJohn Snow# later. See the COPYING file in the top-level directory.
72dd84028fSJohn Snow
73dd84028fSJohn Snowimport argparse
743e4b6b0aSPaolo Bonzinifrom importlib.metadata import (
753e4b6b0aSPaolo Bonzini    Distribution,
763e4b6b0aSPaolo Bonzini    EntryPoint,
773e4b6b0aSPaolo Bonzini    PackageNotFoundError,
783e4b6b0aSPaolo Bonzini    distribution,
793e4b6b0aSPaolo Bonzini    version,
803e4b6b0aSPaolo Bonzini)
81a9dbde71SJohn Snowfrom importlib.util import find_spec
82dd84028fSJohn Snowimport logging
83dd84028fSJohn Snowimport os
84dd84028fSJohn Snowfrom pathlib import Path
854695a22eSJohn Snowimport re
864695a22eSJohn Snowimport shutil
87dee01b82SJohn Snowimport site
88dd84028fSJohn Snowimport subprocess
89dd84028fSJohn Snowimport sys
90dee01b82SJohn Snowimport sysconfig
91dd84028fSJohn Snowfrom types import SimpleNamespace
92c5538eedSJohn Snowfrom typing import (
93c5538eedSJohn Snow    Any,
940f1ec070SPaolo Bonzini    Dict,
9592834894SJohn Snow    Iterator,
96c5538eedSJohn Snow    Optional,
97c5538eedSJohn Snow    Sequence,
984695a22eSJohn Snow    Tuple,
99c5538eedSJohn Snow    Union,
100c5538eedSJohn Snow)
101dd84028fSJohn Snowimport venv
102c5538eedSJohn Snow
10368ea6d17SJohn Snow
10468ea6d17SJohn Snow# Try to load distlib, with a fallback to pip's vendored version.
10568ea6d17SJohn Snow# HAVE_DISTLIB is checked below, just-in-time, so that mkvenv does not fail
10668ea6d17SJohn Snow# outside the venv or before a potential call to ensurepip in checkpip().
10768ea6d17SJohn SnowHAVE_DISTLIB = True
10868ea6d17SJohn Snowtry:
10992834894SJohn Snow    import distlib.scripts
110c5538eedSJohn Snow    import distlib.version
11168ea6d17SJohn Snowexcept ImportError:
11268ea6d17SJohn Snow    try:
11368ea6d17SJohn Snow        # Reach into pip's cookie jar.  pylint and flake8 don't understand
11468ea6d17SJohn Snow        # that these imports will be used via distlib.xxx.
11568ea6d17SJohn Snow        from pip._vendor import distlib
11668ea6d17SJohn Snow        import pip._vendor.distlib.scripts  # noqa, pylint: disable=unused-import
11768ea6d17SJohn Snow        import pip._vendor.distlib.version  # noqa, pylint: disable=unused-import
11868ea6d17SJohn Snow    except ImportError:
11968ea6d17SJohn Snow        HAVE_DISTLIB = False
120dd84028fSJohn Snow
12171ed611cSPaolo Bonzini# Try to load tomllib, with a fallback to tomli.
12271ed611cSPaolo Bonzini# HAVE_TOMLLIB is checked below, just-in-time, so that mkvenv does not fail
12371ed611cSPaolo Bonzini# outside the venv or before a potential call to ensurepip in checkpip().
12471ed611cSPaolo BonziniHAVE_TOMLLIB = True
12571ed611cSPaolo Bonzinitry:
12671ed611cSPaolo Bonzini    import tomllib
12771ed611cSPaolo Bonziniexcept ImportError:
12871ed611cSPaolo Bonzini    try:
12971ed611cSPaolo Bonzini        import tomli as tomllib
13071ed611cSPaolo Bonzini    except ImportError:
13171ed611cSPaolo Bonzini        HAVE_TOMLLIB = False
13271ed611cSPaolo Bonzini
133dd84028fSJohn Snow# Do not add any mandatory dependencies from outside the stdlib:
134dd84028fSJohn Snow# This script *must* be usable standalone!
135dd84028fSJohn Snow
136dd84028fSJohn SnowDirType = Union[str, bytes, "os.PathLike[str]", "os.PathLike[bytes]"]
137dd84028fSJohn Snowlogger = logging.getLogger("mkvenv")
138dd84028fSJohn Snow
139dd84028fSJohn Snow
140dee01b82SJohn Snowdef inside_a_venv() -> bool:
141dee01b82SJohn Snow    """Returns True if it is executed inside of a virtual environment."""
142dee01b82SJohn Snow    return sys.prefix != sys.base_prefix
143dee01b82SJohn Snow
144dee01b82SJohn Snow
145dd84028fSJohn Snowclass Ouch(RuntimeError):
146dd84028fSJohn Snow    """An Exception class we can't confuse with a builtin."""
147dd84028fSJohn Snow
148dd84028fSJohn Snow
149dd84028fSJohn Snowclass QemuEnvBuilder(venv.EnvBuilder):
150dd84028fSJohn Snow    """
151dd84028fSJohn Snow    An extension of venv.EnvBuilder for building QEMU's configure-time venv.
152dd84028fSJohn Snow
153dee01b82SJohn Snow    The primary difference is that it emulates a "nested" virtual
154dee01b82SJohn Snow    environment when invoked from inside of an existing virtual
155f1ad527fSJohn Snow    environment by including packages from the parent.  Also,
156f1ad527fSJohn Snow    "ensurepip" is replaced if possible with just recreating pip's
157f1ad527fSJohn Snow    console_scripts inside the virtual environment.
158dd84028fSJohn Snow
159dd84028fSJohn Snow    Parameters for base class init:
160dd84028fSJohn Snow      - system_site_packages: bool = False
161dd84028fSJohn Snow      - clear: bool = False
162dd84028fSJohn Snow      - symlinks: bool = False
163dd84028fSJohn Snow      - upgrade: bool = False
164dd84028fSJohn Snow      - with_pip: bool = False
165dd84028fSJohn Snow      - prompt: Optional[str] = None
166dd84028fSJohn Snow      - upgrade_deps: bool = False             (Since 3.9)
167dd84028fSJohn Snow    """
168dd84028fSJohn Snow
169dd84028fSJohn Snow    def __init__(self, *args: Any, **kwargs: Any) -> None:
170dd84028fSJohn Snow        logger.debug("QemuEnvBuilder.__init__(...)")
171a9dbde71SJohn Snow
172dee01b82SJohn Snow        # For nested venv emulation:
173dee01b82SJohn Snow        self.use_parent_packages = False
174dee01b82SJohn Snow        if inside_a_venv():
175dee01b82SJohn Snow            # Include parent packages only if we're in a venv and
176dee01b82SJohn Snow            # system_site_packages was True.
177dee01b82SJohn Snow            self.use_parent_packages = kwargs.pop(
178dee01b82SJohn Snow                "system_site_packages", False
179dee01b82SJohn Snow            )
180dee01b82SJohn Snow            # Include system_site_packages only when the parent,
181dee01b82SJohn Snow            # The venv we are currently in, also does so.
182dee01b82SJohn Snow            kwargs["system_site_packages"] = sys.base_prefix in site.PREFIXES
183dee01b82SJohn Snow
184f1ad527fSJohn Snow        # ensurepip is slow: venv creation can be very fast for cases where
185f1ad527fSJohn Snow        # we allow the use of system_site_packages. Therefore, ensurepip is
186f1ad527fSJohn Snow        # replaced with our own script generation once the virtual environment
187f1ad527fSJohn Snow        # is setup.
188f1ad527fSJohn Snow        self.want_pip = kwargs.get("with_pip", False)
189f1ad527fSJohn Snow        if self.want_pip:
190f1ad527fSJohn Snow            if (
191f1ad527fSJohn Snow                kwargs.get("system_site_packages", False)
192f1ad527fSJohn Snow                and not need_ensurepip()
193f1ad527fSJohn Snow            ):
194f1ad527fSJohn Snow                kwargs["with_pip"] = False
195f1ad527fSJohn Snow            else:
196*0a88ac96SPaolo Bonzini                check_ensurepip()
197a9dbde71SJohn Snow
198dd84028fSJohn Snow        super().__init__(*args, **kwargs)
199dd84028fSJohn Snow
200dd84028fSJohn Snow        # Make the context available post-creation:
201dd84028fSJohn Snow        self._context: Optional[SimpleNamespace] = None
202dd84028fSJohn Snow
203dee01b82SJohn Snow    def get_parent_libpath(self) -> Optional[str]:
204dee01b82SJohn Snow        """Return the libpath of the parent venv, if applicable."""
205dee01b82SJohn Snow        if self.use_parent_packages:
206dee01b82SJohn Snow            return sysconfig.get_path("purelib")
207dee01b82SJohn Snow        return None
208dee01b82SJohn Snow
209dee01b82SJohn Snow    @staticmethod
210dee01b82SJohn Snow    def compute_venv_libpath(context: SimpleNamespace) -> str:
211dee01b82SJohn Snow        """
212dee01b82SJohn Snow        Compatibility wrapper for context.lib_path for Python < 3.12
213dee01b82SJohn Snow        """
214dee01b82SJohn Snow        # Python 3.12+, not strictly necessary because it's documented
215dee01b82SJohn Snow        # to be the same as 3.10 code below:
216dee01b82SJohn Snow        if sys.version_info >= (3, 12):
217dee01b82SJohn Snow            return context.lib_path
218dee01b82SJohn Snow
219dee01b82SJohn Snow        # Python 3.10+
220dee01b82SJohn Snow        if "venv" in sysconfig.get_scheme_names():
221dee01b82SJohn Snow            lib_path = sysconfig.get_path(
222dee01b82SJohn Snow                "purelib", scheme="venv", vars={"base": context.env_dir}
223dee01b82SJohn Snow            )
224dee01b82SJohn Snow            assert lib_path is not None
225dee01b82SJohn Snow            return lib_path
226dee01b82SJohn Snow
227dee01b82SJohn Snow        # For Python <= 3.9 we need to hardcode this. Fortunately the
228dee01b82SJohn Snow        # code below was the same in Python 3.6-3.10, so there is only
229dee01b82SJohn Snow        # one case.
230dee01b82SJohn Snow        if sys.platform == "win32":
231dee01b82SJohn Snow            return os.path.join(context.env_dir, "Lib", "site-packages")
232dee01b82SJohn Snow        return os.path.join(
233dee01b82SJohn Snow            context.env_dir,
234dee01b82SJohn Snow            "lib",
235dee01b82SJohn Snow            "python%d.%d" % sys.version_info[:2],
236dee01b82SJohn Snow            "site-packages",
237dee01b82SJohn Snow        )
238dee01b82SJohn Snow
239dd84028fSJohn Snow    def ensure_directories(self, env_dir: DirType) -> SimpleNamespace:
240dd84028fSJohn Snow        logger.debug("ensure_directories(env_dir=%s)", env_dir)
241dd84028fSJohn Snow        self._context = super().ensure_directories(env_dir)
242dd84028fSJohn Snow        return self._context
243dd84028fSJohn Snow
244dee01b82SJohn Snow    def create(self, env_dir: DirType) -> None:
245dee01b82SJohn Snow        logger.debug("create(env_dir=%s)", env_dir)
246dee01b82SJohn Snow        super().create(env_dir)
247dee01b82SJohn Snow        assert self._context is not None
248dee01b82SJohn Snow        self.post_post_setup(self._context)
249dee01b82SJohn Snow
250dee01b82SJohn Snow    def post_post_setup(self, context: SimpleNamespace) -> None:
251dee01b82SJohn Snow        """
252dee01b82SJohn Snow        The final, final hook. Enter the venv and run commands inside of it.
253dee01b82SJohn Snow        """
254dee01b82SJohn Snow        if self.use_parent_packages:
255dee01b82SJohn Snow            # We're inside of a venv and we want to include the parent
256dee01b82SJohn Snow            # venv's packages.
257dee01b82SJohn Snow            parent_libpath = self.get_parent_libpath()
258dee01b82SJohn Snow            assert parent_libpath is not None
259dee01b82SJohn Snow            logger.debug("parent_libpath: %s", parent_libpath)
260dee01b82SJohn Snow
261dee01b82SJohn Snow            our_libpath = self.compute_venv_libpath(context)
262dee01b82SJohn Snow            logger.debug("our_libpath: %s", our_libpath)
263dee01b82SJohn Snow
264dee01b82SJohn Snow            pth_file = os.path.join(our_libpath, "nested.pth")
265dee01b82SJohn Snow            with open(pth_file, "w", encoding="UTF-8") as file:
266dee01b82SJohn Snow                file.write(parent_libpath + os.linesep)
267dee01b82SJohn Snow
268f1ad527fSJohn Snow        if self.want_pip:
269f1ad527fSJohn Snow            args = [
270f1ad527fSJohn Snow                context.env_exe,
271f1ad527fSJohn Snow                __file__,
272f1ad527fSJohn Snow                "post_init",
273f1ad527fSJohn Snow            ]
274f1ad527fSJohn Snow            subprocess.run(args, check=True)
275f1ad527fSJohn Snow
276dd84028fSJohn Snow    def get_value(self, field: str) -> str:
277dd84028fSJohn Snow        """
278dd84028fSJohn Snow        Get a string value from the context namespace after a call to build.
279dd84028fSJohn Snow
280dd84028fSJohn Snow        For valid field names, see:
281dd84028fSJohn Snow        https://docs.python.org/3/library/venv.html#venv.EnvBuilder.ensure_directories
282dd84028fSJohn Snow        """
283dd84028fSJohn Snow        ret = getattr(self._context, field)
284dd84028fSJohn Snow        assert isinstance(ret, str)
285dd84028fSJohn Snow        return ret
286dd84028fSJohn Snow
287dd84028fSJohn Snow
288f1ad527fSJohn Snowdef need_ensurepip() -> bool:
289f1ad527fSJohn Snow    """
290f1ad527fSJohn Snow    Tests for the presence of setuptools and pip.
291f1ad527fSJohn Snow
292f1ad527fSJohn Snow    :return: `True` if we do not detect both packages.
293f1ad527fSJohn Snow    """
294f1ad527fSJohn Snow    # Don't try to actually import them, it's fraught with danger:
295f1ad527fSJohn Snow    # https://github.com/pypa/setuptools/issues/2993
296f1ad527fSJohn Snow    if find_spec("setuptools") and find_spec("pip"):
297f1ad527fSJohn Snow        return False
298f1ad527fSJohn Snow    return True
299f1ad527fSJohn Snow
300f1ad527fSJohn Snow
301*0a88ac96SPaolo Bonzinidef check_ensurepip() -> None:
302a9dbde71SJohn Snow    """
303a9dbde71SJohn Snow    Check that we have ensurepip.
304a9dbde71SJohn Snow
305a9dbde71SJohn Snow    Raise a fatal exception with a helpful hint if it isn't available.
306a9dbde71SJohn Snow    """
307a9dbde71SJohn Snow    if not find_spec("ensurepip"):
308a9dbde71SJohn Snow        msg = (
309a9dbde71SJohn Snow            "Python's ensurepip module is not found.\n"
310a9dbde71SJohn Snow            "It's normally part of the Python standard library, "
311a9dbde71SJohn Snow            "maybe your distribution packages it separately?\n"
312a9dbde71SJohn Snow            "Either install ensurepip, or alleviate the need for it in the "
313a9dbde71SJohn Snow            "first place by installing pip and setuptools for "
314a9dbde71SJohn Snow            f"'{sys.executable}'.\n"
315*0a88ac96SPaolo Bonzini            "(Hint: Debian puts ensurepip in its python3-venv package.)"
316a9dbde71SJohn Snow        )
317*0a88ac96SPaolo Bonzini        raise Ouch(msg)
318a9dbde71SJohn Snow
319a9dbde71SJohn Snow    # ensurepip uses pyexpat, which can also go missing on us:
320a9dbde71SJohn Snow    if not find_spec("pyexpat"):
321a9dbde71SJohn Snow        msg = (
322a9dbde71SJohn Snow            "Python's pyexpat module is not found.\n"
323a9dbde71SJohn Snow            "It's normally part of the Python standard library, "
324a9dbde71SJohn Snow            "maybe your distribution packages it separately?\n"
325a9dbde71SJohn Snow            "Either install pyexpat, or alleviate the need for it in the "
326a9dbde71SJohn Snow            "first place by installing pip and setuptools for "
327*0a88ac96SPaolo Bonzini            f"'{sys.executable}'.\n\n"
328*0a88ac96SPaolo Bonzini            "(Hint: NetBSD's pkgsrc debundles this to e.g. 'py310-expat'.)"
329a9dbde71SJohn Snow        )
330*0a88ac96SPaolo Bonzini        raise Ouch(msg)
331a9dbde71SJohn Snow
332a9dbde71SJohn Snow
333dd84028fSJohn Snowdef make_venv(  # pylint: disable=too-many-arguments
334dd84028fSJohn Snow    env_dir: Union[str, Path],
335dd84028fSJohn Snow    system_site_packages: bool = False,
336dd84028fSJohn Snow    clear: bool = True,
337dd84028fSJohn Snow    symlinks: Optional[bool] = None,
338dd84028fSJohn Snow    with_pip: bool = True,
339dd84028fSJohn Snow) -> None:
340dd84028fSJohn Snow    """
341dd84028fSJohn Snow    Create a venv using `QemuEnvBuilder`.
342dd84028fSJohn Snow
343dd84028fSJohn Snow    This is analogous to the `venv.create` module-level convenience
344dd84028fSJohn Snow    function that is part of the Python stdblib, except it uses
345dd84028fSJohn Snow    `QemuEnvBuilder` instead.
346dd84028fSJohn Snow
347dd84028fSJohn Snow    :param env_dir: The directory to create/install to.
348dd84028fSJohn Snow    :param system_site_packages:
349dd84028fSJohn Snow        Allow inheriting packages from the system installation.
350dd84028fSJohn Snow    :param clear: When True, fully remove any prior venv and files.
351dd84028fSJohn Snow    :param symlinks:
352dd84028fSJohn Snow        Whether to use symlinks to the target interpreter or not. If
353dd84028fSJohn Snow        left unspecified, it will use symlinks except on Windows to
354dd84028fSJohn Snow        match behavior with the "venv" CLI tool.
355dd84028fSJohn Snow    :param with_pip:
356dd84028fSJohn Snow        Whether to install "pip" binaries or not.
357dd84028fSJohn Snow    """
358dd84028fSJohn Snow    logger.debug(
359dd84028fSJohn Snow        "%s: make_venv(env_dir=%s, system_site_packages=%s, "
360dd84028fSJohn Snow        "clear=%s, symlinks=%s, with_pip=%s)",
361dd84028fSJohn Snow        __file__,
362dd84028fSJohn Snow        str(env_dir),
363dd84028fSJohn Snow        system_site_packages,
364dd84028fSJohn Snow        clear,
365dd84028fSJohn Snow        symlinks,
366dd84028fSJohn Snow        with_pip,
367dd84028fSJohn Snow    )
368dd84028fSJohn Snow
369dd84028fSJohn Snow    if symlinks is None:
370dd84028fSJohn Snow        # Default behavior of standard venv CLI
371dd84028fSJohn Snow        symlinks = os.name != "nt"
372dd84028fSJohn Snow
373dd84028fSJohn Snow    builder = QemuEnvBuilder(
374dd84028fSJohn Snow        system_site_packages=system_site_packages,
375dd84028fSJohn Snow        clear=clear,
376dd84028fSJohn Snow        symlinks=symlinks,
377dd84028fSJohn Snow        with_pip=with_pip,
378dd84028fSJohn Snow    )
379dd84028fSJohn Snow
380dd84028fSJohn Snow    style = "non-isolated" if builder.system_site_packages else "isolated"
381dee01b82SJohn Snow    nested = ""
382dee01b82SJohn Snow    if builder.use_parent_packages:
383dee01b82SJohn Snow        nested = f"(with packages from '{builder.get_parent_libpath()}') "
384dd84028fSJohn Snow    print(
385dd84028fSJohn Snow        f"mkvenv: Creating {style} virtual environment"
386dee01b82SJohn Snow        f" {nested}at '{str(env_dir)}'",
387dd84028fSJohn Snow        file=sys.stderr,
388dd84028fSJohn Snow    )
389dd84028fSJohn Snow
390dd84028fSJohn Snow    try:
391dd84028fSJohn Snow        logger.debug("Invoking builder.create()")
392dd84028fSJohn Snow        try:
393dd84028fSJohn Snow            builder.create(str(env_dir))
394dd84028fSJohn Snow        except SystemExit as exc:
395dd84028fSJohn Snow            # Some versions of the venv module raise SystemExit; *nasty*!
396dd84028fSJohn Snow            # We want the exception that prompted it. It might be a subprocess
397dd84028fSJohn Snow            # error that has output we *really* want to see.
398dd84028fSJohn Snow            logger.debug("Intercepted SystemExit from EnvBuilder.create()")
399dd84028fSJohn Snow            raise exc.__cause__ or exc.__context__ or exc
400dd84028fSJohn Snow        logger.debug("builder.create() finished")
401dd84028fSJohn Snow    except subprocess.CalledProcessError as exc:
402dd84028fSJohn Snow        logger.error("mkvenv subprocess failed:")
403dd84028fSJohn Snow        logger.error("cmd: %s", exc.cmd)
404dd84028fSJohn Snow        logger.error("returncode: %d", exc.returncode)
405dd84028fSJohn Snow
406dd84028fSJohn Snow        def _stringify(data: Union[str, bytes]) -> str:
407dd84028fSJohn Snow            if isinstance(data, bytes):
408dd84028fSJohn Snow                return data.decode()
409dd84028fSJohn Snow            return data
410dd84028fSJohn Snow
411dd84028fSJohn Snow        lines = []
412dd84028fSJohn Snow        if exc.stdout:
413dd84028fSJohn Snow            lines.append("========== stdout ==========")
414dd84028fSJohn Snow            lines.append(_stringify(exc.stdout))
415dd84028fSJohn Snow            lines.append("============================")
416dd84028fSJohn Snow        if exc.stderr:
417dd84028fSJohn Snow            lines.append("========== stderr ==========")
418dd84028fSJohn Snow            lines.append(_stringify(exc.stderr))
419dd84028fSJohn Snow            lines.append("============================")
420dd84028fSJohn Snow        if lines:
421dd84028fSJohn Snow            logger.error(os.linesep.join(lines))
422dd84028fSJohn Snow
423dd84028fSJohn Snow        raise Ouch("VENV creation subprocess failed.") from exc
424dd84028fSJohn Snow
425dd84028fSJohn Snow    # print the python executable to stdout for configure.
426dd84028fSJohn Snow    print(builder.get_value("env_exe"))
427dd84028fSJohn Snow
428dd84028fSJohn Snow
4293e4b6b0aSPaolo Bonzinidef _get_entry_points(packages: Sequence[str]) -> Iterator[str]:
43092834894SJohn Snow
43192834894SJohn Snow    def _generator() -> Iterator[str]:
43292834894SJohn Snow        for package in packages:
43392834894SJohn Snow            try:
434ca056f44SPaolo Bonzini                entry_points: Iterator[EntryPoint] = \
435ca056f44SPaolo Bonzini                    iter(distribution(package).entry_points)
43692834894SJohn Snow            except PackageNotFoundError:
43792834894SJohn Snow                continue
43892834894SJohn Snow
43992834894SJohn Snow            # The EntryPoints type is only available in 3.10+,
44092834894SJohn Snow            # treat this as a vanilla list and filter it ourselves.
44192834894SJohn Snow            entry_points = filter(
44292834894SJohn Snow                lambda ep: ep.group == "console_scripts", entry_points
44392834894SJohn Snow            )
44492834894SJohn Snow
44592834894SJohn Snow            for entry_point in entry_points:
44692834894SJohn Snow                yield f"{entry_point.name} = {entry_point.value}"
44792834894SJohn Snow
44892834894SJohn Snow    return _generator()
44992834894SJohn Snow
45092834894SJohn Snow
45192834894SJohn Snowdef generate_console_scripts(
45292834894SJohn Snow    packages: Sequence[str],
45392834894SJohn Snow    python_path: Optional[str] = None,
45492834894SJohn Snow    bin_path: Optional[str] = None,
45592834894SJohn Snow) -> None:
45692834894SJohn Snow    """
45792834894SJohn Snow    Generate script shims for console_script entry points in @packages.
45892834894SJohn Snow    """
45992834894SJohn Snow    if python_path is None:
46092834894SJohn Snow        python_path = sys.executable
46192834894SJohn Snow    if bin_path is None:
46292834894SJohn Snow        bin_path = sysconfig.get_path("scripts")
46392834894SJohn Snow        assert bin_path is not None
46492834894SJohn Snow
46592834894SJohn Snow    logger.debug(
46692834894SJohn Snow        "generate_console_scripts(packages=%s, python_path=%s, bin_path=%s)",
46792834894SJohn Snow        packages,
46892834894SJohn Snow        python_path,
46992834894SJohn Snow        bin_path,
47092834894SJohn Snow    )
47192834894SJohn Snow
47292834894SJohn Snow    if not packages:
47392834894SJohn Snow        return
47492834894SJohn Snow
47592834894SJohn Snow    maker = distlib.scripts.ScriptMaker(None, bin_path)
47692834894SJohn Snow    maker.variants = {""}
47792834894SJohn Snow    maker.clobber = False
47892834894SJohn Snow
4793e4b6b0aSPaolo Bonzini    for entry_point in _get_entry_points(packages):
48092834894SJohn Snow        for filename in maker.make(entry_point):
48192834894SJohn Snow            logger.debug("wrote console_script '%s'", filename)
48292834894SJohn Snow
48392834894SJohn Snow
4844695a22eSJohn Snowdef pkgname_from_depspec(dep_spec: str) -> str:
4854695a22eSJohn Snow    """
4864695a22eSJohn Snow    Parse package name out of a PEP-508 depspec.
4874695a22eSJohn Snow
4884695a22eSJohn Snow    See https://peps.python.org/pep-0508/#names
4894695a22eSJohn Snow    """
4904695a22eSJohn Snow    match = re.match(
4914695a22eSJohn Snow        r"^([A-Z0-9]([A-Z0-9._-]*[A-Z0-9])?)", dep_spec, re.IGNORECASE
4924695a22eSJohn Snow    )
4934695a22eSJohn Snow    if not match:
4944695a22eSJohn Snow        raise ValueError(
4954695a22eSJohn Snow            f"dep_spec '{dep_spec}'"
4964695a22eSJohn Snow            " does not appear to contain a valid package name"
4974695a22eSJohn Snow        )
4984695a22eSJohn Snow    return match.group(0)
4994695a22eSJohn Snow
5004695a22eSJohn Snow
50147a90a51SPaolo Bonzinidef _path_is_prefix(prefix: Optional[str], path: str) -> bool:
50247a90a51SPaolo Bonzini    try:
50347a90a51SPaolo Bonzini        return (
50447a90a51SPaolo Bonzini            prefix is not None and os.path.commonpath([prefix, path]) == prefix
50547a90a51SPaolo Bonzini        )
50647a90a51SPaolo Bonzini    except ValueError:
50747a90a51SPaolo Bonzini        return False
50847a90a51SPaolo Bonzini
50947a90a51SPaolo Bonzini
5103e4b6b0aSPaolo Bonzinidef _is_system_package(dist: Distribution) -> bool:
5113e4b6b0aSPaolo Bonzini    path = str(dist.locate_file("."))
5123e4b6b0aSPaolo Bonzini    return not (
51347a90a51SPaolo Bonzini        _path_is_prefix(sysconfig.get_path("purelib"), path)
51447a90a51SPaolo Bonzini        or _path_is_prefix(sysconfig.get_path("platlib"), path)
51547a90a51SPaolo Bonzini    )
51647a90a51SPaolo Bonzini
51747a90a51SPaolo Bonzini
5184695a22eSJohn Snowdef diagnose(
5194695a22eSJohn Snow    dep_spec: str,
5204695a22eSJohn Snow    online: bool,
5214695a22eSJohn Snow    wheels_dir: Optional[Union[str, Path]],
5224695a22eSJohn Snow    prog: Optional[str],
5234695a22eSJohn Snow) -> Tuple[str, bool]:
5244695a22eSJohn Snow    """
5254695a22eSJohn Snow    Offer a summary to the user as to why a package failed to be installed.
5264695a22eSJohn Snow
5274695a22eSJohn Snow    :param dep_spec: The package we tried to ensure, e.g. 'meson>=0.61.5'
5284695a22eSJohn Snow    :param online: Did we allow PyPI access?
5294695a22eSJohn Snow    :param prog:
5304695a22eSJohn Snow        Optionally, a shell program name that can be used as a
5314695a22eSJohn Snow        bellwether to detect if this program is installed elsewhere on
5324695a22eSJohn Snow        the system. This is used to offer advice when a program is
5334695a22eSJohn Snow        detected for a different python version.
5344695a22eSJohn Snow    :param wheels_dir:
5354695a22eSJohn Snow        Optionally, a directory that was searched for vendored packages.
5364695a22eSJohn Snow    """
5374695a22eSJohn Snow    # pylint: disable=too-many-branches
5384695a22eSJohn Snow
5394695a22eSJohn Snow    # Some errors are not particularly serious
5404695a22eSJohn Snow    bad = False
5414695a22eSJohn Snow
5424695a22eSJohn Snow    pkg_name = pkgname_from_depspec(dep_spec)
5433e4b6b0aSPaolo Bonzini    pkg_version: Optional[str] = None
5443e4b6b0aSPaolo Bonzini    try:
5453e4b6b0aSPaolo Bonzini        pkg_version = version(pkg_name)
5463e4b6b0aSPaolo Bonzini    except PackageNotFoundError:
5473e4b6b0aSPaolo Bonzini        pass
5484695a22eSJohn Snow
5494695a22eSJohn Snow    lines = []
5504695a22eSJohn Snow
5514695a22eSJohn Snow    if pkg_version:
5524695a22eSJohn Snow        lines.append(
5534695a22eSJohn Snow            f"Python package '{pkg_name}' version '{pkg_version}' was found,"
5544695a22eSJohn Snow            " but isn't suitable."
5554695a22eSJohn Snow        )
5564695a22eSJohn Snow    else:
5574695a22eSJohn Snow        lines.append(
558c673f3d0SPaolo Bonzini            f"Python package '{pkg_name}' was not found nor installed."
5594695a22eSJohn Snow        )
5604695a22eSJohn Snow
5614695a22eSJohn Snow    if wheels_dir:
5624695a22eSJohn Snow        lines.append(
5634695a22eSJohn Snow            "No suitable version found in, or failed to install from"
5644695a22eSJohn Snow            f" '{wheels_dir}'."
5654695a22eSJohn Snow        )
5664695a22eSJohn Snow        bad = True
5674695a22eSJohn Snow
5684695a22eSJohn Snow    if online:
5694695a22eSJohn Snow        lines.append("A suitable version could not be obtained from PyPI.")
5704695a22eSJohn Snow        bad = True
5714695a22eSJohn Snow    else:
5724695a22eSJohn Snow        lines.append(
5734695a22eSJohn Snow            "mkvenv was configured to operate offline and did not check PyPI."
5744695a22eSJohn Snow        )
5754695a22eSJohn Snow
5764695a22eSJohn Snow    if prog and not pkg_version:
5774695a22eSJohn Snow        which = shutil.which(prog)
5784695a22eSJohn Snow        if which:
5794695a22eSJohn Snow            if sys.base_prefix in site.PREFIXES:
5804695a22eSJohn Snow                pypath = Path(sys.executable).resolve()
5814695a22eSJohn Snow                lines.append(
5824695a22eSJohn Snow                    f"'{prog}' was detected on your system at '{which}', "
5834695a22eSJohn Snow                    f"but the Python package '{pkg_name}' was not found by "
5844695a22eSJohn Snow                    f"this Python interpreter ('{pypath}'). "
5854695a22eSJohn Snow                    f"Typically this means that '{prog}' has been installed "
5864695a22eSJohn Snow                    "against a different Python interpreter on your system."
5874695a22eSJohn Snow                )
5884695a22eSJohn Snow            else:
5894695a22eSJohn Snow                lines.append(
5904695a22eSJohn Snow                    f"'{prog}' was detected on your system at '{which}', "
5914695a22eSJohn Snow                    "but the build is using an isolated virtual environment."
5924695a22eSJohn Snow                )
5934695a22eSJohn Snow            bad = True
5944695a22eSJohn Snow
5954695a22eSJohn Snow    lines = [f" • {line}" for line in lines]
5964695a22eSJohn Snow    if bad:
5974695a22eSJohn Snow        lines.insert(0, f"Could not provide build dependency '{dep_spec}':")
5984695a22eSJohn Snow    else:
5994695a22eSJohn Snow        lines.insert(0, f"'{dep_spec}' not found:")
6004695a22eSJohn Snow    return os.linesep.join(lines), bad
6014695a22eSJohn Snow
6024695a22eSJohn Snow
603c5538eedSJohn Snowdef pip_install(
604c5538eedSJohn Snow    args: Sequence[str],
605c5538eedSJohn Snow    online: bool = False,
606c5538eedSJohn Snow    wheels_dir: Optional[Union[str, Path]] = None,
607c5538eedSJohn Snow) -> None:
608c5538eedSJohn Snow    """
609c5538eedSJohn Snow    Use pip to install a package or package(s) as specified in @args.
610c5538eedSJohn Snow    """
611c5538eedSJohn Snow    loud = bool(
612c5538eedSJohn Snow        os.environ.get("DEBUG")
613c5538eedSJohn Snow        or os.environ.get("GITLAB_CI")
614c5538eedSJohn Snow        or os.environ.get("V")
615c5538eedSJohn Snow    )
616c5538eedSJohn Snow
617c5538eedSJohn Snow    full_args = [
618c5538eedSJohn Snow        sys.executable,
619c5538eedSJohn Snow        "-m",
620c5538eedSJohn Snow        "pip",
621c5538eedSJohn Snow        "install",
622c5538eedSJohn Snow        "--disable-pip-version-check",
623c5538eedSJohn Snow        "-v" if loud else "-q",
624c5538eedSJohn Snow    ]
625c5538eedSJohn Snow    if not online:
626c5538eedSJohn Snow        full_args += ["--no-index"]
627c5538eedSJohn Snow    if wheels_dir:
628c5538eedSJohn Snow        full_args += ["--find-links", f"file://{str(wheels_dir)}"]
629c5538eedSJohn Snow    full_args += list(args)
630c5538eedSJohn Snow    subprocess.run(
631c5538eedSJohn Snow        full_args,
632c5538eedSJohn Snow        check=True,
633c5538eedSJohn Snow    )
634c5538eedSJohn Snow
635c5538eedSJohn Snow
6360f1ec070SPaolo Bonzinidef _make_version_constraint(info: Dict[str, str], install: bool) -> str:
6370f1ec070SPaolo Bonzini    """
6380f1ec070SPaolo Bonzini    Construct the version constraint part of a PEP 508 dependency
6390f1ec070SPaolo Bonzini    specification (for example '>=0.61.5') from the accepted and
6400f1ec070SPaolo Bonzini    installed keys of the provided dictionary.
6410f1ec070SPaolo Bonzini
6420f1ec070SPaolo Bonzini    :param info: A dictionary corresponding to a TOML key-value list.
6430f1ec070SPaolo Bonzini    :param install: True generates install constraints, False generates
6440f1ec070SPaolo Bonzini        presence constraints
6450f1ec070SPaolo Bonzini    """
6460f1ec070SPaolo Bonzini    if install and "installed" in info:
6470f1ec070SPaolo Bonzini        return "==" + info["installed"]
6480f1ec070SPaolo Bonzini
6490f1ec070SPaolo Bonzini    dep_spec = info.get("accepted", "")
6500f1ec070SPaolo Bonzini    dep_spec = dep_spec.strip()
6510f1ec070SPaolo Bonzini    # Double check that they didn't just use a version number
6520f1ec070SPaolo Bonzini    if dep_spec and dep_spec[0] not in "!~><=(":
6530f1ec070SPaolo Bonzini        raise Ouch(
6540f1ec070SPaolo Bonzini            "invalid dependency specifier " + dep_spec + " in dependency file"
6550f1ec070SPaolo Bonzini        )
6560f1ec070SPaolo Bonzini
6570f1ec070SPaolo Bonzini    return dep_spec
6580f1ec070SPaolo Bonzini
6590f1ec070SPaolo Bonzini
6604695a22eSJohn Snowdef _do_ensure(
6610f1ec070SPaolo Bonzini    group: Dict[str, Dict[str, str]],
662c5538eedSJohn Snow    online: bool = False,
663c5538eedSJohn Snow    wheels_dir: Optional[Union[str, Path]] = None,
664d37c21b5SPaolo Bonzini) -> Optional[Tuple[str, bool]]:
665c5538eedSJohn Snow    """
6660f1ec070SPaolo Bonzini    Use pip to ensure we have the packages specified in @group.
667c5538eedSJohn Snow
6680f1ec070SPaolo Bonzini    If the packages are already installed, do nothing. If online and
669c5538eedSJohn Snow    wheels_dir are both provided, prefer packages found in wheels_dir
670c5538eedSJohn Snow    first before connecting to PyPI.
671c5538eedSJohn Snow
6720f1ec070SPaolo Bonzini    :param group: A dictionary of dictionaries, corresponding to a
6730f1ec070SPaolo Bonzini        section in a pythondeps.toml file.
674c5538eedSJohn Snow    :param online: If True, fall back to PyPI.
675c5538eedSJohn Snow    :param wheels_dir: If specified, search this path for packages.
676c5538eedSJohn Snow    """
677c5538eedSJohn Snow    absent = []
67892834894SJohn Snow    present = []
67967b9a83dSPaolo Bonzini    canary = None
6800f1ec070SPaolo Bonzini    for name, info in group.items():
6810f1ec070SPaolo Bonzini        constraint = _make_version_constraint(info, False)
6820f1ec070SPaolo Bonzini        matcher = distlib.version.LegacyMatcher(name + constraint)
68371ed611cSPaolo Bonzini        print(f"mkvenv: checking for {matcher}", file=sys.stderr)
6843e4b6b0aSPaolo Bonzini
6853e4b6b0aSPaolo Bonzini        dist: Optional[Distribution] = None
6863e4b6b0aSPaolo Bonzini        try:
6873e4b6b0aSPaolo Bonzini            dist = distribution(matcher.name)
6883e4b6b0aSPaolo Bonzini        except PackageNotFoundError:
6893e4b6b0aSPaolo Bonzini            pass
6903e4b6b0aSPaolo Bonzini
69147a90a51SPaolo Bonzini        if (
6923e4b6b0aSPaolo Bonzini            dist is None
69347a90a51SPaolo Bonzini            # Always pass installed package to pip, so that they can be
69447a90a51SPaolo Bonzini            # updated if the requested version changes
6953e4b6b0aSPaolo Bonzini            or not _is_system_package(dist)
6963e4b6b0aSPaolo Bonzini            or not matcher.match(distlib.version.LegacyVersion(dist.version))
697c673f3d0SPaolo Bonzini        ):
6980f1ec070SPaolo Bonzini            absent.append(name + _make_version_constraint(info, True))
6990f1ec070SPaolo Bonzini            if len(absent) == 1:
7000f1ec070SPaolo Bonzini                canary = info.get("canary", None)
701c5538eedSJohn Snow        else:
7023e4b6b0aSPaolo Bonzini            logger.info("found %s %s", name, dist.version)
7030f1ec070SPaolo Bonzini            present.append(name)
70492834894SJohn Snow
70592834894SJohn Snow    if present:
70692834894SJohn Snow        generate_console_scripts(present)
707c5538eedSJohn Snow
708c5538eedSJohn Snow    if absent:
709d37c21b5SPaolo Bonzini        if online or wheels_dir:
710c5538eedSJohn Snow            # Some packages are missing or aren't a suitable version,
711c5538eedSJohn Snow            # install a suitable (possibly vendored) package.
712c5538eedSJohn Snow            print(f"mkvenv: installing {', '.join(absent)}", file=sys.stderr)
713d37c21b5SPaolo Bonzini            try:
714c5538eedSJohn Snow                pip_install(args=absent, online=online, wheels_dir=wheels_dir)
715d37c21b5SPaolo Bonzini                return None
716d37c21b5SPaolo Bonzini            except subprocess.CalledProcessError:
717d37c21b5SPaolo Bonzini                pass
718d37c21b5SPaolo Bonzini
719d37c21b5SPaolo Bonzini        return diagnose(
720d37c21b5SPaolo Bonzini            absent[0],
721d37c21b5SPaolo Bonzini            online,
722d37c21b5SPaolo Bonzini            wheels_dir,
72367b9a83dSPaolo Bonzini            canary,
724d37c21b5SPaolo Bonzini        )
725d37c21b5SPaolo Bonzini
726d37c21b5SPaolo Bonzini    return None
727c5538eedSJohn Snow
728c5538eedSJohn Snow
7294695a22eSJohn Snowdef ensure(
7304695a22eSJohn Snow    dep_specs: Sequence[str],
7314695a22eSJohn Snow    online: bool = False,
7324695a22eSJohn Snow    wheels_dir: Optional[Union[str, Path]] = None,
7334695a22eSJohn Snow    prog: Optional[str] = None,
7344695a22eSJohn Snow) -> None:
7354695a22eSJohn Snow    """
7364695a22eSJohn Snow    Use pip to ensure we have the package specified by @dep_specs.
7374695a22eSJohn Snow
7384695a22eSJohn Snow    If the package is already installed, do nothing. If online and
7394695a22eSJohn Snow    wheels_dir are both provided, prefer packages found in wheels_dir
7404695a22eSJohn Snow    first before connecting to PyPI.
7414695a22eSJohn Snow
7424695a22eSJohn Snow    :param dep_specs:
7434695a22eSJohn Snow        PEP 508 dependency specifications. e.g. ['meson>=0.61.5'].
7444695a22eSJohn Snow    :param online: If True, fall back to PyPI.
7454695a22eSJohn Snow    :param wheels_dir: If specified, search this path for packages.
7464695a22eSJohn Snow    :param prog:
7474695a22eSJohn Snow        If specified, use this program name for error diagnostics that will
7484695a22eSJohn Snow        be presented to the user. e.g., 'sphinx-build' can be used as a
7494695a22eSJohn Snow        bellwether for the presence of 'sphinx'.
7504695a22eSJohn Snow    """
75168ea6d17SJohn Snow
75268ea6d17SJohn Snow    if not HAVE_DISTLIB:
75368ea6d17SJohn Snow        raise Ouch("a usable distlib could not be found, please install it")
75468ea6d17SJohn Snow
7550f1ec070SPaolo Bonzini    # Convert the depspecs to a dictionary, as if they came
7560f1ec070SPaolo Bonzini    # from a section in a pythondeps.toml file
7570f1ec070SPaolo Bonzini    group: Dict[str, Dict[str, str]] = {}
7580f1ec070SPaolo Bonzini    for spec in dep_specs:
7590f1ec070SPaolo Bonzini        name = distlib.version.LegacyMatcher(spec).name
7600f1ec070SPaolo Bonzini        group[name] = {}
7610f1ec070SPaolo Bonzini
7620f1ec070SPaolo Bonzini        spec = spec.strip()
7630f1ec070SPaolo Bonzini        pos = len(name)
7640f1ec070SPaolo Bonzini        ver = spec[pos:].strip()
7650f1ec070SPaolo Bonzini        if ver:
7660f1ec070SPaolo Bonzini            group[name]["accepted"] = ver
7670f1ec070SPaolo Bonzini
7680f1ec070SPaolo Bonzini        if prog:
7690f1ec070SPaolo Bonzini            group[name]["canary"] = prog
7700f1ec070SPaolo Bonzini            prog = None
7710f1ec070SPaolo Bonzini
7720f1ec070SPaolo Bonzini    result = _do_ensure(group, online, wheels_dir)
773d37c21b5SPaolo Bonzini    if result:
7744695a22eSJohn Snow        # Well, that's not good.
775d37c21b5SPaolo Bonzini        if result[1]:
776d37c21b5SPaolo Bonzini            raise Ouch(result[0])
777d37c21b5SPaolo Bonzini        raise SystemExit(f"\n{result[0]}\n\n")
7784695a22eSJohn Snow
7794695a22eSJohn Snow
78071ed611cSPaolo Bonzinidef _parse_groups(file: str) -> Dict[str, Dict[str, Any]]:
78171ed611cSPaolo Bonzini    if not HAVE_TOMLLIB:
78271ed611cSPaolo Bonzini        if sys.version_info < (3, 11):
78371ed611cSPaolo Bonzini            raise Ouch("found no usable tomli, please install it")
78471ed611cSPaolo Bonzini
78571ed611cSPaolo Bonzini        raise Ouch(
78671ed611cSPaolo Bonzini            "Python >=3.11 does not have tomllib... what have you done!?"
78771ed611cSPaolo Bonzini        )
78871ed611cSPaolo Bonzini
78971ed611cSPaolo Bonzini    # Use loads() to support both tomli v1.2.x (Ubuntu 22.04,
79071ed611cSPaolo Bonzini    # Debian bullseye-backports) and v2.0.x
79171ed611cSPaolo Bonzini    with open(file, "r", encoding="ascii") as depfile:
79271ed611cSPaolo Bonzini        contents = depfile.read()
79371ed611cSPaolo Bonzini        return tomllib.loads(contents)  # type: ignore
79471ed611cSPaolo Bonzini
79571ed611cSPaolo Bonzini
79671ed611cSPaolo Bonzinidef ensure_group(
79771ed611cSPaolo Bonzini    file: str,
79871ed611cSPaolo Bonzini    groups: Sequence[str],
79971ed611cSPaolo Bonzini    online: bool = False,
80071ed611cSPaolo Bonzini    wheels_dir: Optional[Union[str, Path]] = None,
80171ed611cSPaolo Bonzini) -> None:
80271ed611cSPaolo Bonzini    """
80371ed611cSPaolo Bonzini    Use pip to ensure we have the package specified by @dep_specs.
80471ed611cSPaolo Bonzini
80571ed611cSPaolo Bonzini    If the package is already installed, do nothing. If online and
80671ed611cSPaolo Bonzini    wheels_dir are both provided, prefer packages found in wheels_dir
80771ed611cSPaolo Bonzini    first before connecting to PyPI.
80871ed611cSPaolo Bonzini
80971ed611cSPaolo Bonzini    :param dep_specs:
81071ed611cSPaolo Bonzini        PEP 508 dependency specifications. e.g. ['meson>=0.61.5'].
81171ed611cSPaolo Bonzini    :param online: If True, fall back to PyPI.
81271ed611cSPaolo Bonzini    :param wheels_dir: If specified, search this path for packages.
81371ed611cSPaolo Bonzini    """
81471ed611cSPaolo Bonzini
81571ed611cSPaolo Bonzini    if not HAVE_DISTLIB:
81671ed611cSPaolo Bonzini        raise Ouch("found no usable distlib, please install it")
81771ed611cSPaolo Bonzini
81871ed611cSPaolo Bonzini    parsed_deps = _parse_groups(file)
81971ed611cSPaolo Bonzini
82071ed611cSPaolo Bonzini    to_install: Dict[str, Dict[str, str]] = {}
82171ed611cSPaolo Bonzini    for group in groups:
82271ed611cSPaolo Bonzini        try:
82371ed611cSPaolo Bonzini            to_install.update(parsed_deps[group])
82471ed611cSPaolo Bonzini        except KeyError as exc:
82571ed611cSPaolo Bonzini            raise Ouch(f"group {group} not defined") from exc
82671ed611cSPaolo Bonzini
82771ed611cSPaolo Bonzini    result = _do_ensure(to_install, online, wheels_dir)
82871ed611cSPaolo Bonzini    if result:
82971ed611cSPaolo Bonzini        # Well, that's not good.
83071ed611cSPaolo Bonzini        if result[1]:
83171ed611cSPaolo Bonzini            raise Ouch(result[0])
83271ed611cSPaolo Bonzini        raise SystemExit(f"\n{result[0]}\n\n")
83371ed611cSPaolo Bonzini
83471ed611cSPaolo Bonzini
835f1ad527fSJohn Snowdef post_venv_setup() -> None:
836f1ad527fSJohn Snow    """
837f1ad527fSJohn Snow    This is intended to be run *inside the venv* after it is created.
838f1ad527fSJohn Snow    """
839f1ad527fSJohn Snow    logger.debug("post_venv_setup()")
840*0a88ac96SPaolo Bonzini    # Generate a 'pip' script so the venv is usable in a normal
841f1ad527fSJohn Snow    # way from the CLI. This only happens when we inherited pip from a
842f1ad527fSJohn Snow    # parent/system-site and haven't run ensurepip in some way.
843f1ad527fSJohn Snow    generate_console_scripts(["pip"])
844f1ad527fSJohn Snow
845f1ad527fSJohn Snow
846dd84028fSJohn Snowdef _add_create_subcommand(subparsers: Any) -> None:
847dd84028fSJohn Snow    subparser = subparsers.add_parser("create", help="create a venv")
848dd84028fSJohn Snow    subparser.add_argument(
849dd84028fSJohn Snow        "target",
850dd84028fSJohn Snow        type=str,
851dd84028fSJohn Snow        action="store",
852dd84028fSJohn Snow        help="Target directory to install virtual environment into.",
853dd84028fSJohn Snow    )
854dd84028fSJohn Snow
855dd84028fSJohn Snow
856f1ad527fSJohn Snowdef _add_post_init_subcommand(subparsers: Any) -> None:
857f1ad527fSJohn Snow    subparsers.add_parser("post_init", help="post-venv initialization")
858f1ad527fSJohn Snow
859f1ad527fSJohn Snow
86071ed611cSPaolo Bonzinidef _add_ensuregroup_subcommand(subparsers: Any) -> None:
86171ed611cSPaolo Bonzini    subparser = subparsers.add_parser(
86271ed611cSPaolo Bonzini        "ensuregroup",
86371ed611cSPaolo Bonzini        help="Ensure that the specified package group is installed.",
86471ed611cSPaolo Bonzini    )
86571ed611cSPaolo Bonzini    subparser.add_argument(
86671ed611cSPaolo Bonzini        "--online",
86771ed611cSPaolo Bonzini        action="store_true",
86871ed611cSPaolo Bonzini        help="Install packages from PyPI, if necessary.",
86971ed611cSPaolo Bonzini    )
87071ed611cSPaolo Bonzini    subparser.add_argument(
87171ed611cSPaolo Bonzini        "--dir",
87271ed611cSPaolo Bonzini        type=str,
87371ed611cSPaolo Bonzini        action="store",
87471ed611cSPaolo Bonzini        help="Path to vendored packages where we may install from.",
87571ed611cSPaolo Bonzini    )
87671ed611cSPaolo Bonzini    subparser.add_argument(
87771ed611cSPaolo Bonzini        "file",
87871ed611cSPaolo Bonzini        type=str,
87971ed611cSPaolo Bonzini        action="store",
88071ed611cSPaolo Bonzini        help=("Path to a TOML file describing package groups"),
88171ed611cSPaolo Bonzini    )
88271ed611cSPaolo Bonzini    subparser.add_argument(
88371ed611cSPaolo Bonzini        "group",
88471ed611cSPaolo Bonzini        type=str,
88571ed611cSPaolo Bonzini        action="store",
88671ed611cSPaolo Bonzini        help="One or more package group names",
88771ed611cSPaolo Bonzini        nargs="+",
88871ed611cSPaolo Bonzini    )
88971ed611cSPaolo Bonzini
89071ed611cSPaolo Bonzini
891c5538eedSJohn Snowdef _add_ensure_subcommand(subparsers: Any) -> None:
892c5538eedSJohn Snow    subparser = subparsers.add_parser(
893c5538eedSJohn Snow        "ensure", help="Ensure that the specified package is installed."
894c5538eedSJohn Snow    )
895c5538eedSJohn Snow    subparser.add_argument(
896c5538eedSJohn Snow        "--online",
897c5538eedSJohn Snow        action="store_true",
898c5538eedSJohn Snow        help="Install packages from PyPI, if necessary.",
899c5538eedSJohn Snow    )
900c5538eedSJohn Snow    subparser.add_argument(
901c5538eedSJohn Snow        "--dir",
902c5538eedSJohn Snow        type=str,
903c5538eedSJohn Snow        action="store",
904c5538eedSJohn Snow        help="Path to vendored packages where we may install from.",
905c5538eedSJohn Snow    )
906c5538eedSJohn Snow    subparser.add_argument(
9074695a22eSJohn Snow        "--diagnose",
9084695a22eSJohn Snow        type=str,
9094695a22eSJohn Snow        action="store",
9104695a22eSJohn Snow        help=(
9114695a22eSJohn Snow            "Name of a shell utility to use for "
9124695a22eSJohn Snow            "diagnostics if this command fails."
9134695a22eSJohn Snow        ),
9144695a22eSJohn Snow    )
9154695a22eSJohn Snow    subparser.add_argument(
916c5538eedSJohn Snow        "dep_specs",
917c5538eedSJohn Snow        type=str,
918c5538eedSJohn Snow        action="store",
919c5538eedSJohn Snow        help="PEP 508 Dependency specification, e.g. 'meson>=0.61.5'",
920c5538eedSJohn Snow        nargs="+",
921c5538eedSJohn Snow    )
922c5538eedSJohn Snow
923c5538eedSJohn Snow
924dd84028fSJohn Snowdef main() -> int:
925dd84028fSJohn Snow    """CLI interface to make_qemu_venv. See module docstring."""
926dd84028fSJohn Snow    if os.environ.get("DEBUG") or os.environ.get("GITLAB_CI"):
927dd84028fSJohn Snow        # You're welcome.
928dd84028fSJohn Snow        logging.basicConfig(level=logging.DEBUG)
929c5538eedSJohn Snow    else:
930c5538eedSJohn Snow        if os.environ.get("V"):
931dd84028fSJohn Snow            logging.basicConfig(level=logging.INFO)
932dd84028fSJohn Snow
933dd84028fSJohn Snow    parser = argparse.ArgumentParser(
934dd84028fSJohn Snow        prog="mkvenv",
935dd84028fSJohn Snow        description="QEMU pyvenv bootstrapping utility",
936dd84028fSJohn Snow    )
937dd84028fSJohn Snow    subparsers = parser.add_subparsers(
938dd84028fSJohn Snow        title="Commands",
939dd84028fSJohn Snow        dest="command",
94002312f1aSPaolo Bonzini        required=True,
941dd84028fSJohn Snow        metavar="command",
942dd84028fSJohn Snow        help="Description",
943dd84028fSJohn Snow    )
944dd84028fSJohn Snow
945dd84028fSJohn Snow    _add_create_subcommand(subparsers)
946f1ad527fSJohn Snow    _add_post_init_subcommand(subparsers)
947c5538eedSJohn Snow    _add_ensure_subcommand(subparsers)
94871ed611cSPaolo Bonzini    _add_ensuregroup_subcommand(subparsers)
949dd84028fSJohn Snow
950dd84028fSJohn Snow    args = parser.parse_args()
951dd84028fSJohn Snow    try:
952dd84028fSJohn Snow        if args.command == "create":
953dd84028fSJohn Snow            make_venv(
954dd84028fSJohn Snow                args.target,
955dd84028fSJohn Snow                system_site_packages=True,
956dd84028fSJohn Snow                clear=True,
957dd84028fSJohn Snow            )
958f1ad527fSJohn Snow        if args.command == "post_init":
959f1ad527fSJohn Snow            post_venv_setup()
960c5538eedSJohn Snow        if args.command == "ensure":
961c5538eedSJohn Snow            ensure(
962c5538eedSJohn Snow                dep_specs=args.dep_specs,
963c5538eedSJohn Snow                online=args.online,
964c5538eedSJohn Snow                wheels_dir=args.dir,
9654695a22eSJohn Snow                prog=args.diagnose,
966c5538eedSJohn Snow            )
96771ed611cSPaolo Bonzini        if args.command == "ensuregroup":
96871ed611cSPaolo Bonzini            ensure_group(
96971ed611cSPaolo Bonzini                file=args.file,
97071ed611cSPaolo Bonzini                groups=args.group,
97171ed611cSPaolo Bonzini                online=args.online,
97271ed611cSPaolo Bonzini                wheels_dir=args.dir,
97371ed611cSPaolo Bonzini            )
974dd84028fSJohn Snow        logger.debug("mkvenv.py %s: exiting", args.command)
975dd84028fSJohn Snow    except Ouch as exc:
976dd84028fSJohn Snow        print("\n*** Ouch! ***\n", file=sys.stderr)
977dd84028fSJohn Snow        print(str(exc), "\n\n", file=sys.stderr)
978dd84028fSJohn Snow        return 1
979dd84028fSJohn Snow    except SystemExit:
980dd84028fSJohn Snow        raise
981dd84028fSJohn Snow    except:  # pylint: disable=bare-except
982dd84028fSJohn Snow        logger.exception("mkvenv did not complete successfully:")
983dd84028fSJohn Snow        return 2
984dd84028fSJohn Snow    return 0
985dd84028fSJohn Snow
986dd84028fSJohn Snow
987dd84028fSJohn Snowif __name__ == "__main__":
988dd84028fSJohn Snow    sys.exit(main())
989