1""" 2mkvenv - QEMU pyvenv bootstrapping utility 3 4usage: mkvenv [-h] command ... 5 6QEMU pyvenv bootstrapping utility 7 8options: 9 -h, --help show this help message and exit 10 11Commands: 12 command Description 13 create create a venv 14 ensure Ensure that the specified package is installed. 15 16-------------------------------------------------- 17 18usage: mkvenv create [-h] target 19 20positional arguments: 21 target Target directory to install virtual environment into. 22 23options: 24 -h, --help show this help message and exit 25 26-------------------------------------------------- 27 28usage: mkvenv ensure [-h] [--online] [--dir DIR] dep_spec... 29 30positional arguments: 31 dep_spec PEP 508 Dependency specification, e.g. 'meson>=0.61.5' 32 33options: 34 -h, --help show this help message and exit 35 --online Install packages from PyPI, if necessary. 36 --dir DIR Path to vendored packages where we may install from. 37 38""" 39 40# Copyright (C) 2022-2023 Red Hat, Inc. 41# 42# Authors: 43# John Snow <jsnow@redhat.com> 44# Paolo Bonzini <pbonzini@redhat.com> 45# 46# This work is licensed under the terms of the GNU GPL, version 2 or 47# later. See the COPYING file in the top-level directory. 48 49import argparse 50from importlib.util import find_spec 51import logging 52import os 53from pathlib import Path 54import re 55import shutil 56import site 57import subprocess 58import sys 59import sysconfig 60from types import SimpleNamespace 61from typing import ( 62 Any, 63 Iterator, 64 Optional, 65 Sequence, 66 Tuple, 67 Union, 68) 69import venv 70import warnings 71 72import distlib.database 73import distlib.scripts 74import distlib.version 75 76 77# Do not add any mandatory dependencies from outside the stdlib: 78# This script *must* be usable standalone! 79 80DirType = Union[str, bytes, "os.PathLike[str]", "os.PathLike[bytes]"] 81logger = logging.getLogger("mkvenv") 82 83 84def inside_a_venv() -> bool: 85 """Returns True if it is executed inside of a virtual environment.""" 86 return sys.prefix != sys.base_prefix 87 88 89class Ouch(RuntimeError): 90 """An Exception class we can't confuse with a builtin.""" 91 92 93class QemuEnvBuilder(venv.EnvBuilder): 94 """ 95 An extension of venv.EnvBuilder for building QEMU's configure-time venv. 96 97 The primary difference is that it emulates a "nested" virtual 98 environment when invoked from inside of an existing virtual 99 environment by including packages from the parent. 100 101 Parameters for base class init: 102 - system_site_packages: bool = False 103 - clear: bool = False 104 - symlinks: bool = False 105 - upgrade: bool = False 106 - with_pip: bool = False 107 - prompt: Optional[str] = None 108 - upgrade_deps: bool = False (Since 3.9) 109 """ 110 111 def __init__(self, *args: Any, **kwargs: Any) -> None: 112 logger.debug("QemuEnvBuilder.__init__(...)") 113 114 # For nested venv emulation: 115 self.use_parent_packages = False 116 if inside_a_venv(): 117 # Include parent packages only if we're in a venv and 118 # system_site_packages was True. 119 self.use_parent_packages = kwargs.pop( 120 "system_site_packages", False 121 ) 122 # Include system_site_packages only when the parent, 123 # The venv we are currently in, also does so. 124 kwargs["system_site_packages"] = sys.base_prefix in site.PREFIXES 125 126 if kwargs.get("with_pip", False): 127 check_ensurepip() 128 129 super().__init__(*args, **kwargs) 130 131 # Make the context available post-creation: 132 self._context: Optional[SimpleNamespace] = None 133 134 def get_parent_libpath(self) -> Optional[str]: 135 """Return the libpath of the parent venv, if applicable.""" 136 if self.use_parent_packages: 137 return sysconfig.get_path("purelib") 138 return None 139 140 @staticmethod 141 def compute_venv_libpath(context: SimpleNamespace) -> str: 142 """ 143 Compatibility wrapper for context.lib_path for Python < 3.12 144 """ 145 # Python 3.12+, not strictly necessary because it's documented 146 # to be the same as 3.10 code below: 147 if sys.version_info >= (3, 12): 148 return context.lib_path 149 150 # Python 3.10+ 151 if "venv" in sysconfig.get_scheme_names(): 152 lib_path = sysconfig.get_path( 153 "purelib", scheme="venv", vars={"base": context.env_dir} 154 ) 155 assert lib_path is not None 156 return lib_path 157 158 # For Python <= 3.9 we need to hardcode this. Fortunately the 159 # code below was the same in Python 3.6-3.10, so there is only 160 # one case. 161 if sys.platform == "win32": 162 return os.path.join(context.env_dir, "Lib", "site-packages") 163 return os.path.join( 164 context.env_dir, 165 "lib", 166 "python%d.%d" % sys.version_info[:2], 167 "site-packages", 168 ) 169 170 def ensure_directories(self, env_dir: DirType) -> SimpleNamespace: 171 logger.debug("ensure_directories(env_dir=%s)", env_dir) 172 self._context = super().ensure_directories(env_dir) 173 return self._context 174 175 def create(self, env_dir: DirType) -> None: 176 logger.debug("create(env_dir=%s)", env_dir) 177 super().create(env_dir) 178 assert self._context is not None 179 self.post_post_setup(self._context) 180 181 def post_post_setup(self, context: SimpleNamespace) -> None: 182 """ 183 The final, final hook. Enter the venv and run commands inside of it. 184 """ 185 if self.use_parent_packages: 186 # We're inside of a venv and we want to include the parent 187 # venv's packages. 188 parent_libpath = self.get_parent_libpath() 189 assert parent_libpath is not None 190 logger.debug("parent_libpath: %s", parent_libpath) 191 192 our_libpath = self.compute_venv_libpath(context) 193 logger.debug("our_libpath: %s", our_libpath) 194 195 pth_file = os.path.join(our_libpath, "nested.pth") 196 with open(pth_file, "w", encoding="UTF-8") as file: 197 file.write(parent_libpath + os.linesep) 198 199 def get_value(self, field: str) -> str: 200 """ 201 Get a string value from the context namespace after a call to build. 202 203 For valid field names, see: 204 https://docs.python.org/3/library/venv.html#venv.EnvBuilder.ensure_directories 205 """ 206 ret = getattr(self._context, field) 207 assert isinstance(ret, str) 208 return ret 209 210 211def check_ensurepip() -> None: 212 """ 213 Check that we have ensurepip. 214 215 Raise a fatal exception with a helpful hint if it isn't available. 216 """ 217 if not find_spec("ensurepip"): 218 msg = ( 219 "Python's ensurepip module is not found.\n" 220 "It's normally part of the Python standard library, " 221 "maybe your distribution packages it separately?\n" 222 "Either install ensurepip, or alleviate the need for it in the " 223 "first place by installing pip and setuptools for " 224 f"'{sys.executable}'.\n" 225 "(Hint: Debian puts ensurepip in its python3-venv package.)" 226 ) 227 raise Ouch(msg) 228 229 # ensurepip uses pyexpat, which can also go missing on us: 230 if not find_spec("pyexpat"): 231 msg = ( 232 "Python's pyexpat module is not found.\n" 233 "It's normally part of the Python standard library, " 234 "maybe your distribution packages it separately?\n" 235 "Either install pyexpat, or alleviate the need for it in the " 236 "first place by installing pip and setuptools for " 237 f"'{sys.executable}'.\n\n" 238 "(Hint: NetBSD's pkgsrc debundles this to e.g. 'py310-expat'.)" 239 ) 240 raise Ouch(msg) 241 242 243def make_venv( # pylint: disable=too-many-arguments 244 env_dir: Union[str, Path], 245 system_site_packages: bool = False, 246 clear: bool = True, 247 symlinks: Optional[bool] = None, 248 with_pip: bool = True, 249) -> None: 250 """ 251 Create a venv using `QemuEnvBuilder`. 252 253 This is analogous to the `venv.create` module-level convenience 254 function that is part of the Python stdblib, except it uses 255 `QemuEnvBuilder` instead. 256 257 :param env_dir: The directory to create/install to. 258 :param system_site_packages: 259 Allow inheriting packages from the system installation. 260 :param clear: When True, fully remove any prior venv and files. 261 :param symlinks: 262 Whether to use symlinks to the target interpreter or not. If 263 left unspecified, it will use symlinks except on Windows to 264 match behavior with the "venv" CLI tool. 265 :param with_pip: 266 Whether to install "pip" binaries or not. 267 """ 268 logger.debug( 269 "%s: make_venv(env_dir=%s, system_site_packages=%s, " 270 "clear=%s, symlinks=%s, with_pip=%s)", 271 __file__, 272 str(env_dir), 273 system_site_packages, 274 clear, 275 symlinks, 276 with_pip, 277 ) 278 279 if symlinks is None: 280 # Default behavior of standard venv CLI 281 symlinks = os.name != "nt" 282 283 builder = QemuEnvBuilder( 284 system_site_packages=system_site_packages, 285 clear=clear, 286 symlinks=symlinks, 287 with_pip=with_pip, 288 ) 289 290 style = "non-isolated" if builder.system_site_packages else "isolated" 291 nested = "" 292 if builder.use_parent_packages: 293 nested = f"(with packages from '{builder.get_parent_libpath()}') " 294 print( 295 f"mkvenv: Creating {style} virtual environment" 296 f" {nested}at '{str(env_dir)}'", 297 file=sys.stderr, 298 ) 299 300 try: 301 logger.debug("Invoking builder.create()") 302 try: 303 builder.create(str(env_dir)) 304 except SystemExit as exc: 305 # Some versions of the venv module raise SystemExit; *nasty*! 306 # We want the exception that prompted it. It might be a subprocess 307 # error that has output we *really* want to see. 308 logger.debug("Intercepted SystemExit from EnvBuilder.create()") 309 raise exc.__cause__ or exc.__context__ or exc 310 logger.debug("builder.create() finished") 311 except subprocess.CalledProcessError as exc: 312 logger.error("mkvenv subprocess failed:") 313 logger.error("cmd: %s", exc.cmd) 314 logger.error("returncode: %d", exc.returncode) 315 316 def _stringify(data: Union[str, bytes]) -> str: 317 if isinstance(data, bytes): 318 return data.decode() 319 return data 320 321 lines = [] 322 if exc.stdout: 323 lines.append("========== stdout ==========") 324 lines.append(_stringify(exc.stdout)) 325 lines.append("============================") 326 if exc.stderr: 327 lines.append("========== stderr ==========") 328 lines.append(_stringify(exc.stderr)) 329 lines.append("============================") 330 if lines: 331 logger.error(os.linesep.join(lines)) 332 333 raise Ouch("VENV creation subprocess failed.") from exc 334 335 # print the python executable to stdout for configure. 336 print(builder.get_value("env_exe")) 337 338 339def _gen_importlib(packages: Sequence[str]) -> Iterator[str]: 340 # pylint: disable=import-outside-toplevel 341 # pylint: disable=no-name-in-module 342 # pylint: disable=import-error 343 try: 344 # First preference: Python 3.8+ stdlib 345 from importlib.metadata import ( # type: ignore 346 PackageNotFoundError, 347 distribution, 348 ) 349 except ImportError as exc: 350 logger.debug("%s", str(exc)) 351 # Second preference: Commonly available PyPI backport 352 from importlib_metadata import ( # type: ignore 353 PackageNotFoundError, 354 distribution, 355 ) 356 357 def _generator() -> Iterator[str]: 358 for package in packages: 359 try: 360 entry_points = distribution(package).entry_points 361 except PackageNotFoundError: 362 continue 363 364 # The EntryPoints type is only available in 3.10+, 365 # treat this as a vanilla list and filter it ourselves. 366 entry_points = filter( 367 lambda ep: ep.group == "console_scripts", entry_points 368 ) 369 370 for entry_point in entry_points: 371 yield f"{entry_point.name} = {entry_point.value}" 372 373 return _generator() 374 375 376def _gen_pkg_resources(packages: Sequence[str]) -> Iterator[str]: 377 # pylint: disable=import-outside-toplevel 378 # Bundled with setuptools; has a good chance of being available. 379 import pkg_resources 380 381 def _generator() -> Iterator[str]: 382 for package in packages: 383 try: 384 eps = pkg_resources.get_entry_map(package, "console_scripts") 385 except pkg_resources.DistributionNotFound: 386 continue 387 388 for entry_point in eps.values(): 389 yield str(entry_point) 390 391 return _generator() 392 393 394def generate_console_scripts( 395 packages: Sequence[str], 396 python_path: Optional[str] = None, 397 bin_path: Optional[str] = None, 398) -> None: 399 """ 400 Generate script shims for console_script entry points in @packages. 401 """ 402 if python_path is None: 403 python_path = sys.executable 404 if bin_path is None: 405 bin_path = sysconfig.get_path("scripts") 406 assert bin_path is not None 407 408 logger.debug( 409 "generate_console_scripts(packages=%s, python_path=%s, bin_path=%s)", 410 packages, 411 python_path, 412 bin_path, 413 ) 414 415 if not packages: 416 return 417 418 def _get_entry_points() -> Iterator[str]: 419 """Python 3.7 compatibility shim for iterating entry points.""" 420 # Python 3.8+, or Python 3.7 with importlib_metadata installed. 421 try: 422 return _gen_importlib(packages) 423 except ImportError as exc: 424 logger.debug("%s", str(exc)) 425 426 # Python 3.7 with setuptools installed. 427 try: 428 return _gen_pkg_resources(packages) 429 except ImportError as exc: 430 logger.debug("%s", str(exc)) 431 raise Ouch( 432 "Neither importlib.metadata nor pkg_resources found, " 433 "can't generate console script shims.\n" 434 "Use Python 3.8+, or install importlib-metadata or setuptools." 435 ) from exc 436 437 maker = distlib.scripts.ScriptMaker(None, bin_path) 438 maker.variants = {""} 439 maker.clobber = False 440 441 for entry_point in _get_entry_points(): 442 for filename in maker.make(entry_point): 443 logger.debug("wrote console_script '%s'", filename) 444 445 446def pkgname_from_depspec(dep_spec: str) -> str: 447 """ 448 Parse package name out of a PEP-508 depspec. 449 450 See https://peps.python.org/pep-0508/#names 451 """ 452 match = re.match( 453 r"^([A-Z0-9]([A-Z0-9._-]*[A-Z0-9])?)", dep_spec, re.IGNORECASE 454 ) 455 if not match: 456 raise ValueError( 457 f"dep_spec '{dep_spec}'" 458 " does not appear to contain a valid package name" 459 ) 460 return match.group(0) 461 462 463def diagnose( 464 dep_spec: str, 465 online: bool, 466 wheels_dir: Optional[Union[str, Path]], 467 prog: Optional[str], 468) -> Tuple[str, bool]: 469 """ 470 Offer a summary to the user as to why a package failed to be installed. 471 472 :param dep_spec: The package we tried to ensure, e.g. 'meson>=0.61.5' 473 :param online: Did we allow PyPI access? 474 :param prog: 475 Optionally, a shell program name that can be used as a 476 bellwether to detect if this program is installed elsewhere on 477 the system. This is used to offer advice when a program is 478 detected for a different python version. 479 :param wheels_dir: 480 Optionally, a directory that was searched for vendored packages. 481 """ 482 # pylint: disable=too-many-branches 483 484 # Some errors are not particularly serious 485 bad = False 486 487 pkg_name = pkgname_from_depspec(dep_spec) 488 pkg_version = None 489 490 has_importlib = False 491 try: 492 # Python 3.8+ stdlib 493 # pylint: disable=import-outside-toplevel 494 # pylint: disable=no-name-in-module 495 # pylint: disable=import-error 496 from importlib.metadata import ( # type: ignore 497 PackageNotFoundError, 498 version, 499 ) 500 501 has_importlib = True 502 try: 503 pkg_version = version(pkg_name) 504 except PackageNotFoundError: 505 pass 506 except ModuleNotFoundError: 507 pass 508 509 lines = [] 510 511 if pkg_version: 512 lines.append( 513 f"Python package '{pkg_name}' version '{pkg_version}' was found," 514 " but isn't suitable." 515 ) 516 elif has_importlib: 517 lines.append( 518 f"Python package '{pkg_name}' was not found nor installed." 519 ) 520 else: 521 lines.append( 522 f"Python package '{pkg_name}' is either not found or" 523 " not a suitable version." 524 ) 525 526 if wheels_dir: 527 lines.append( 528 "No suitable version found in, or failed to install from" 529 f" '{wheels_dir}'." 530 ) 531 bad = True 532 533 if online: 534 lines.append("A suitable version could not be obtained from PyPI.") 535 bad = True 536 else: 537 lines.append( 538 "mkvenv was configured to operate offline and did not check PyPI." 539 ) 540 541 if prog and not pkg_version: 542 which = shutil.which(prog) 543 if which: 544 if sys.base_prefix in site.PREFIXES: 545 pypath = Path(sys.executable).resolve() 546 lines.append( 547 f"'{prog}' was detected on your system at '{which}', " 548 f"but the Python package '{pkg_name}' was not found by " 549 f"this Python interpreter ('{pypath}'). " 550 f"Typically this means that '{prog}' has been installed " 551 "against a different Python interpreter on your system." 552 ) 553 else: 554 lines.append( 555 f"'{prog}' was detected on your system at '{which}', " 556 "but the build is using an isolated virtual environment." 557 ) 558 bad = True 559 560 lines = [f" • {line}" for line in lines] 561 if bad: 562 lines.insert(0, f"Could not provide build dependency '{dep_spec}':") 563 else: 564 lines.insert(0, f"'{dep_spec}' not found:") 565 return os.linesep.join(lines), bad 566 567 568def pip_install( 569 args: Sequence[str], 570 online: bool = False, 571 wheels_dir: Optional[Union[str, Path]] = None, 572) -> None: 573 """ 574 Use pip to install a package or package(s) as specified in @args. 575 """ 576 loud = bool( 577 os.environ.get("DEBUG") 578 or os.environ.get("GITLAB_CI") 579 or os.environ.get("V") 580 ) 581 582 full_args = [ 583 sys.executable, 584 "-m", 585 "pip", 586 "install", 587 "--disable-pip-version-check", 588 "-v" if loud else "-q", 589 ] 590 if not online: 591 full_args += ["--no-index"] 592 if wheels_dir: 593 full_args += ["--find-links", f"file://{str(wheels_dir)}"] 594 full_args += list(args) 595 subprocess.run( 596 full_args, 597 check=True, 598 ) 599 600 601def _do_ensure( 602 dep_specs: Sequence[str], 603 online: bool = False, 604 wheels_dir: Optional[Union[str, Path]] = None, 605) -> None: 606 """ 607 Use pip to ensure we have the package specified by @dep_specs. 608 609 If the package is already installed, do nothing. If online and 610 wheels_dir are both provided, prefer packages found in wheels_dir 611 first before connecting to PyPI. 612 613 :param dep_specs: 614 PEP 508 dependency specifications. e.g. ['meson>=0.61.5']. 615 :param online: If True, fall back to PyPI. 616 :param wheels_dir: If specified, search this path for packages. 617 """ 618 with warnings.catch_warnings(): 619 warnings.filterwarnings( 620 "ignore", category=UserWarning, module="distlib" 621 ) 622 dist_path = distlib.database.DistributionPath(include_egg=True) 623 absent = [] 624 present = [] 625 for spec in dep_specs: 626 matcher = distlib.version.LegacyMatcher(spec) 627 dist = dist_path.get_distribution(matcher.name) 628 if dist is None or not matcher.match(dist.version): 629 absent.append(spec) 630 else: 631 logger.info("found %s", dist) 632 present.append(matcher.name) 633 634 if present: 635 generate_console_scripts(present) 636 637 if absent: 638 # Some packages are missing or aren't a suitable version, 639 # install a suitable (possibly vendored) package. 640 print(f"mkvenv: installing {', '.join(absent)}", file=sys.stderr) 641 pip_install(args=absent, online=online, wheels_dir=wheels_dir) 642 643 644def ensure( 645 dep_specs: Sequence[str], 646 online: bool = False, 647 wheels_dir: Optional[Union[str, Path]] = None, 648 prog: Optional[str] = None, 649) -> None: 650 """ 651 Use pip to ensure we have the package specified by @dep_specs. 652 653 If the package is already installed, do nothing. If online and 654 wheels_dir are both provided, prefer packages found in wheels_dir 655 first before connecting to PyPI. 656 657 :param dep_specs: 658 PEP 508 dependency specifications. e.g. ['meson>=0.61.5']. 659 :param online: If True, fall back to PyPI. 660 :param wheels_dir: If specified, search this path for packages. 661 :param prog: 662 If specified, use this program name for error diagnostics that will 663 be presented to the user. e.g., 'sphinx-build' can be used as a 664 bellwether for the presence of 'sphinx'. 665 """ 666 print(f"mkvenv: checking for {', '.join(dep_specs)}", file=sys.stderr) 667 try: 668 _do_ensure(dep_specs, online, wheels_dir) 669 except subprocess.CalledProcessError as exc: 670 # Well, that's not good. 671 msg, bad = diagnose(dep_specs[0], online, wheels_dir, prog) 672 if bad: 673 raise Ouch(msg) from exc 674 raise SystemExit(f"\n{msg}\n\n") from exc 675 676 677def _add_create_subcommand(subparsers: Any) -> None: 678 subparser = subparsers.add_parser("create", help="create a venv") 679 subparser.add_argument( 680 "target", 681 type=str, 682 action="store", 683 help="Target directory to install virtual environment into.", 684 ) 685 686 687def _add_ensure_subcommand(subparsers: Any) -> None: 688 subparser = subparsers.add_parser( 689 "ensure", help="Ensure that the specified package is installed." 690 ) 691 subparser.add_argument( 692 "--online", 693 action="store_true", 694 help="Install packages from PyPI, if necessary.", 695 ) 696 subparser.add_argument( 697 "--dir", 698 type=str, 699 action="store", 700 help="Path to vendored packages where we may install from.", 701 ) 702 subparser.add_argument( 703 "--diagnose", 704 type=str, 705 action="store", 706 help=( 707 "Name of a shell utility to use for " 708 "diagnostics if this command fails." 709 ), 710 ) 711 subparser.add_argument( 712 "dep_specs", 713 type=str, 714 action="store", 715 help="PEP 508 Dependency specification, e.g. 'meson>=0.61.5'", 716 nargs="+", 717 ) 718 719 720def main() -> int: 721 """CLI interface to make_qemu_venv. See module docstring.""" 722 if os.environ.get("DEBUG") or os.environ.get("GITLAB_CI"): 723 # You're welcome. 724 logging.basicConfig(level=logging.DEBUG) 725 else: 726 if os.environ.get("V"): 727 logging.basicConfig(level=logging.INFO) 728 729 # These are incredibly noisy even for V=1 730 logging.getLogger("distlib.metadata").addFilter(lambda record: False) 731 logging.getLogger("distlib.database").addFilter(lambda record: False) 732 733 parser = argparse.ArgumentParser( 734 prog="mkvenv", 735 description="QEMU pyvenv bootstrapping utility", 736 ) 737 subparsers = parser.add_subparsers( 738 title="Commands", 739 dest="command", 740 metavar="command", 741 help="Description", 742 ) 743 744 _add_create_subcommand(subparsers) 745 _add_ensure_subcommand(subparsers) 746 747 args = parser.parse_args() 748 try: 749 if args.command == "create": 750 make_venv( 751 args.target, 752 system_site_packages=True, 753 clear=True, 754 ) 755 if args.command == "ensure": 756 ensure( 757 dep_specs=args.dep_specs, 758 online=args.online, 759 wheels_dir=args.dir, 760 prog=args.diagnose, 761 ) 762 logger.debug("mkvenv.py %s: exiting", args.command) 763 except Ouch as exc: 764 print("\n*** Ouch! ***\n", file=sys.stderr) 765 print(str(exc), "\n\n", file=sys.stderr) 766 return 1 767 except SystemExit: 768 raise 769 except: # pylint: disable=bare-except 770 logger.exception("mkvenv did not complete successfully:") 771 return 2 772 return 0 773 774 775if __name__ == "__main__": 776 sys.exit(main()) 777