1#!/usr/bin/env python3 2# SPDX-License-Identifier: GPL-2.0 3# Copyright(c) 2025: Mauro Carvalho Chehab <mchehab+huawei@kernel.org> 4# 5# pylint: disable=R0903,R0912,R0913,R0914,R0917,C0301 6 7""" 8Install minimal supported requirements for different Sphinx versions 9and optionally test the build. 10""" 11 12import argparse 13import asyncio 14import os.path 15import shutil 16import sys 17import time 18import subprocess 19 20# Minimal python version supported by the building system. 21 22PYTHON = os.path.basename(sys.executable) 23 24min_python_bin = None 25 26for i in range(9, 13): 27 p = f"python3.{i}" 28 if shutil.which(p): 29 min_python_bin = p 30 break 31 32if not min_python_bin: 33 min_python_bin = PYTHON 34 35# Starting from 8.0, Python 3.9 is not supported anymore. 36PYTHON_VER_CHANGES = {(8, 0, 0): PYTHON} 37 38DEFAULT_VERSIONS_TO_TEST = [ 39 (3, 4, 3), # Minimal supported version 40 (5, 3, 0), # CentOS Stream 9 / AlmaLinux 9 41 (6, 1, 1), # Debian 12 42 (7, 2, 1), # openSUSE Leap 15.6 43 (7, 2, 6), # Ubuntu 24.04 LTS 44 (7, 4, 7), # Ubuntu 24.10 45 (7, 3, 0), # openSUSE Tumbleweed 46 (8, 1, 3), # Fedora 42 47 (8, 2, 3) # Latest version - covers rolling distros 48] 49 50# Sphinx versions to be installed and their incremental requirements 51SPHINX_REQUIREMENTS = { 52 # Oldest versions we support for each package required by Sphinx 3.4.3 53 (3, 4, 3): { 54 "docutils": "0.16", 55 "alabaster": "0.7.12", 56 "babel": "2.8.0", 57 "certifi": "2020.6.20", 58 "docutils": "0.16", 59 "idna": "2.10", 60 "imagesize": "1.2.0", 61 "Jinja2": "2.11.2", 62 "MarkupSafe": "1.1.1", 63 "packaging": "20.4", 64 "Pygments": "2.6.1", 65 "PyYAML": "5.1", 66 "requests": "2.24.0", 67 "snowballstemmer": "2.0.0", 68 "sphinxcontrib-applehelp": "1.0.2", 69 "sphinxcontrib-devhelp": "1.0.2", 70 "sphinxcontrib-htmlhelp": "1.0.3", 71 "sphinxcontrib-jsmath": "1.0.1", 72 "sphinxcontrib-qthelp": "1.0.3", 73 "sphinxcontrib-serializinghtml": "1.1.4", 74 "urllib3": "1.25.9", 75 }, 76 77 # Update package dependencies to a more modern base. The goal here 78 # is to avoid to many incremental changes for the next entries 79 (3, 5, 0): { 80 "alabaster": "0.7.13", 81 "babel": "2.17.0", 82 "certifi": "2025.6.15", 83 "idna": "3.10", 84 "imagesize": "1.4.1", 85 "packaging": "25.0", 86 "Pygments": "2.8.1", 87 "requests": "2.32.4", 88 "snowballstemmer": "3.0.1", 89 "sphinxcontrib-applehelp": "1.0.4", 90 "sphinxcontrib-htmlhelp": "2.0.1", 91 "sphinxcontrib-serializinghtml": "1.1.5", 92 "urllib3": "2.0.0", 93 }, 94 95 # Starting from here, ensure all docutils versions are covered with 96 # supported Sphinx versions. Other packages are upgraded only when 97 # required by pip 98 (4, 0, 0): { 99 "PyYAML": "5.1", 100 }, 101 (4, 1, 0): { 102 "docutils": "0.17", 103 "Pygments": "2.19.1", 104 "Jinja2": "3.0.3", 105 "MarkupSafe": "2.0", 106 }, 107 (4, 3, 0): {}, 108 (4, 4, 0): {}, 109 (4, 5, 0): { 110 "docutils": "0.17.1", 111 }, 112 (5, 0, 0): {}, 113 (5, 1, 0): {}, 114 (5, 2, 0): { 115 "docutils": "0.18", 116 "Jinja2": "3.1.2", 117 "MarkupSafe": "2.0", 118 "PyYAML": "5.3.1", 119 }, 120 (5, 3, 0): { 121 "docutils": "0.18.1", 122 }, 123 (6, 0, 0): {}, 124 (6, 1, 0): {}, 125 (6, 2, 0): { 126 "PyYAML": "5.4.1", 127 }, 128 (7, 0, 0): {}, 129 (7, 1, 0): {}, 130 (7, 2, 0): { 131 "docutils": "0.19", 132 "PyYAML": "6.0.1", 133 "sphinxcontrib-serializinghtml": "1.1.9", 134 }, 135 (7, 2, 6): { 136 "docutils": "0.20", 137 }, 138 (7, 3, 0): { 139 "alabaster": "0.7.14", 140 "PyYAML": "6.0.1", 141 "tomli": "2.0.1", 142 }, 143 (7, 4, 0): { 144 "docutils": "0.20.1", 145 "PyYAML": "6.0.1", 146 }, 147 (8, 0, 0): { 148 "docutils": "0.21", 149 }, 150 (8, 1, 0): { 151 "docutils": "0.21.1", 152 "PyYAML": "6.0.1", 153 "sphinxcontrib-applehelp": "1.0.7", 154 "sphinxcontrib-devhelp": "1.0.6", 155 "sphinxcontrib-htmlhelp": "2.0.6", 156 "sphinxcontrib-qthelp": "1.0.6", 157 }, 158 (8, 2, 0): { 159 "docutils": "0.21.2", 160 "PyYAML": "6.0.1", 161 "sphinxcontrib-serializinghtml": "1.1.9", 162 }, 163} 164 165 166class AsyncCommands: 167 """Excecute command synchronously""" 168 169 def __init__(self, fp=None): 170 171 self.stdout = None 172 self.stderr = None 173 self.output = None 174 self.fp = fp 175 176 def log(self, out, verbose, is_info=True): 177 out = out.removesuffix('\n') 178 179 if verbose: 180 if is_info: 181 print(out) 182 else: 183 print(out, file=sys.stderr) 184 185 if self.fp: 186 self.fp.write(out + "\n") 187 188 async def _read(self, stream, verbose, is_info): 189 """Ancillary routine to capture while displaying""" 190 191 while stream is not None: 192 line = await stream.readline() 193 if line: 194 out = line.decode("utf-8", errors="backslashreplace") 195 self.log(out, verbose, is_info) 196 if is_info: 197 self.stdout += out 198 else: 199 self.stderr += out 200 else: 201 break 202 203 async def run(self, cmd, capture_output=False, check=False, 204 env=None, verbose=True): 205 206 """ 207 Execute an arbitrary command, handling errors. 208 209 Please notice that this class is not thread safe 210 """ 211 212 self.stdout = "" 213 self.stderr = "" 214 215 self.log("$ " + " ".join(cmd), verbose) 216 217 proc = await asyncio.create_subprocess_exec(cmd[0], 218 *cmd[1:], 219 env=env, 220 stdout=asyncio.subprocess.PIPE, 221 stderr=asyncio.subprocess.PIPE) 222 223 # Handle input and output in realtime 224 await asyncio.gather( 225 self._read(proc.stdout, verbose, True), 226 self._read(proc.stderr, verbose, False), 227 ) 228 229 await proc.wait() 230 231 if check and proc.returncode > 0: 232 raise subprocess.CalledProcessError(returncode=proc.returncode, 233 cmd=" ".join(cmd), 234 output=self.stdout, 235 stderr=self.stderr) 236 237 if capture_output: 238 if proc.returncode > 0: 239 self.log(f"Error {proc.returncode}", verbose=True, is_info=False) 240 return "" 241 242 return self.output 243 244 ret = subprocess.CompletedProcess(args=cmd, 245 returncode=proc.returncode, 246 stdout=self.stdout, 247 stderr=self.stderr) 248 249 return ret 250 251 252class SphinxVenv: 253 """ 254 Installs Sphinx on one virtual env per Sphinx version with a minimal 255 set of dependencies, adjusting them to each specific version. 256 """ 257 258 def __init__(self): 259 """Initialize instance variables""" 260 261 self.built_time = {} 262 self.first_run = True 263 264 async def _handle_version(self, args, fp, 265 cur_ver, cur_requirements, python_bin): 266 """Handle a single Sphinx version""" 267 268 cmd = AsyncCommands(fp) 269 270 ver = ".".join(map(str, cur_ver)) 271 272 if not self.first_run and args.wait_input and args.build: 273 ret = input("Press Enter to continue or 'a' to abort: ").strip().lower() 274 if ret == "a": 275 print("Aborted.") 276 sys.exit() 277 else: 278 self.first_run = False 279 280 venv_dir = f"Sphinx_{ver}" 281 req_file = f"requirements_{ver}.txt" 282 283 cmd.log(f"\nSphinx {ver} with {python_bin}", verbose=True) 284 285 # Create venv 286 await cmd.run([python_bin, "-m", "venv", venv_dir], 287 verbose=args.verbose, check=True) 288 pip = os.path.join(venv_dir, "bin/pip") 289 290 # Create install list 291 reqs = [] 292 for pkg, verstr in cur_requirements.items(): 293 reqs.append(f"{pkg}=={verstr}") 294 295 reqs.append(f"Sphinx=={ver}") 296 297 await cmd.run([pip, "install"] + reqs, check=True, verbose=args.verbose) 298 299 # Freeze environment 300 result = await cmd.run([pip, "freeze"], verbose=False, check=True) 301 302 # Pip install succeeded. Write requirements file 303 if args.req_file: 304 with open(req_file, "w", encoding="utf-8") as fp: 305 fp.write(result.stdout) 306 307 if args.build: 308 start_time = time.time() 309 310 # Prepare a venv environment 311 env = os.environ.copy() 312 bin_dir = os.path.join(venv_dir, "bin") 313 env["PATH"] = bin_dir + ":" + env["PATH"] 314 env["VIRTUAL_ENV"] = venv_dir 315 if "PYTHONHOME" in env: 316 del env["PYTHONHOME"] 317 318 # Test doc build 319 await cmd.run(["make", "cleandocs"], env=env, check=True) 320 make = ["make"] 321 322 if args.output: 323 sphinx_build = os.path.realpath(f"{bin_dir}/sphinx-build") 324 make += [f"O={args.output}", f"SPHINXBUILD={sphinx_build}"] 325 326 if args.make_args: 327 make += args.make_args 328 329 make += args.targets 330 331 if args.verbose: 332 cmd.log(f". {bin_dir}/activate", verbose=True) 333 await cmd.run(make, env=env, check=True, verbose=True) 334 if args.verbose: 335 cmd.log("deactivate", verbose=True) 336 337 end_time = time.time() 338 elapsed_time = end_time - start_time 339 hours, minutes = divmod(elapsed_time, 3600) 340 minutes, seconds = divmod(minutes, 60) 341 342 hours = int(hours) 343 minutes = int(minutes) 344 seconds = int(seconds) 345 346 self.built_time[ver] = f"{hours:02d}:{minutes:02d}:{seconds:02d}" 347 348 cmd.log(f"Finished doc build for Sphinx {ver}. Elapsed time: {self.built_time[ver]}", verbose=True) 349 350 async def run(self, args): 351 """ 352 Navigate though multiple Sphinx versions, handling each of them 353 on a loop. 354 """ 355 356 if args.log: 357 fp = open(args.log, "w", encoding="utf-8") 358 if not args.verbose: 359 args.verbose = False 360 else: 361 fp = None 362 if not args.verbose: 363 args.verbose = True 364 365 cur_requirements = {} 366 python_bin = min_python_bin 367 368 vers = set(SPHINX_REQUIREMENTS.keys()) | set(args.versions) 369 370 for cur_ver in sorted(vers): 371 if cur_ver in SPHINX_REQUIREMENTS: 372 new_reqs = SPHINX_REQUIREMENTS[cur_ver] 373 cur_requirements.update(new_reqs) 374 375 if cur_ver in PYTHON_VER_CHANGES: # pylint: disable=R1715 376 python_bin = PYTHON_VER_CHANGES[cur_ver] 377 378 if cur_ver not in args.versions: 379 continue 380 381 if args.min_version: 382 if cur_ver < args.min_version: 383 continue 384 385 if args.max_version: 386 if cur_ver > args.max_version: 387 break 388 389 await self._handle_version(args, fp, cur_ver, cur_requirements, 390 python_bin) 391 392 if args.build: 393 cmd = AsyncCommands(fp) 394 cmd.log("\nSummary:", verbose=True) 395 for ver, elapsed_time in sorted(self.built_time.items()): 396 cmd.log(f"\tSphinx {ver} elapsed time: {elapsed_time}", 397 verbose=True) 398 399 if fp: 400 fp.close() 401 402def parse_version(ver_str): 403 """Convert a version string into a tuple.""" 404 405 return tuple(map(int, ver_str.split("."))) 406 407 408DEFAULT_VERS = " - " 409DEFAULT_VERS += "\n - ".join(map(lambda v: f"{v[0]}.{v[1]}.{v[2]}", 410 DEFAULT_VERSIONS_TO_TEST)) 411 412SCRIPT = os.path.relpath(__file__) 413 414DESCRIPTION = f""" 415This tool allows creating Python virtual environments for different 416Sphinx versions that are supported by the Linux Kernel build system. 417 418Besides creating the virtual environment, it can also test building 419the documentation using "make htmldocs" (and/or other doc targets). 420 421If called without "--versions" argument, it covers the versions shipped 422on major distros, plus the lowest supported version: 423 424{DEFAULT_VERS} 425 426A typical usage is to run: 427 428 {SCRIPT} -m -l sphinx_builds.log 429 430This will create one virtual env for the default version set and run 431"make htmldocs" for each version, creating a log file with the 432excecuted commands on it. 433 434NOTE: The build time can be very long, specially on old versions. Also, there 435is a known bug with Sphinx version 6.0.x: each subprocess uses a lot of 436memory. That, together with "-jauto" may cause OOM killer to cause 437failures at the doc generation. To minimize the risk, you may use the 438"-a" command line parameter to constrain the built directories and/or 439reduce the number of threads from "-jauto" to, for instance, "-j4": 440 441 {SCRIPT} -m -V 6.0.1 -a "SPHINXDIRS=process" "SPHINXOPTS='-j4'" 442 443""" 444 445MAKE_TARGETS = [ 446 "htmldocs", 447 "texinfodocs", 448 "infodocs", 449 "latexdocs", 450 "pdfdocs", 451 "epubdocs", 452 "xmldocs", 453] 454 455async def main(): 456 """Main program""" 457 458 parser = argparse.ArgumentParser(description=DESCRIPTION, 459 formatter_class=argparse.RawDescriptionHelpFormatter) 460 461 ver_group = parser.add_argument_group("Version range options") 462 463 ver_group.add_argument('-V', '--versions', nargs="*", 464 default=DEFAULT_VERSIONS_TO_TEST,type=parse_version, 465 help='Sphinx versions to test') 466 ver_group.add_argument('--min-version', "--min", type=parse_version, 467 help='Sphinx minimal version') 468 ver_group.add_argument('--max-version', "--max", type=parse_version, 469 help='Sphinx maximum version') 470 ver_group.add_argument('-f', '--full', action='store_true', 471 help='Add all Sphinx (major,minor) supported versions to the version range') 472 473 build_group = parser.add_argument_group("Build options") 474 475 build_group.add_argument('-b', '--build', action='store_true', 476 help='Build documentation') 477 build_group.add_argument('-a', '--make-args', nargs="*", 478 help='extra arguments for make, like SPHINXDIRS=netlink/specs', 479 ) 480 build_group.add_argument('-t', '--targets', nargs="+", choices=MAKE_TARGETS, 481 default=[MAKE_TARGETS[0]], 482 help="make build targets. Default: htmldocs.") 483 build_group.add_argument("-o", '--output', 484 help="output directory for the make O=OUTPUT") 485 486 other_group = parser.add_argument_group("Other options") 487 488 other_group.add_argument('-r', '--req-file', action='store_true', 489 help='write a requirements.txt file') 490 other_group.add_argument('-l', '--log', 491 help='Log command output on a file') 492 other_group.add_argument('-v', '--verbose', action='store_true', 493 help='Verbose all commands') 494 other_group.add_argument('-i', '--wait-input', action='store_true', 495 help='Wait for an enter before going to the next version') 496 497 args = parser.parse_args() 498 499 if not args.make_args: 500 args.make_args = [] 501 502 sphinx_versions = sorted(list(SPHINX_REQUIREMENTS.keys())) 503 504 if args.full: 505 args.versions += list(SPHINX_REQUIREMENTS.keys()) 506 507 venv = SphinxVenv() 508 await venv.run(args) 509 510 511# Call main method 512if __name__ == "__main__": 513 asyncio.run(main()) 514