1# coding=utf-8 2# 3# QEMU qapidoc QAPI file parsing extension 4# 5# Copyright (c) 2024-2025 Red Hat 6# Copyright (c) 2020 Linaro 7# 8# This work is licensed under the terms of the GNU GPLv2 or later. 9# See the COPYING file in the top-level directory. 10 11""" 12qapidoc is a Sphinx extension that implements the qapi-doc directive 13 14The purpose of this extension is to read the documentation comments 15in QAPI schema files, and insert them all into the current document. 16 17It implements one new rST directive, "qapi-doc::". 18Each qapi-doc:: directive takes one argument, which is the 19pathname of the schema file to process, relative to the source tree. 20 21The docs/conf.py file must set the qapidoc_srctree config value to 22the root of the QEMU source tree. 23 24The Sphinx documentation on writing extensions is at: 25https://www.sphinx-doc.org/en/master/development/index.html 26""" 27 28from __future__ import annotations 29 30__version__ = "2.0" 31 32from contextlib import contextmanager 33import os 34from pathlib import Path 35import re 36import sys 37from typing import TYPE_CHECKING 38 39from docutils import nodes 40from docutils.parsers.rst import directives 41from docutils.statemachine import StringList 42from qapi.error import QAPIError 43from qapi.parser import QAPIDoc 44from qapi.schema import ( 45 QAPISchema, 46 QAPISchemaArrayType, 47 QAPISchemaCommand, 48 QAPISchemaDefinition, 49 QAPISchemaEnumMember, 50 QAPISchemaEvent, 51 QAPISchemaFeature, 52 QAPISchemaMember, 53 QAPISchemaObjectType, 54 QAPISchemaObjectTypeMember, 55 QAPISchemaType, 56 QAPISchemaVisitor, 57) 58from qapi.source import QAPISourceInfo 59 60from qapidoc_legacy import QAPISchemaGenRSTVisitor # type: ignore 61from sphinx import addnodes 62from sphinx.directives.code import CodeBlock 63from sphinx.errors import ExtensionError 64from sphinx.util import logging 65from sphinx.util.docutils import SphinxDirective, switch_source_input 66from sphinx.util.nodes import nested_parse_with_titles 67 68 69if TYPE_CHECKING: 70 from typing import ( 71 Any, 72 Generator, 73 List, 74 Optional, 75 Sequence, 76 Union, 77 ) 78 79 from sphinx.application import Sphinx 80 from sphinx.util.typing import ExtensionMetadata 81 82 83logger = logging.getLogger(__name__) 84 85 86class Transmogrifier: 87 # pylint: disable=too-many-public-methods 88 89 # Field names used for different entity types: 90 field_types = { 91 "enum": "value", 92 "struct": "memb", 93 "union": "memb", 94 "event": "memb", 95 "command": "arg", 96 "alternate": "alt", 97 } 98 99 def __init__(self) -> None: 100 self._curr_ent: Optional[QAPISchemaDefinition] = None 101 self._result = StringList() 102 self.indent = 0 103 104 @property 105 def result(self) -> StringList: 106 return self._result 107 108 @property 109 def entity(self) -> QAPISchemaDefinition: 110 assert self._curr_ent is not None 111 return self._curr_ent 112 113 @property 114 def member_field_type(self) -> str: 115 return self.field_types[self.entity.meta] 116 117 # General-purpose rST generation functions 118 119 def get_indent(self) -> str: 120 return " " * self.indent 121 122 @contextmanager 123 def indented(self) -> Generator[None]: 124 self.indent += 1 125 try: 126 yield 127 finally: 128 self.indent -= 1 129 130 def add_line_raw(self, line: str, source: str, *lineno: int) -> None: 131 """Append one line of generated reST to the output.""" 132 133 # NB: Sphinx uses zero-indexed lines; subtract one. 134 lineno = tuple((n - 1 for n in lineno)) 135 136 if line.strip(): 137 # not a blank line 138 self._result.append( 139 self.get_indent() + line.rstrip("\n"), source, *lineno 140 ) 141 else: 142 self._result.append("", source, *lineno) 143 144 def add_line(self, content: str, info: QAPISourceInfo) -> None: 145 # NB: We *require* an info object; this works out OK because we 146 # don't document built-in objects that don't have 147 # one. Everything else should. 148 self.add_line_raw(content, info.fname, info.line) 149 150 def add_lines( 151 self, 152 content: str, 153 info: QAPISourceInfo, 154 ) -> None: 155 lines = content.splitlines(True) 156 for i, line in enumerate(lines): 157 self.add_line_raw(line, info.fname, info.line + i) 158 159 def ensure_blank_line(self) -> None: 160 # Empty document -- no blank line required. 161 if not self._result: 162 return 163 164 # Last line isn't blank, add one. 165 if self._result[-1].strip(): # pylint: disable=no-member 166 fname, line = self._result.info(-1) 167 assert isinstance(line, int) 168 # New blank line is credited to one-after the current last line. 169 # +2: correct for zero/one index, then increment by one. 170 self.add_line_raw("", fname, line + 2) 171 172 def add_field( 173 self, 174 kind: str, 175 name: str, 176 body: str, 177 info: QAPISourceInfo, 178 typ: Optional[str] = None, 179 ) -> None: 180 if typ: 181 text = f":{kind} {typ} {name}: {body}" 182 else: 183 text = f":{kind} {name}: {body}" 184 self.add_lines(text, info) 185 186 def format_type( 187 self, ent: Union[QAPISchemaDefinition | QAPISchemaMember] 188 ) -> Optional[str]: 189 if isinstance(ent, (QAPISchemaEnumMember, QAPISchemaFeature)): 190 return None 191 192 qapi_type = ent 193 optional = False 194 if isinstance(ent, QAPISchemaObjectTypeMember): 195 qapi_type = ent.type 196 optional = ent.optional 197 198 if isinstance(qapi_type, QAPISchemaArrayType): 199 ret = f"[{qapi_type.element_type.doc_type()}]" 200 else: 201 assert isinstance(qapi_type, QAPISchemaType) 202 tmp = qapi_type.doc_type() 203 assert tmp 204 ret = tmp 205 if optional: 206 ret += "?" 207 208 return ret 209 210 def generate_field( 211 self, 212 kind: str, 213 member: QAPISchemaMember, 214 body: str, 215 info: QAPISourceInfo, 216 ) -> None: 217 typ = self.format_type(member) 218 self.add_field(kind, member.name, body, info, typ) 219 220 # Transmogrification helpers 221 222 def visit_paragraph(self, section: QAPIDoc.Section) -> None: 223 # Squelch empty paragraphs. 224 if not section.text: 225 return 226 227 self.ensure_blank_line() 228 self.add_lines(section.text, section.info) 229 self.ensure_blank_line() 230 231 def visit_member(self, section: QAPIDoc.ArgSection) -> None: 232 # FIXME: ifcond for members 233 # TODO: features for members (documented at entity-level, 234 # but sometimes defined per-member. Should we add such 235 # information to member descriptions when we can?) 236 assert section.member 237 self.generate_field( 238 self.member_field_type, 239 section.member, 240 # TODO drop fallbacks when undocumented members are outlawed 241 section.text if section.text else "Not documented", 242 section.info, 243 ) 244 245 def visit_feature(self, section: QAPIDoc.ArgSection) -> None: 246 # FIXME - ifcond for features is not handled at all yet! 247 # Proposal: decorate the right-hand column with some graphical 248 # element to indicate conditional availability? 249 assert section.text # Guaranteed by parser.py 250 assert section.member 251 252 self.generate_field("feat", section.member, section.text, section.info) 253 254 def visit_returns(self, section: QAPIDoc.Section) -> None: 255 assert isinstance(self.entity, QAPISchemaCommand) 256 rtype = self.entity.ret_type 257 # q_empty can produce None, but we won't be documenting anything 258 # without an explicit return statement in the doc block, and we 259 # should not have any such explicit statements when there is no 260 # return value. 261 assert rtype 262 263 typ = self.format_type(rtype) 264 assert typ 265 assert section.text 266 self.add_field("return", typ, section.text, section.info) 267 268 def visit_errors(self, section: QAPIDoc.Section) -> None: 269 # FIXME: the formatting for errors may be inconsistent and may 270 # or may not require different newline placement to ensure 271 # proper rendering as a nested list. 272 self.add_lines(f":error:\n{section.text}", section.info) 273 274 def preamble(self, ent: QAPISchemaDefinition) -> None: 275 """ 276 Generate option lines for QAPI entity directives. 277 """ 278 if ent.doc and ent.doc.since: 279 assert ent.doc.since.kind == QAPIDoc.Kind.SINCE 280 # Generated from the entity's docblock; info location is exact. 281 self.add_line(f":since: {ent.doc.since.text}", ent.doc.since.info) 282 283 if ent.ifcond.is_present(): 284 doc = ent.ifcond.docgen() 285 assert ent.info 286 # Generated from entity definition; info location is approximate. 287 self.add_line(f":ifcond: {doc}", ent.info) 288 289 # Hoist special features such as :deprecated: and :unstable: 290 # into the options block for the entity. If, in the future, new 291 # special features are added, qapi-domain will chirp about 292 # unrecognized options and fail until they are handled in 293 # qapi-domain. 294 for feat in ent.features: 295 if feat.is_special(): 296 # FIXME: handle ifcond if present. How to display that 297 # information is TBD. 298 # Generated from entity def; info location is approximate. 299 assert feat.info 300 self.add_line(f":{feat.name}:", feat.info) 301 302 self.ensure_blank_line() 303 304 def _insert_member_pointer(self, ent: QAPISchemaDefinition) -> None: 305 306 def _get_target( 307 ent: QAPISchemaDefinition, 308 ) -> Optional[QAPISchemaDefinition]: 309 if isinstance(ent, (QAPISchemaCommand, QAPISchemaEvent)): 310 return ent.arg_type 311 if isinstance(ent, QAPISchemaObjectType): 312 return ent.base 313 return None 314 315 target = _get_target(ent) 316 if target is not None and not target.is_implicit(): 317 assert ent.info 318 self.add_field( 319 self.member_field_type, 320 "q_dummy", 321 f"The members of :qapi:type:`{target.name}`.", 322 ent.info, 323 "q_dummy", 324 ) 325 326 if isinstance(ent, QAPISchemaObjectType) and ent.branches is not None: 327 for variant in ent.branches.variants: 328 if variant.type.name == "q_empty": 329 continue 330 assert ent.info 331 self.add_field( 332 self.member_field_type, 333 "q_dummy", 334 f" When ``{ent.branches.tag_member.name}`` is " 335 f"``{variant.name}``: " 336 f"The members of :qapi:type:`{variant.type.name}`.", 337 ent.info, 338 "q_dummy", 339 ) 340 341 def visit_sections(self, ent: QAPISchemaDefinition) -> None: 342 sections = ent.doc.all_sections if ent.doc else [] 343 344 # Determine the index location at which we should generate 345 # documentation for "The members of ..." pointers. This should 346 # go at the end of the members section(s) if any. Note that 347 # index 0 is assumed to be a plain intro section, even if it is 348 # empty; and that a members section if present will always 349 # immediately follow the opening PLAIN section. 350 gen_index = 1 351 if len(sections) > 1: 352 while sections[gen_index].kind == QAPIDoc.Kind.MEMBER: 353 gen_index += 1 354 if gen_index >= len(sections): 355 break 356 357 # Add sections in source order: 358 for i, section in enumerate(sections): 359 # @var is translated to ``var``: 360 section.text = re.sub(r"@([\w-]+)", r"``\1``", section.text) 361 362 if section.kind == QAPIDoc.Kind.PLAIN: 363 self.visit_paragraph(section) 364 elif section.kind == QAPIDoc.Kind.MEMBER: 365 assert isinstance(section, QAPIDoc.ArgSection) 366 self.visit_member(section) 367 elif section.kind == QAPIDoc.Kind.FEATURE: 368 assert isinstance(section, QAPIDoc.ArgSection) 369 self.visit_feature(section) 370 elif section.kind in (QAPIDoc.Kind.SINCE, QAPIDoc.Kind.TODO): 371 # Since is handled in preamble, TODO is skipped intentionally. 372 pass 373 elif section.kind == QAPIDoc.Kind.RETURNS: 374 self.visit_returns(section) 375 elif section.kind == QAPIDoc.Kind.ERRORS: 376 self.visit_errors(section) 377 else: 378 assert False 379 380 # Generate "The members of ..." entries if necessary: 381 if i == gen_index - 1: 382 self._insert_member_pointer(ent) 383 384 self.ensure_blank_line() 385 386 # Transmogrification core methods 387 388 def visit_module(self, path: str) -> None: 389 name = Path(path).stem 390 # module directives are credited to the first line of a module file. 391 self.add_line_raw(f".. qapi:module:: {name}", path, 1) 392 self.ensure_blank_line() 393 394 def visit_freeform(self, doc: QAPIDoc) -> None: 395 # TODO: Once the old qapidoc transformer is deprecated, freeform 396 # sections can be updated to pure rST, and this transformed removed. 397 # 398 # For now, translate our micro-format into rST. Code adapted 399 # from Peter Maydell's freeform(). 400 401 assert len(doc.all_sections) == 1, doc.all_sections 402 body = doc.all_sections[0] 403 text = body.text 404 info = doc.info 405 406 if re.match(r"=+ ", text): 407 # Section/subsection heading (if present, will always be the 408 # first line of the block) 409 (heading, _, text) = text.partition("\n") 410 (leader, _, heading) = heading.partition(" ") 411 # Implicit +1 for heading in the containing .rst doc 412 level = len(leader) + 1 413 414 # https://www.sphinx-doc.org/en/master/usage/restructuredtext/basics.html#sections 415 markers = ' #*=_^"' 416 overline = level <= 2 417 marker = markers[level] 418 419 self.ensure_blank_line() 420 # This credits all 2 or 3 lines to the single source line. 421 if overline: 422 self.add_line(marker * len(heading), info) 423 self.add_line(heading, info) 424 self.add_line(marker * len(heading), info) 425 self.ensure_blank_line() 426 427 # Eat blank line(s) and advance info 428 trimmed = text.lstrip("\n") 429 text = trimmed 430 info = info.next_line(len(text) - len(trimmed) + 1) 431 432 self.add_lines(text, info) 433 self.ensure_blank_line() 434 435 def visit_entity(self, ent: QAPISchemaDefinition) -> None: 436 assert ent.info 437 438 try: 439 self._curr_ent = ent 440 441 # Squish structs and unions together into an "object" directive. 442 meta = ent.meta 443 if meta in ("struct", "union"): 444 meta = "object" 445 446 # This line gets credited to the start of the /definition/. 447 self.add_line(f".. qapi:{meta}:: {ent.name}", ent.info) 448 with self.indented(): 449 self.preamble(ent) 450 self.visit_sections(ent) 451 finally: 452 self._curr_ent = None 453 454 def set_namespace(self, namespace: str, source: str, lineno: int) -> None: 455 self.add_line_raw( 456 f".. qapi:namespace:: {namespace}", source, lineno + 1 457 ) 458 self.ensure_blank_line() 459 460 461class QAPISchemaGenDepVisitor(QAPISchemaVisitor): 462 """A QAPI schema visitor which adds Sphinx dependencies each module 463 464 This class calls the Sphinx note_dependency() function to tell Sphinx 465 that the generated documentation output depends on the input 466 schema file associated with each module in the QAPI input. 467 """ 468 469 def __init__(self, env: Any, qapidir: str) -> None: 470 self._env = env 471 self._qapidir = qapidir 472 473 def visit_module(self, name: str) -> None: 474 if name != "./builtin": 475 qapifile = self._qapidir + "/" + name 476 self._env.note_dependency(os.path.abspath(qapifile)) 477 super().visit_module(name) 478 479 480class NestedDirective(SphinxDirective): 481 def run(self) -> Sequence[nodes.Node]: 482 raise NotImplementedError 483 484 def do_parse(self, rstlist: StringList, node: nodes.Node) -> None: 485 """ 486 Parse rST source lines and add them to the specified node 487 488 Take the list of rST source lines rstlist, parse them as 489 rST, and add the resulting docutils nodes as children of node. 490 The nodes are parsed in a way that allows them to include 491 subheadings (titles) without confusing the rendering of 492 anything else. 493 """ 494 with switch_source_input(self.state, rstlist): 495 nested_parse_with_titles(self.state, rstlist, node) 496 497 498class QAPIDocDirective(NestedDirective): 499 """Extract documentation from the specified QAPI .json file""" 500 501 required_argument = 1 502 optional_arguments = 1 503 option_spec = { 504 "qapifile": directives.unchanged_required, 505 "namespace": directives.unchanged, 506 "transmogrify": directives.flag, 507 } 508 has_content = False 509 510 def new_serialno(self) -> str: 511 """Return a unique new ID string suitable for use as a node's ID""" 512 env = self.state.document.settings.env 513 return "qapidoc-%d" % env.new_serialno("qapidoc") 514 515 def transmogrify(self, schema: QAPISchema) -> nodes.Element: 516 logger.info("Transmogrifying QAPI to rST ...") 517 vis = Transmogrifier() 518 modules = set() 519 520 if "namespace" in self.options: 521 vis.set_namespace( 522 self.options["namespace"], *self.get_source_info() 523 ) 524 525 for doc in schema.docs: 526 module_source = doc.info.fname 527 if module_source not in modules: 528 vis.visit_module(module_source) 529 modules.add(module_source) 530 531 if doc.symbol: 532 ent = schema.lookup_entity(doc.symbol) 533 assert isinstance(ent, QAPISchemaDefinition) 534 vis.visit_entity(ent) 535 else: 536 vis.visit_freeform(doc) 537 538 logger.info("Transmogrification complete.") 539 540 contentnode = nodes.section() 541 content = vis.result 542 titles_allowed = True 543 544 logger.info("Transmogrifier running nested parse ...") 545 with switch_source_input(self.state, content): 546 if titles_allowed: 547 node: nodes.Element = nodes.section() 548 node.document = self.state.document 549 nested_parse_with_titles(self.state, content, contentnode) 550 else: 551 node = nodes.paragraph() 552 node.document = self.state.document 553 self.state.nested_parse(content, 0, contentnode) 554 logger.info("Transmogrifier's nested parse completed.") 555 556 if self.env.app.verbosity >= 2 or os.environ.get("DEBUG"): 557 argname = "_".join(Path(self.arguments[0]).parts) 558 name = Path(argname).stem + ".ir" 559 self.write_intermediate(content, name) 560 561 sys.stdout.flush() 562 return contentnode 563 564 def write_intermediate(self, content: StringList, filename: str) -> None: 565 logger.info( 566 "writing intermediate rST for '%s' to '%s'", 567 self.arguments[0], 568 filename, 569 ) 570 571 srctree = Path(self.env.app.config.qapidoc_srctree).resolve() 572 outlines = [] 573 lcol_width = 0 574 575 for i, line in enumerate(content): 576 src, lineno = content.info(i) 577 srcpath = Path(src).resolve() 578 srcpath = srcpath.relative_to(srctree) 579 580 lcol = f"{srcpath}:{lineno:04d}" 581 lcol_width = max(lcol_width, len(lcol)) 582 outlines.append((lcol, line)) 583 584 with open(filename, "w", encoding="UTF-8") as outfile: 585 for lcol, rcol in outlines: 586 outfile.write(lcol.rjust(lcol_width)) 587 outfile.write(" |") 588 if rcol: 589 outfile.write(f" {rcol}") 590 outfile.write("\n") 591 592 def legacy(self, schema: QAPISchema) -> nodes.Element: 593 vis = QAPISchemaGenRSTVisitor(self) 594 vis.visit_begin(schema) 595 for doc in schema.docs: 596 if doc.symbol: 597 vis.symbol(doc, schema.lookup_entity(doc.symbol)) 598 else: 599 vis.freeform(doc) 600 return vis.get_document_node() # type: ignore 601 602 def run(self) -> Sequence[nodes.Node]: 603 env = self.state.document.settings.env 604 qapifile = env.config.qapidoc_srctree + "/" + self.arguments[0] 605 qapidir = os.path.dirname(qapifile) 606 transmogrify = "transmogrify" in self.options 607 608 try: 609 schema = QAPISchema(qapifile) 610 611 # First tell Sphinx about all the schema files that the 612 # output documentation depends on (including 'qapifile' itself) 613 schema.visit(QAPISchemaGenDepVisitor(env, qapidir)) 614 except QAPIError as err: 615 # Launder QAPI parse errors into Sphinx extension errors 616 # so they are displayed nicely to the user 617 raise ExtensionError(str(err)) from err 618 619 if transmogrify: 620 contentnode = self.transmogrify(schema) 621 else: 622 contentnode = self.legacy(schema) 623 624 return contentnode.children 625 626 627class QMPExample(CodeBlock, NestedDirective): 628 """ 629 Custom admonition for QMP code examples. 630 631 When the :annotated: option is present, the body of this directive 632 is parsed as normal rST, but with any '::' code blocks set to use 633 the QMP lexer. Code blocks must be explicitly written by the user, 634 but this allows for intermingling explanatory paragraphs with 635 arbitrary rST syntax and code blocks for more involved examples. 636 637 When :annotated: is absent, the directive body is treated as a 638 simple standalone QMP code block literal. 639 """ 640 641 required_argument = 0 642 optional_arguments = 0 643 has_content = True 644 option_spec = { 645 "annotated": directives.flag, 646 "title": directives.unchanged, 647 } 648 649 def _highlightlang(self) -> addnodes.highlightlang: 650 """Return the current highlightlang setting for the document""" 651 node = None 652 doc = self.state.document 653 654 if hasattr(doc, "findall"): 655 # docutils >= 0.18.1 656 for node in doc.findall(addnodes.highlightlang): 657 pass 658 else: 659 for elem in doc.traverse(): 660 if isinstance(elem, addnodes.highlightlang): 661 node = elem 662 663 if node: 664 return node 665 666 # No explicit directive found, use defaults 667 node = addnodes.highlightlang( 668 lang=self.env.config.highlight_language, 669 force=False, 670 # Yes, Sphinx uses this value to effectively disable line 671 # numbers and not 0 or None or -1 or something. ¯\_(ツ)_/¯ 672 linenothreshold=sys.maxsize, 673 ) 674 return node 675 676 def admonition_wrap(self, *content: nodes.Node) -> List[nodes.Node]: 677 title = "Example:" 678 if "title" in self.options: 679 title = f"{title} {self.options['title']}" 680 681 admon = nodes.admonition( 682 "", 683 nodes.title("", title), 684 *content, 685 classes=["admonition", "admonition-example"], 686 ) 687 return [admon] 688 689 def run_annotated(self) -> List[nodes.Node]: 690 lang_node = self._highlightlang() 691 692 content_node: nodes.Element = nodes.section() 693 694 # Configure QMP highlighting for "::" blocks, if needed 695 if lang_node["lang"] != "QMP": 696 content_node += addnodes.highlightlang( 697 lang="QMP", 698 force=False, # "True" ignores lexing errors 699 linenothreshold=lang_node["linenothreshold"], 700 ) 701 702 self.do_parse(self.content, content_node) 703 704 # Restore prior language highlighting, if needed 705 if lang_node["lang"] != "QMP": 706 content_node += addnodes.highlightlang(**lang_node.attributes) 707 708 return content_node.children 709 710 def run(self) -> List[nodes.Node]: 711 annotated = "annotated" in self.options 712 713 if annotated: 714 content_nodes = self.run_annotated() 715 else: 716 self.arguments = ["QMP"] 717 content_nodes = super().run() 718 719 return self.admonition_wrap(*content_nodes) 720 721 722def setup(app: Sphinx) -> ExtensionMetadata: 723 """Register qapi-doc directive with Sphinx""" 724 app.setup_extension("qapi_domain") 725 app.add_config_value("qapidoc_srctree", None, "env") 726 app.add_directive("qapi-doc", QAPIDocDirective) 727 app.add_directive("qmp-example", QMPExample) 728 729 return { 730 "version": __version__, 731 "parallel_read_safe": True, 732 "parallel_write_safe": True, 733 } 734