1# coding=utf-8 2# 3# QEMU qapidoc QAPI file parsing extension 4# 5# Copyright (c) 2024-2025 Red Hat 6# Copyright (c) 2020 Linaro 7# 8# This work is licensed under the terms of the GNU GPLv2 or later. 9# See the COPYING file in the top-level directory. 10 11""" 12qapidoc is a Sphinx extension that implements the qapi-doc directive 13 14The purpose of this extension is to read the documentation comments 15in QAPI schema files, and insert them all into the current document. 16 17It implements one new rST directive, "qapi-doc::". 18Each qapi-doc:: directive takes one argument, which is the 19pathname of the schema file to process, relative to the source tree. 20 21The docs/conf.py file must set the qapidoc_srctree config value to 22the root of the QEMU source tree. 23 24The Sphinx documentation on writing extensions is at: 25https://www.sphinx-doc.org/en/master/development/index.html 26""" 27 28from __future__ import annotations 29 30 31__version__ = "2.0" 32 33from contextlib import contextmanager 34import os 35from pathlib import Path 36import re 37import sys 38from typing import TYPE_CHECKING 39 40from docutils import nodes 41from docutils.parsers.rst import directives 42from docutils.statemachine import StringList 43from qapi.error import QAPIError 44from qapi.parser import QAPIDoc 45from qapi.schema import ( 46 QAPISchema, 47 QAPISchemaArrayType, 48 QAPISchemaCommand, 49 QAPISchemaDefinition, 50 QAPISchemaEnumMember, 51 QAPISchemaEvent, 52 QAPISchemaFeature, 53 QAPISchemaMember, 54 QAPISchemaObjectType, 55 QAPISchemaObjectTypeMember, 56 QAPISchemaType, 57 QAPISchemaVisitor, 58) 59from qapi.source import QAPISourceInfo 60from sphinx import addnodes 61from sphinx.directives.code import CodeBlock 62from sphinx.errors import ExtensionError 63from sphinx.util import logging 64from sphinx.util.docutils import SphinxDirective, switch_source_input 65from sphinx.util.nodes import nested_parse_with_titles 66 67from qapidoc_legacy import QAPISchemaGenRSTVisitor # type: ignore 68 69 70if TYPE_CHECKING: 71 from typing import ( 72 Any, 73 Generator, 74 List, 75 Optional, 76 Sequence, 77 Union, 78 ) 79 80 from sphinx.application import Sphinx 81 from sphinx.util.typing import ExtensionMetadata 82 83 84logger = logging.getLogger(__name__) 85 86 87class Transmogrifier: 88 # pylint: disable=too-many-public-methods 89 90 # Field names used for different entity types: 91 field_types = { 92 "enum": "value", 93 "struct": "memb", 94 "union": "memb", 95 "event": "memb", 96 "command": "arg", 97 "alternate": "alt", 98 } 99 100 def __init__(self) -> None: 101 self._curr_ent: Optional[QAPISchemaDefinition] = None 102 self._result = StringList() 103 self.indent = 0 104 105 @property 106 def result(self) -> StringList: 107 return self._result 108 109 @property 110 def entity(self) -> QAPISchemaDefinition: 111 assert self._curr_ent is not None 112 return self._curr_ent 113 114 @property 115 def member_field_type(self) -> str: 116 return self.field_types[self.entity.meta] 117 118 # General-purpose rST generation functions 119 120 def get_indent(self) -> str: 121 return " " * self.indent 122 123 @contextmanager 124 def indented(self) -> Generator[None]: 125 self.indent += 1 126 try: 127 yield 128 finally: 129 self.indent -= 1 130 131 def add_line_raw(self, line: str, source: str, *lineno: int) -> None: 132 """Append one line of generated reST to the output.""" 133 134 # NB: Sphinx uses zero-indexed lines; subtract one. 135 lineno = tuple((n - 1 for n in lineno)) 136 137 if line.strip(): 138 # not a blank line 139 self._result.append( 140 self.get_indent() + line.rstrip("\n"), source, *lineno 141 ) 142 else: 143 self._result.append("", source, *lineno) 144 145 def add_line(self, content: str, info: QAPISourceInfo) -> None: 146 # NB: We *require* an info object; this works out OK because we 147 # don't document built-in objects that don't have 148 # one. Everything else should. 149 self.add_line_raw(content, info.fname, info.line) 150 151 def add_lines( 152 self, 153 content: str, 154 info: QAPISourceInfo, 155 ) -> None: 156 lines = content.splitlines(True) 157 for i, line in enumerate(lines): 158 self.add_line_raw(line, info.fname, info.line + i) 159 160 def ensure_blank_line(self) -> None: 161 # Empty document -- no blank line required. 162 if not self._result: 163 return 164 165 # Last line isn't blank, add one. 166 if self._result[-1].strip(): # pylint: disable=no-member 167 fname, line = self._result.info(-1) 168 assert isinstance(line, int) 169 # New blank line is credited to one-after the current last line. 170 # +2: correct for zero/one index, then increment by one. 171 self.add_line_raw("", fname, line + 2) 172 173 def add_field( 174 self, 175 kind: str, 176 name: str, 177 body: str, 178 info: QAPISourceInfo, 179 typ: Optional[str] = None, 180 ) -> None: 181 if typ: 182 text = f":{kind} {typ} {name}: {body}" 183 else: 184 text = f":{kind} {name}: {body}" 185 self.add_lines(text, info) 186 187 def format_type( 188 self, ent: Union[QAPISchemaDefinition | QAPISchemaMember] 189 ) -> Optional[str]: 190 if isinstance(ent, (QAPISchemaEnumMember, QAPISchemaFeature)): 191 return None 192 193 qapi_type = ent 194 optional = False 195 if isinstance(ent, QAPISchemaObjectTypeMember): 196 qapi_type = ent.type 197 optional = ent.optional 198 199 if isinstance(qapi_type, QAPISchemaArrayType): 200 ret = f"[{qapi_type.element_type.doc_type()}]" 201 else: 202 assert isinstance(qapi_type, QAPISchemaType) 203 tmp = qapi_type.doc_type() 204 assert tmp 205 ret = tmp 206 if optional: 207 ret += "?" 208 209 return ret 210 211 def generate_field( 212 self, 213 kind: str, 214 member: QAPISchemaMember, 215 body: str, 216 info: QAPISourceInfo, 217 ) -> None: 218 typ = self.format_type(member) 219 self.add_field(kind, member.name, body, info, typ) 220 221 # Transmogrification helpers 222 223 def visit_paragraph(self, section: QAPIDoc.Section) -> None: 224 # Squelch empty paragraphs. 225 if not section.text: 226 return 227 228 self.ensure_blank_line() 229 self.add_lines(section.text, section.info) 230 self.ensure_blank_line() 231 232 def visit_member(self, section: QAPIDoc.ArgSection) -> None: 233 # FIXME: ifcond for members 234 # TODO: features for members (documented at entity-level, 235 # but sometimes defined per-member. Should we add such 236 # information to member descriptions when we can?) 237 assert section.member 238 self.generate_field( 239 self.member_field_type, 240 section.member, 241 # TODO drop fallbacks when undocumented members are outlawed 242 section.text if section.text else "Not documented", 243 section.info, 244 ) 245 246 def visit_feature(self, section: QAPIDoc.ArgSection) -> None: 247 # FIXME - ifcond for features is not handled at all yet! 248 # Proposal: decorate the right-hand column with some graphical 249 # element to indicate conditional availability? 250 assert section.text # Guaranteed by parser.py 251 assert section.member 252 253 self.generate_field("feat", section.member, section.text, section.info) 254 255 def visit_returns(self, section: QAPIDoc.Section) -> None: 256 assert isinstance(self.entity, QAPISchemaCommand) 257 rtype = self.entity.ret_type 258 # q_empty can produce None, but we won't be documenting anything 259 # without an explicit return statement in the doc block, and we 260 # should not have any such explicit statements when there is no 261 # return value. 262 assert rtype 263 264 typ = self.format_type(rtype) 265 assert typ 266 assert section.text 267 self.add_field("return", typ, section.text, section.info) 268 269 def visit_errors(self, section: QAPIDoc.Section) -> None: 270 # FIXME: the formatting for errors may be inconsistent and may 271 # or may not require different newline placement to ensure 272 # proper rendering as a nested list. 273 self.add_lines(f":error:\n{section.text}", section.info) 274 275 def preamble(self, ent: QAPISchemaDefinition) -> None: 276 """ 277 Generate option lines for QAPI entity directives. 278 """ 279 if ent.doc and ent.doc.since: 280 assert ent.doc.since.kind == QAPIDoc.Kind.SINCE 281 # Generated from the entity's docblock; info location is exact. 282 self.add_line(f":since: {ent.doc.since.text}", ent.doc.since.info) 283 284 if ent.ifcond.is_present(): 285 doc = ent.ifcond.docgen() 286 assert ent.info 287 # Generated from entity definition; info location is approximate. 288 self.add_line(f":ifcond: {doc}", ent.info) 289 290 # Hoist special features such as :deprecated: and :unstable: 291 # into the options block for the entity. If, in the future, new 292 # special features are added, qapi-domain will chirp about 293 # unrecognized options and fail until they are handled in 294 # qapi-domain. 295 for feat in ent.features: 296 if feat.is_special(): 297 # FIXME: handle ifcond if present. How to display that 298 # information is TBD. 299 # Generated from entity def; info location is approximate. 300 assert feat.info 301 self.add_line(f":{feat.name}:", feat.info) 302 303 self.ensure_blank_line() 304 305 def _insert_member_pointer(self, ent: QAPISchemaDefinition) -> None: 306 307 def _get_target( 308 ent: QAPISchemaDefinition, 309 ) -> Optional[QAPISchemaDefinition]: 310 if isinstance(ent, (QAPISchemaCommand, QAPISchemaEvent)): 311 return ent.arg_type 312 if isinstance(ent, QAPISchemaObjectType): 313 return ent.base 314 return None 315 316 target = _get_target(ent) 317 if target is not None and not target.is_implicit(): 318 assert ent.info 319 self.add_field( 320 self.member_field_type, 321 "q_dummy", 322 f"The members of :qapi:type:`{target.name}`.", 323 ent.info, 324 "q_dummy", 325 ) 326 327 if isinstance(ent, QAPISchemaObjectType) and ent.branches is not None: 328 for variant in ent.branches.variants: 329 if variant.type.name == "q_empty": 330 continue 331 assert ent.info 332 self.add_field( 333 self.member_field_type, 334 "q_dummy", 335 f" When ``{ent.branches.tag_member.name}`` is " 336 f"``{variant.name}``: " 337 f"The members of :qapi:type:`{variant.type.name}`.", 338 ent.info, 339 "q_dummy", 340 ) 341 342 def visit_sections(self, ent: QAPISchemaDefinition) -> None: 343 sections = ent.doc.all_sections if ent.doc else [] 344 345 # Determine the index location at which we should generate 346 # documentation for "The members of ..." pointers. This should 347 # go at the end of the members section(s) if any. Note that 348 # index 0 is assumed to be a plain intro section, even if it is 349 # empty; and that a members section if present will always 350 # immediately follow the opening PLAIN section. 351 gen_index = 1 352 if len(sections) > 1: 353 while sections[gen_index].kind == QAPIDoc.Kind.MEMBER: 354 gen_index += 1 355 if gen_index >= len(sections): 356 break 357 358 # Add sections in source order: 359 for i, section in enumerate(sections): 360 # @var is translated to ``var``: 361 section.text = re.sub(r"@([\w-]+)", r"``\1``", section.text) 362 363 if section.kind == QAPIDoc.Kind.PLAIN: 364 self.visit_paragraph(section) 365 elif section.kind == QAPIDoc.Kind.MEMBER: 366 assert isinstance(section, QAPIDoc.ArgSection) 367 self.visit_member(section) 368 elif section.kind == QAPIDoc.Kind.FEATURE: 369 assert isinstance(section, QAPIDoc.ArgSection) 370 self.visit_feature(section) 371 elif section.kind in (QAPIDoc.Kind.SINCE, QAPIDoc.Kind.TODO): 372 # Since is handled in preamble, TODO is skipped intentionally. 373 pass 374 elif section.kind == QAPIDoc.Kind.RETURNS: 375 self.visit_returns(section) 376 elif section.kind == QAPIDoc.Kind.ERRORS: 377 self.visit_errors(section) 378 else: 379 assert False 380 381 # Generate "The members of ..." entries if necessary: 382 if i == gen_index - 1: 383 self._insert_member_pointer(ent) 384 385 self.ensure_blank_line() 386 387 # Transmogrification core methods 388 389 def visit_module(self, path: str) -> None: 390 name = Path(path).stem 391 # module directives are credited to the first line of a module file. 392 self.add_line_raw(f".. qapi:module:: {name}", path, 1) 393 self.ensure_blank_line() 394 395 def visit_freeform(self, doc: QAPIDoc) -> None: 396 # TODO: Once the old qapidoc transformer is deprecated, freeform 397 # sections can be updated to pure rST, and this transformed removed. 398 # 399 # For now, translate our micro-format into rST. Code adapted 400 # from Peter Maydell's freeform(). 401 402 assert len(doc.all_sections) == 1, doc.all_sections 403 body = doc.all_sections[0] 404 text = body.text 405 info = doc.info 406 407 if re.match(r"=+ ", text): 408 # Section/subsection heading (if present, will always be the 409 # first line of the block) 410 (heading, _, text) = text.partition("\n") 411 (leader, _, heading) = heading.partition(" ") 412 # Implicit +1 for heading in the containing .rst doc 413 level = len(leader) + 1 414 415 # https://www.sphinx-doc.org/en/master/usage/restructuredtext/basics.html#sections 416 markers = ' #*=_^"' 417 overline = level <= 2 418 marker = markers[level] 419 420 self.ensure_blank_line() 421 # This credits all 2 or 3 lines to the single source line. 422 if overline: 423 self.add_line(marker * len(heading), info) 424 self.add_line(heading, info) 425 self.add_line(marker * len(heading), info) 426 self.ensure_blank_line() 427 428 # Eat blank line(s) and advance info 429 trimmed = text.lstrip("\n") 430 text = trimmed 431 info = info.next_line(len(text) - len(trimmed) + 1) 432 433 self.add_lines(text, info) 434 self.ensure_blank_line() 435 436 def visit_entity(self, ent: QAPISchemaDefinition) -> None: 437 assert ent.info 438 439 try: 440 self._curr_ent = ent 441 442 # Squish structs and unions together into an "object" directive. 443 meta = ent.meta 444 if meta in ("struct", "union"): 445 meta = "object" 446 447 # This line gets credited to the start of the /definition/. 448 self.add_line(f".. qapi:{meta}:: {ent.name}", ent.info) 449 with self.indented(): 450 self.preamble(ent) 451 self.visit_sections(ent) 452 finally: 453 self._curr_ent = None 454 455 def set_namespace(self, namespace: str, source: str, lineno: int) -> None: 456 self.add_line_raw( 457 f".. qapi:namespace:: {namespace}", source, lineno + 1 458 ) 459 self.ensure_blank_line() 460 461 462class QAPISchemaGenDepVisitor(QAPISchemaVisitor): 463 """A QAPI schema visitor which adds Sphinx dependencies each module 464 465 This class calls the Sphinx note_dependency() function to tell Sphinx 466 that the generated documentation output depends on the input 467 schema file associated with each module in the QAPI input. 468 """ 469 470 def __init__(self, env: Any, qapidir: str) -> None: 471 self._env = env 472 self._qapidir = qapidir 473 474 def visit_module(self, name: str) -> None: 475 if name != "./builtin": 476 qapifile = self._qapidir + "/" + name 477 self._env.note_dependency(os.path.abspath(qapifile)) 478 super().visit_module(name) 479 480 481class NestedDirective(SphinxDirective): 482 def run(self) -> Sequence[nodes.Node]: 483 raise NotImplementedError 484 485 def do_parse(self, rstlist: StringList, node: nodes.Node) -> None: 486 """ 487 Parse rST source lines and add them to the specified node 488 489 Take the list of rST source lines rstlist, parse them as 490 rST, and add the resulting docutils nodes as children of node. 491 The nodes are parsed in a way that allows them to include 492 subheadings (titles) without confusing the rendering of 493 anything else. 494 """ 495 with switch_source_input(self.state, rstlist): 496 nested_parse_with_titles(self.state, rstlist, node) 497 498 499class QAPIDocDirective(NestedDirective): 500 """Extract documentation from the specified QAPI .json file""" 501 502 required_argument = 1 503 optional_arguments = 1 504 option_spec = { 505 "qapifile": directives.unchanged_required, 506 "namespace": directives.unchanged, 507 "transmogrify": directives.flag, 508 } 509 has_content = False 510 511 def new_serialno(self) -> str: 512 """Return a unique new ID string suitable for use as a node's ID""" 513 env = self.state.document.settings.env 514 return "qapidoc-%d" % env.new_serialno("qapidoc") 515 516 def transmogrify(self, schema: QAPISchema) -> nodes.Element: 517 logger.info("Transmogrifying QAPI to rST ...") 518 vis = Transmogrifier() 519 modules = set() 520 521 if "namespace" in self.options: 522 vis.set_namespace( 523 self.options["namespace"], *self.get_source_info() 524 ) 525 526 for doc in schema.docs: 527 module_source = doc.info.fname 528 if module_source not in modules: 529 vis.visit_module(module_source) 530 modules.add(module_source) 531 532 if doc.symbol: 533 ent = schema.lookup_entity(doc.symbol) 534 assert isinstance(ent, QAPISchemaDefinition) 535 vis.visit_entity(ent) 536 else: 537 vis.visit_freeform(doc) 538 539 logger.info("Transmogrification complete.") 540 541 contentnode = nodes.section() 542 content = vis.result 543 titles_allowed = True 544 545 logger.info("Transmogrifier running nested parse ...") 546 with switch_source_input(self.state, content): 547 if titles_allowed: 548 node: nodes.Element = nodes.section() 549 node.document = self.state.document 550 nested_parse_with_titles(self.state, content, contentnode) 551 else: 552 node = nodes.paragraph() 553 node.document = self.state.document 554 self.state.nested_parse(content, 0, contentnode) 555 logger.info("Transmogrifier's nested parse completed.") 556 557 if self.env.app.verbosity >= 2 or os.environ.get("DEBUG"): 558 argname = "_".join(Path(self.arguments[0]).parts) 559 name = Path(argname).stem + ".ir" 560 self.write_intermediate(content, name) 561 562 sys.stdout.flush() 563 return contentnode 564 565 def write_intermediate(self, content: StringList, filename: str) -> None: 566 logger.info( 567 "writing intermediate rST for '%s' to '%s'", 568 self.arguments[0], 569 filename, 570 ) 571 572 srctree = Path(self.env.app.config.qapidoc_srctree).resolve() 573 outlines = [] 574 lcol_width = 0 575 576 for i, line in enumerate(content): 577 src, lineno = content.info(i) 578 srcpath = Path(src).resolve() 579 srcpath = srcpath.relative_to(srctree) 580 581 lcol = f"{srcpath}:{lineno:04d}" 582 lcol_width = max(lcol_width, len(lcol)) 583 outlines.append((lcol, line)) 584 585 with open(filename, "w", encoding="UTF-8") as outfile: 586 for lcol, rcol in outlines: 587 outfile.write(lcol.rjust(lcol_width)) 588 outfile.write(" |") 589 if rcol: 590 outfile.write(f" {rcol}") 591 outfile.write("\n") 592 593 def legacy(self, schema: QAPISchema) -> nodes.Element: 594 vis = QAPISchemaGenRSTVisitor(self) 595 vis.visit_begin(schema) 596 for doc in schema.docs: 597 if doc.symbol: 598 vis.symbol(doc, schema.lookup_entity(doc.symbol)) 599 else: 600 vis.freeform(doc) 601 return vis.get_document_node() # type: ignore 602 603 def run(self) -> Sequence[nodes.Node]: 604 env = self.state.document.settings.env 605 qapifile = env.config.qapidoc_srctree + "/" + self.arguments[0] 606 qapidir = os.path.dirname(qapifile) 607 transmogrify = "transmogrify" in self.options 608 609 try: 610 schema = QAPISchema(qapifile) 611 612 # First tell Sphinx about all the schema files that the 613 # output documentation depends on (including 'qapifile' itself) 614 schema.visit(QAPISchemaGenDepVisitor(env, qapidir)) 615 except QAPIError as err: 616 # Launder QAPI parse errors into Sphinx extension errors 617 # so they are displayed nicely to the user 618 raise ExtensionError(str(err)) from err 619 620 if transmogrify: 621 contentnode = self.transmogrify(schema) 622 else: 623 contentnode = self.legacy(schema) 624 625 return contentnode.children 626 627 628class QMPExample(CodeBlock, NestedDirective): 629 """ 630 Custom admonition for QMP code examples. 631 632 When the :annotated: option is present, the body of this directive 633 is parsed as normal rST, but with any '::' code blocks set to use 634 the QMP lexer. Code blocks must be explicitly written by the user, 635 but this allows for intermingling explanatory paragraphs with 636 arbitrary rST syntax and code blocks for more involved examples. 637 638 When :annotated: is absent, the directive body is treated as a 639 simple standalone QMP code block literal. 640 """ 641 642 required_argument = 0 643 optional_arguments = 0 644 has_content = True 645 option_spec = { 646 "annotated": directives.flag, 647 "title": directives.unchanged, 648 } 649 650 def _highlightlang(self) -> addnodes.highlightlang: 651 """Return the current highlightlang setting for the document""" 652 node = None 653 doc = self.state.document 654 655 if hasattr(doc, "findall"): 656 # docutils >= 0.18.1 657 for node in doc.findall(addnodes.highlightlang): 658 pass 659 else: 660 for elem in doc.traverse(): 661 if isinstance(elem, addnodes.highlightlang): 662 node = elem 663 664 if node: 665 return node 666 667 # No explicit directive found, use defaults 668 node = addnodes.highlightlang( 669 lang=self.env.config.highlight_language, 670 force=False, 671 # Yes, Sphinx uses this value to effectively disable line 672 # numbers and not 0 or None or -1 or something. ¯\_(ツ)_/¯ 673 linenothreshold=sys.maxsize, 674 ) 675 return node 676 677 def admonition_wrap(self, *content: nodes.Node) -> List[nodes.Node]: 678 title = "Example:" 679 if "title" in self.options: 680 title = f"{title} {self.options['title']}" 681 682 admon = nodes.admonition( 683 "", 684 nodes.title("", title), 685 *content, 686 classes=["admonition", "admonition-example"], 687 ) 688 return [admon] 689 690 def run_annotated(self) -> List[nodes.Node]: 691 lang_node = self._highlightlang() 692 693 content_node: nodes.Element = nodes.section() 694 695 # Configure QMP highlighting for "::" blocks, if needed 696 if lang_node["lang"] != "QMP": 697 content_node += addnodes.highlightlang( 698 lang="QMP", 699 force=False, # "True" ignores lexing errors 700 linenothreshold=lang_node["linenothreshold"], 701 ) 702 703 self.do_parse(self.content, content_node) 704 705 # Restore prior language highlighting, if needed 706 if lang_node["lang"] != "QMP": 707 content_node += addnodes.highlightlang(**lang_node.attributes) 708 709 return content_node.children 710 711 def run(self) -> List[nodes.Node]: 712 annotated = "annotated" in self.options 713 714 if annotated: 715 content_nodes = self.run_annotated() 716 else: 717 self.arguments = ["QMP"] 718 content_nodes = super().run() 719 720 return self.admonition_wrap(*content_nodes) 721 722 723def setup(app: Sphinx) -> ExtensionMetadata: 724 """Register qapi-doc directive with Sphinx""" 725 app.setup_extension("qapi_domain") 726 app.add_config_value("qapidoc_srctree", None, "env") 727 app.add_directive("qapi-doc", QAPIDocDirective) 728 app.add_directive("qmp-example", QMPExample) 729 730 return { 731 "version": __version__, 732 "parallel_read_safe": True, 733 "parallel_write_safe": True, 734 } 735