Merge pull request #15 from ferrocene/pa-pull-from-ferrocene

Pull changes from the Ferrocene monorepo

Author: pietroalbini

.github/workflows/ci.yml

@@ -16,11 +16,11 @@ jobs:
     steps:
       - uses: actions/checkout@v3
 
-      # Some Ferrocene builders require the use of Python 3.6. Use that on CI
+      # Some Ferrocene builders require the use of Python 3.9. Use that on CI
       # to make sure there are no surprises when we import into Ferrocene.
       - uses: actions/setup-python@v3
         with:
-          python-version: "3.6.x"
+          python-version: "3.9.x"
 
       - name: Check that the requirements are installable
         run: python3 -m pip install -r requirements.txt

exts/ferrocene_intersphinx_support.py

@@ -0,0 +1,93 @@
+# SPDX-License-Identifier: MIT OR Apache-2.0
+# SPDX-FileCopyrightText: Ferrous Systems and AdaCore
+
+# This extension adds some helpers needed to integrate Ferrocene's build system
+# with InterSphinx. More specifically, the extension:
+#
+# - Defines the "ferrocene-intersphinx" Sphinx builder, which only produces the
+#   objects.inv file required by InterSphinx. This is used to gather all the
+#   inventories for all of our documentation before actually building anything,
+#   as we have circular references between documents.
+#
+# - Defines the "ferrocene_intersphinx_mappings" configuration, which this
+#   extension deserializes from JSON and then adds to the intersphinx_mapping
+#   configuration. This is needed because the format of intersphinx_mapping is
+#   too complex to be provided with the -D flag.
+
+from sphinx.builders import Builder
+from sphinx.builders.html import StandaloneHTMLBuilder
+import json
+import sphinx
+
+
+class IntersphinxBuilder(Builder):
+    name = "ferrocene-intersphinx"
+    format = ""
+    epilog = "InterSphinx inventory file generated."
+    allow_parallel = True
+
+    def init(self):
+        self.standalone_html_builder = StandaloneHTMLBuilder(self.app, self.env)
+
+        # Do not emit any warning in the ferrocene-intersphinx builder: there
+        # will be warnings when using the builder, as the rest of the documents
+        # won't be built yet, but we don't care about them.
+        #
+        # Keeping the warnings will confuse people who read the build logs,
+        # thinking they should fix them while they're expected to happen.
+        #
+        # Unfortunately the only reliable way to suppress the warnings is
+        # monkey-patching Sphinx's code, as you cannot set a global filter in
+        # Python's logging module.
+        sphinx.util.logging.WarningStreamHandler.emit = lambda _self, _record: None
+
+    def build(self, *args, **kwargs):
+        # Normally you're not supposed to override the build() method, as
+        # Sphinx calls all the relevant overrideable methods from it.
+        #
+        # Unfortunately though, Sphinx doesn't execute the finish() method if
+        # there are no outdated docs (as we're simulating in this builder).
+        #
+        # Returning all documents from get_outdated_docs() would fix that
+        # problem, but would also execute all the post_transforms for all
+        # documents, which on large documents can take a while.
+        #
+        # Instead, we're returning an empty list of outdated documents, and
+        # manually dumping the inventory here after the parent build() returns.
+        super().build(*args, **kwargs)
+        self.standalone_html_builder.dump_inventory()
+
+    def get_outdated_docs(self):
+        return []
+
+    def prepare_writing(self, docnames):
+        pass
+
+    def write_doc(self, docname, doctree):
+        pass
+
+    def get_target_uri(self, docname, typ=None):
+        # Defer to the standalone HTML builder to generate builders.
+        return self.standalone_html_builder.get_target_uri(docname, typ)
+
+
+def inject_intersphinx_mappings(app, config):
+    if config.ferrocene_intersphinx_mappings is not None:
+        for inventory in json.loads(config.ferrocene_intersphinx_mappings):
+            config.intersphinx_mapping[inventory["name"]] = (
+                inventory["html_root"],
+                inventory["inventory"],
+            )
+
+
+def setup(app):
+    app.add_builder(IntersphinxBuilder)
+
+    app.add_config_value("ferrocene_intersphinx_mappings", None, "env", [str])
+    app.connect("config-inited", inject_intersphinx_mappings, priority=1)
+
+    return {
+        "version": "0",
+        "parallel_read_safe": True,
+        "parallel_write_safe": True,
+    }

exts/ferrocene_qualification/__init__.py

@@ -0,0 +1,32 @@
+# SPDX-License-Identifier: MIT OR Apache-2.0
+# SPDX-FileCopyrightText: Ferrous Systems and AdaCore
+
+from . import substitutions, document_id, domain, signature_page
+import string
+
+
+def setup(app):
+    substitutions.setup(app)
+    document_id.setup(app)
+    domain.setup(app)
+    signature_page.setup(app)
+
+    app.connect("config-inited", validate_config)
+    app.add_config_value("ferrocene_id", None, "env", [str])
+    app.add_config_value("ferrocene_substitutions_path", None, "env", [str])
+    app.add_config_value("ferrocene_signed", False, "env", [str])
+
+    return {
+        "version": "0",
+        "parallel_read_safe": True,
+        "parallel_write_safe": True,
+    }
+
+
+def validate_config(app, config):
+    for required in ["ferrocene_id", "ferrocene_substitutions_path"]:
+        if config[required] is None:
+            raise ValueError(f"Missing required {required} configuration")
+
+    if any(c not in string.ascii_uppercase for c in config["ferrocene_id"]):
+        raise ValueError("ferrocene_id can only be uppercase letters")

exts/ferrocene_qualification/document_id.py

@@ -0,0 +1,122 @@
+# SPDX-License-Identifier: MIT OR Apache-2.0
+# SPDX-FileCopyrightText: Ferrous Systems and AdaCore
+
+# This module is responsible for generating the ID of the whole document. This
+# ID is supposed to uniquely identify the revision of the document, and it must
+# not change if the underlying content doesn't change.
+#
+# We can't use the git commit for the ID, as a commit could change something
+# unrelated to the content. Instead, we hash the parsed doctrees after all the
+# transforms executed, to capture the manually written content and the content
+# generated by other extensions, and use that as the ID.
+
+from docutils import nodes
+from sphinx.transforms import SphinxTransform
+import hashlib
+import os
+import sphinx
+import struct
+
+
+class Hasher:
+    def __init__(self):
+        self.state = hashlib.sha1()
+
+    def string(self, string):
+        encoded = string.encode("utf-8")
+
+        self.state.update(struct.pack("<Q", len(encoded)))
+        self.state.update(encoded)
+
+    def node(self, node):
+        self.string(node.tagname)
+
+        if isinstance(node, nodes.Text):
+            self.state.update(b"\xFF")  # Text node
+            self.string(str(node))
+        else:
+            self.state.update(struct.pack("<Q", len(node.non_default_attributes())))
+            for name, value in node.attlist():
+                # The <document source=".."> attribute contains absolute paths
+                # in it, breaking reproducibility.
+                if isinstance(node, nodes.document) and name == "source":
+                    continue
+
+                self.string(name)
+                self.string(repr(value))
+
+            for child in node.children:
+                self.state.update(b"\xFE")  # Enter child node
+                self.node(child)
+                self.state.update(b"\xFD")  # Exit child node
+
+    def finalize(self):
+        return self.state.hexdigest()
+
+
+class HashDocument(SphinxTransform):
+    default_priority = 999
+
+    def apply(self):
+        hasher = Hasher()
+        hasher.node(self.document)
+
+        self.env.ferrocene_document_ids[self.env.docname] = hasher.finalize()
+
+
+def env_purge_doc(app, env, docname):
+    if not hasattr(env, "ferrocene_document_ids"):
+        env.ferrocene_document_ids = {}
+    if docname in env.ferrocene_document_ids:
+        del env.ferrocene_document_ids[docname]
+
+
+def env_merge_info(app, env, docnames, other):
+    for doc in docnames:
+        env.ferrocene_document_ids[doc] = other.ferrocene_document_ids[doc]
+
+
+def env_updated(app, env):
+    hasher = Hasher()
+    for document, hash in sorted(app.env.ferrocene_document_ids.items()):
+        hasher.string(document)
+        hasher.string(hash)
+
+    old_id = getattr(env, "ferrocene_document_id", None)
+    new_id = f"{app.config.ferrocene_id}-{hasher.finalize()}"
+
+    app.env.ferrocene_document_id = new_id
+
+    # Mark all documents as updated if the ID changed since the last
+    # invocation, to force all templates to use the new ID.
+    if old_id != new_id:
+        return env.all_docs.keys()
+
+
+def html_page_context(app, pagename, templatename, context, doctree):
+    context["document_id"] = app.env.ferrocene_document_id
+
+
+def write_document_id(app):
+    with sphinx.util.progress_message("writing document id"):
+        with open(os.path.join(app.outdir, "document-id.txt"), "w") as f:
+            f.write(f"{app.env.ferrocene_document_id}\n")
+
+
+def build_finished(app, exception):
+    if exception is not None:
+        return
+    write_document_id(app)
+
+
+def setup(app):
+    # We're not using an EnvCollector since it doesn't allow to set the
+    # priority for the transform. Instead, our custom transform can set its
+    # priority to 999. The downside is we have to connect events manually.
+    app.add_transform(HashDocument)
+    app.connect("env-purge-doc", env_purge_doc)
+    app.connect("env-merge-info", env_merge_info)
+
+    app.connect("env-updated", env_updated)
+    app.connect("html-page-context", html_page_context)
+    app.connect("build-finished", build_finished)

exts/ferrocene_qualification/domain.py

@@ -0,0 +1,90 @@
+# SPDX-License-Identifier: MIT OR Apache-2.0
+# SPDX-FileCopyrightText: Ferrous Systems and AdaCore
+
+from docutils import nodes
+from sphinx.directives import ObjectDescription
+from sphinx.domains import Domain, ObjType
+from sphinx.roles import XRefRole
+import sphinx
+
+
+class Id:
+    def __init__(self, document, id):
+        self.document = document
+        self.id = id
+
+
+class IdDirective(ObjectDescription):
+    has_content = False
+    required_arguments = 1
+    option_spec = {}
+
+    def handle_signature(self, sig, signode):
+        prefix = nodes.inline()
+        prefix["classes"].append("hide-inside-tables")
+        prefix += nodes.strong("", "Identifier:")
+        prefix += nodes.Text(" ")
+
+        signode += prefix
+        signode += nodes.literal("", sig)
+
+    def add_target_and_index(self, name_cls, sig, signode):
+        id = Id(self.env.docname, sig)
+        signode["ids"].append(id.id)
+
+        domain = self.env.get_domain("qualification")
+        domain.add_id(id)
+
+
+class QualificationDomain(Domain):
+    name = "qualification"
+    labels = "Qualification Documents"
+    directives = {
+        "id": IdDirective,
+    }
+    roles = {
+        "id": XRefRole(),
+    }
+    object_types = {
+        "id": ObjType("Identifier", "id"),
+    }
+
+    initial_data = {"ids": {}}
+    # Bump whenever the format of the data changes!
+    data_version = 1
+
+    def add_id(self, id):
+        self.data["ids"][id.id] = id
+
+    def clear_doc(self, docname):
+        self.data["ids"] = {
+            key: value
+            for key, value in self.data["ids"].items()
+            if value.document != docname
+        }
+
+    def merge_domaindata(self, docnames, otherdata):
+        for key, value in otherdata["ids"].items():
+            if value.document in docnames:
+                self.data["ids"][value.id] = value
+
+    def resolve_xref(self, env, fromdocname, builder, type, target, node, contnode):
+        if type != "id":
+            raise RuntimeError(f"unsupported xref type {type}")
+
+        if target not in self.data["ids"]:
+            return
+        id = self.data["ids"][target]
+
+        return sphinx.util.nodes.make_refnode(
+            builder, fromdocname, id.document, id.id, contnode
+        )
+
+    def get_objects(self):
+        for id in self.data["ids"].values():
+            # (name, display_name, type, document, anchor, priority)
+            yield id.id, id.id, "id", id.document, id.id, 1
+
+
+def setup(app):
+    app.add_domain(QualificationDomain)