diff --git a/Documentation/Makefile b/Documentation/Makefile index 3609cb86137b..65d184eab739 100644 --- a/Documentation/Makefile +++ b/Documentation/Makefile @@ -23,164 +23,76 @@ SPHINXOPTS = SPHINXDIRS = . DOCS_THEME = DOCS_CSS = -_SPHINXDIRS = $(sort $(patsubst $(srctree)/Documentation/%/index.rst,%,$(wildcard $(srctree)/Documentation/*/index.rst))) -SPHINX_CONF = conf.py +RUSTDOC = PAPER = BUILDDIR = $(obj)/output PDFLATEX = xelatex LATEXOPTS = -interaction=batchmode -no-shell-escape +PYTHONPYCACHEPREFIX ?= $(abspath $(BUILDDIR)/__pycache__) + +# Wrapper for sphinx-build + +BUILD_WRAPPER = $(srctree)/tools/docs/sphinx-build-wrapper + # For denylisting "variable font" files # Can be overridden by setting as an env variable FONTS_CONF_DENY_VF ?= $(HOME)/deny-vf -ifeq ($(findstring 1, $(KBUILD_VERBOSE)),) -SPHINXOPTS += "-q" -endif - # User-friendly check for sphinx-build HAVE_SPHINX := $(shell if which $(SPHINXBUILD) >/dev/null 2>&1; then echo 1; else echo 0; fi) +ifneq ($(wildcard $(srctree)/.config),) +ifeq ($(CONFIG_RUST),y) + RUSTDOC=--rustdoc +endif +endif + ifeq ($(HAVE_SPHINX),0) .DEFAULT: $(warning The '$(SPHINXBUILD)' command was not found. Make sure you have Sphinx installed and in PATH, or set the SPHINXBUILD make variable to point to the full path of the '$(SPHINXBUILD)' executable.) @echo - @$(srctree)/scripts/sphinx-pre-install + @$(srctree)/tools/docs/sphinx-pre-install @echo " SKIP Sphinx $@ target." else # HAVE_SPHINX -# User-friendly check for pdflatex and latexmk -HAVE_PDFLATEX := $(shell if which $(PDFLATEX) >/dev/null 2>&1; then echo 1; else echo 0; fi) -HAVE_LATEXMK := $(shell if which latexmk >/dev/null 2>&1; then echo 1; else echo 0; fi) +# Common documentation targets +htmldocs mandocs infodocs texinfodocs latexdocs epubdocs xmldocs pdfdocs linkcheckdocs: + $(Q)PYTHONPYCACHEPREFIX="$(PYTHONPYCACHEPREFIX)" \ + $(srctree)/tools/docs/sphinx-pre-install --version-check + +$(Q)PYTHONPYCACHEPREFIX="$(PYTHONPYCACHEPREFIX)" \ + $(PYTHON3) $(BUILD_WRAPPER) $@ \ + --sphinxdirs="$(SPHINXDIRS)" $(RUSTDOC) \ + --builddir="$(BUILDDIR)" --deny-vf=$(FONTS_CONF_DENY_VF) \ + --theme=$(DOCS_THEME) --css=$(DOCS_CSS) --paper=$(PAPER) -ifeq ($(HAVE_LATEXMK),1) - PDFLATEX := latexmk -$(PDFLATEX) -endif #HAVE_LATEXMK -# Internal variables. -PAPEROPT_a4 = -D latex_elements.papersize=a4paper -PAPEROPT_letter = -D latex_elements.papersize=letterpaper -ALLSPHINXOPTS = -D kerneldoc_srctree=$(srctree) -D kerneldoc_bin=$(KERNELDOC) -ALLSPHINXOPTS += $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) -ifneq ($(wildcard $(srctree)/.config),) -ifeq ($(CONFIG_RUST),y) - # Let Sphinx know we will include rustdoc - ALLSPHINXOPTS += -t rustdoc endif -endif -# the i18n builder cannot share the environment and doctrees with the others -I18NSPHINXOPTS = $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) . - -# commands; the 'cmd' from scripts/Kbuild.include is not *loopable* -loop_cmd = $(echo-cmd) $(cmd_$(1)) || exit; - -# $2 sphinx builder e.g. "html" -# $3 name of the build subfolder / e.g. "userspace-api/media", used as: -# * dest folder relative to $(BUILDDIR) and -# * cache folder relative to $(BUILDDIR)/.doctrees -# $4 dest subfolder e.g. "man" for man pages at userspace-api/media/man -# $5 reST source folder relative to $(src), -# e.g. "userspace-api/media" for the linux-tv book-set at ./Documentation/userspace-api/media - -PYTHONPYCACHEPREFIX ?= $(abspath $(BUILDDIR)/__pycache__) - -quiet_cmd_sphinx = SPHINX $@ --> file://$(abspath $(BUILDDIR)/$3/$4) - cmd_sphinx = \ - PYTHONPYCACHEPREFIX="$(PYTHONPYCACHEPREFIX)" \ - BUILDDIR=$(abspath $(BUILDDIR)) SPHINX_CONF=$(abspath $(src)/$5/$(SPHINX_CONF)) \ - $(PYTHON3) $(srctree)/scripts/jobserver-exec \ - $(CONFIG_SHELL) $(srctree)/Documentation/sphinx/parallel-wrapper.sh \ - $(SPHINXBUILD) \ - -b $2 \ - -c $(abspath $(src)) \ - -d $(abspath $(BUILDDIR)/.doctrees/$3) \ - -D version=$(KERNELVERSION) -D release=$(KERNELRELEASE) \ - $(ALLSPHINXOPTS) \ - $(abspath $(src)/$5) \ - $(abspath $(BUILDDIR)/$3/$4) && \ - if [ "x$(DOCS_CSS)" != "x" ]; then \ - cp $(if $(patsubst /%,,$(DOCS_CSS)),$(abspath $(srctree)/$(DOCS_CSS)),$(DOCS_CSS)) $(BUILDDIR)/$3/_static/; \ - fi - -htmldocs: - @$(srctree)/scripts/sphinx-pre-install --version-check - @+$(foreach var,$(SPHINXDIRS),$(call loop_cmd,sphinx,html,$(var),,$(var))) - -htmldocs-redirects: $(srctree)/Documentation/.renames.txt - @tools/docs/gen-redirects.py --output $(BUILDDIR) < $< - -# If Rust support is available and .config exists, add rustdoc generated contents. -# If there are any, the errors from this make rustdoc will be displayed but -# won't stop the execution of htmldocs - -ifneq ($(wildcard $(srctree)/.config),) -ifeq ($(CONFIG_RUST),y) - $(Q)$(MAKE) rustdoc || true -endif -endif - -texinfodocs: - @$(srctree)/scripts/sphinx-pre-install --version-check - @+$(foreach var,$(SPHINXDIRS),$(call loop_cmd,sphinx,texinfo,$(var),texinfo,$(var))) - -# Note: the 'info' Make target is generated by sphinx itself when -# running the texinfodocs target define above. -infodocs: texinfodocs - $(MAKE) -C $(BUILDDIR)/texinfo info - -linkcheckdocs: - @$(foreach var,$(SPHINXDIRS),$(call loop_cmd,sphinx,linkcheck,$(var),,$(var))) - -latexdocs: - @$(srctree)/scripts/sphinx-pre-install --version-check - @+$(foreach var,$(SPHINXDIRS),$(call loop_cmd,sphinx,latex,$(var),latex,$(var))) - -ifeq ($(HAVE_PDFLATEX),0) - -pdfdocs: - $(warning The '$(PDFLATEX)' command was not found. Make sure you have it installed and in PATH to produce PDF output.) - @echo " SKIP Sphinx $@ target." - -else # HAVE_PDFLATEX - -pdfdocs: DENY_VF = XDG_CONFIG_HOME=$(FONTS_CONF_DENY_VF) -pdfdocs: latexdocs - @$(srctree)/scripts/sphinx-pre-install --version-check - $(foreach var,$(SPHINXDIRS), \ - $(MAKE) PDFLATEX="$(PDFLATEX)" LATEXOPTS="$(LATEXOPTS)" $(DENY_VF) -C $(BUILDDIR)/$(var)/latex || sh $(srctree)/scripts/check-variable-fonts.sh || exit; \ - mkdir -p $(BUILDDIR)/$(var)/pdf; \ - mv $(subst .tex,.pdf,$(wildcard $(BUILDDIR)/$(var)/latex/*.tex)) $(BUILDDIR)/$(var)/pdf/; \ - ) - -endif # HAVE_PDFLATEX - -epubdocs: - @$(srctree)/scripts/sphinx-pre-install --version-check - @+$(foreach var,$(SPHINXDIRS),$(call loop_cmd,sphinx,epub,$(var),epub,$(var))) - -xmldocs: - @$(srctree)/scripts/sphinx-pre-install --version-check - @+$(foreach var,$(SPHINXDIRS),$(call loop_cmd,sphinx,xml,$(var),xml,$(var))) - -endif # HAVE_SPHINX # The following targets are independent of HAVE_SPHINX, and the rules should # work or silently pass without Sphinx. +htmldocs-redirects: $(srctree)/Documentation/.renames.txt + @tools/docs/gen-redirects.py --output $(BUILDDIR) < $< + refcheckdocs: $(Q)cd $(srctree);scripts/documentation-file-ref-check cleandocs: $(Q)rm -rf $(BUILDDIR) +# Used only on help +_SPHINXDIRS = $(sort $(patsubst $(srctree)/Documentation/%/index.rst,%,$(wildcard $(srctree)/Documentation/*/index.rst))) + dochelp: @echo ' Linux kernel internal documentation in different formats from ReST:' @echo ' htmldocs - HTML' @echo ' htmldocs-redirects - generate HTML redirects for moved pages' @echo ' texinfodocs - Texinfo' @echo ' infodocs - Info' + @echo ' mandocs - Man pages' @echo ' latexdocs - LaTeX' @echo ' pdfdocs - PDF' @echo ' epubdocs - EPUB' @@ -194,11 +106,13 @@ dochelp: @echo ' make SPHINXDIRS="s1 s2" [target] Generate only docs of folder s1, s2' @echo ' valid values for SPHINXDIRS are: $(_SPHINXDIRS)' @echo - @echo ' make SPHINX_CONF={conf-file} [target] use *additional* sphinx-build' - @echo ' configuration. This is e.g. useful to build with nit-picking config.' - @echo @echo ' make DOCS_THEME={sphinx-theme} selects a different Sphinx theme.' @echo @echo ' make DOCS_CSS={a .css file} adds a DOCS_CSS override file for html/epub output.' @echo + @echo ' make PAPER={a4|letter} Specifies the paper size used for LaTeX/PDF output.' + @echo + @echo ' make FONTS_CONF_DENY_VF={path} sets a deny list to block variable Noto CJK fonts' + @echo ' for PDF build. See tools/docs/lib/latex_fonts.py for more details' + @echo @echo ' Default location for the generated documents is Documentation/output' diff --git a/Documentation/conf.py b/Documentation/conf.py index 574896cca198..1ea2ae5c6276 100644 --- a/Documentation/conf.py +++ b/Documentation/conf.py @@ -18,8 +18,6 @@ import sphinx # documentation root, use os.path.abspath to make it absolute, like shown here. sys.path.insert(0, os.path.abspath("sphinx")) -from load_config import loadConfig # pylint: disable=C0413,E0401 - # Minimal supported version needs_sphinx = "3.4.3" @@ -93,8 +91,12 @@ def config_init(app, config): # LaTeX and PDF output require a list of documents with are dependent # of the app.srcdir. Add them here - # When SPHINXDIRS is used, we just need to get index.rst, if it exists + # Handle the case where SPHINXDIRS is used if not os.path.samefile(doctree, app.srcdir): + # Add a tag to mark that the build is actually a subproject + tags.add("subproject") + + # get index.rst, if it exists doc = os.path.basename(app.srcdir) fname = "index" if os.path.exists(os.path.join(app.srcdir, fname + ".rst")): @@ -583,13 +585,6 @@ pdf_documents = [ kerneldoc_bin = "../scripts/kernel-doc.py" kerneldoc_srctree = ".." -# ------------------------------------------------------------------------------ -# Since loadConfig overwrites settings from the global namespace, it has to be -# the last statement in the conf.py file -# ------------------------------------------------------------------------------ -loadConfig(globals()) - - def setup(app): """Patterns need to be updated at init time on older Sphinx versions""" diff --git a/Documentation/doc-guide/kernel-doc.rst b/Documentation/doc-guide/kernel-doc.rst index af9697e60165..4370cc8fbcf5 100644 --- a/Documentation/doc-guide/kernel-doc.rst +++ b/Documentation/doc-guide/kernel-doc.rst @@ -579,20 +579,23 @@ source. How to use kernel-doc to generate man pages ------------------------------------------- -If you just want to use kernel-doc to generate man pages you can do this -from the kernel git tree:: +To generate man pages for all files that contain kernel-doc markups, run:: - $ scripts/kernel-doc -man \ - $(git grep -l '/\*\*' -- :^Documentation :^tools) \ - | scripts/split-man.pl /tmp/man + $ make mandocs -Some older versions of git do not support some of the variants of syntax for -path exclusion. One of the following commands may work for those versions:: +Or calling ``script-build-wrapper`` directly:: - $ scripts/kernel-doc -man \ - $(git grep -l '/\*\*' -- . ':!Documentation' ':!tools') \ - | scripts/split-man.pl /tmp/man + $ ./tools/docs/sphinx-build-wrapper mandocs - $ scripts/kernel-doc -man \ - $(git grep -l '/\*\*' -- . ":(exclude)Documentation" ":(exclude)tools") \ - | scripts/split-man.pl /tmp/man +The output will be at ``/man`` directory inside the output directory +(by default: ``Documentation/output``). + +Optionally, it is possible to generate a partial set of man pages by +using SPHINXDIRS: + + $ make SPHINXDIRS=driver-api/media mandocs + +.. note:: + + When SPHINXDIRS={subdir} is used, it will only generate man pages for + the files explicitly inside a ``Documentation/{subdir}/.../*.rst`` file. diff --git a/Documentation/doc-guide/sphinx.rst b/Documentation/doc-guide/sphinx.rst index 607589592bfb..932f68c53075 100644 --- a/Documentation/doc-guide/sphinx.rst +++ b/Documentation/doc-guide/sphinx.rst @@ -106,7 +106,7 @@ There's a script that automatically checks for Sphinx dependencies. If it can recognize your distribution, it will also give a hint about the install command line options for your distro:: - $ ./scripts/sphinx-pre-install + $ ./tools/docs/sphinx-pre-install Checking if the needed tools for Fedora release 26 (Twenty Six) are available Warning: better to also install "texlive-luatex85". You should run: @@ -116,7 +116,7 @@ command line options for your distro:: . sphinx_2.4.4/bin/activate pip install -r Documentation/sphinx/requirements.txt - Can't build as 1 mandatory dependency is missing at ./scripts/sphinx-pre-install line 468. + Can't build as 1 mandatory dependency is missing at ./tools/docs/sphinx-pre-install line 468. By default, it checks all the requirements for both html and PDF, including the requirements for images, math expressions and LaTeX build, and assumes diff --git a/Documentation/sphinx/kerneldoc-preamble.sty b/Documentation/sphinx/kerneldoc-preamble.sty index 5d68395539fe..16d9ff46fdf6 100644 --- a/Documentation/sphinx/kerneldoc-preamble.sty +++ b/Documentation/sphinx/kerneldoc-preamble.sty @@ -220,7 +220,7 @@ If you want them, please install non-variable ``Noto Sans CJK'' font families along with the texlive-xecjk package by following instructions from - \sphinxcode{./scripts/sphinx-pre-install}. + \sphinxcode{./tools/docs/sphinx-pre-install}. Having optional non-variable ``Noto Serif CJK'' font families will improve the looks of those translations. \end{sphinxadmonition}} diff --git a/Documentation/sphinx/load_config.py b/Documentation/sphinx/load_config.py deleted file mode 100644 index 1afb0c97f06b..000000000000 --- a/Documentation/sphinx/load_config.py +++ /dev/null @@ -1,60 +0,0 @@ -# -*- coding: utf-8; mode: python -*- -# SPDX-License-Identifier: GPL-2.0 -# pylint: disable=R0903, C0330, R0914, R0912, E0401 - -import os -import sys -from sphinx.util.osutil import fs_encoding - -# ------------------------------------------------------------------------------ -def loadConfig(namespace): -# ------------------------------------------------------------------------------ - - """Load an additional configuration file into *namespace*. - - The name of the configuration file is taken from the environment - ``SPHINX_CONF``. The external configuration file extends (or overwrites) the - configuration values from the origin ``conf.py``. With this you are able to - maintain *build themes*. """ - - config_file = os.environ.get("SPHINX_CONF", None) - if (config_file is not None - and os.path.normpath(namespace["__file__"]) != os.path.normpath(config_file) ): - config_file = os.path.abspath(config_file) - - # Let's avoid one conf.py file just due to latex_documents - start = config_file.find('Documentation/') - if start >= 0: - start = config_file.find('/', start + 1) - - end = config_file.rfind('/') - if start >= 0 and end > 0: - dir = config_file[start + 1:end] - - print("source directory: %s" % dir) - new_latex_docs = [] - latex_documents = namespace['latex_documents'] - - for l in latex_documents: - if l[0].find(dir + '/') == 0: - has = True - fn = l[0][len(dir) + 1:] - new_latex_docs.append((fn, l[1], l[2], l[3], l[4])) - break - - namespace['latex_documents'] = new_latex_docs - - # If there is an extra conf.py file, load it - if os.path.isfile(config_file): - sys.stdout.write("load additional sphinx-config: %s\n" % config_file) - config = namespace.copy() - config['__file__'] = config_file - with open(config_file, 'rb') as f: - code = compile(f.read(), fs_encoding, 'exec') - exec(code, config) - del config['__file__'] - namespace.update(config) - else: - config = namespace.copy() - config['tags'].add("subproject") - namespace.update(config) diff --git a/Documentation/sphinx/parallel-wrapper.sh b/Documentation/sphinx/parallel-wrapper.sh deleted file mode 100644 index e54c44ce117d..000000000000 --- a/Documentation/sphinx/parallel-wrapper.sh +++ /dev/null @@ -1,33 +0,0 @@ -#!/bin/sh -# SPDX-License-Identifier: GPL-2.0+ -# -# Figure out if we should follow a specific parallelism from the make -# environment (as exported by scripts/jobserver-exec), or fall back to -# the "auto" parallelism when "-jN" is not specified at the top-level -# "make" invocation. - -sphinx="$1" -shift || true - -parallel="$PARALLELISM" -if [ -z "$parallel" ] ; then - # If no parallelism is specified at the top-level make, then - # fall back to the expected "-jauto" mode that the "htmldocs" - # target has had. - auto=$(perl -e 'open IN,"'"$sphinx"' --version 2>&1 |"; - while () { - if (m/([\d\.]+)/) { - print "auto" if ($1 >= "1.7") - } - } - close IN') - if [ -n "$auto" ] ; then - parallel="$auto" - fi -fi -# Only if some parallelism has been determined do we add the -jN option. -if [ -n "$parallel" ] ; then - parallel="-j$parallel" -fi - -exec "$sphinx" $parallel "$@" diff --git a/Documentation/translations/it_IT/doc-guide/sphinx.rst b/Documentation/translations/it_IT/doc-guide/sphinx.rst index 1f513bc33618..a5c5d935febf 100644 --- a/Documentation/translations/it_IT/doc-guide/sphinx.rst +++ b/Documentation/translations/it_IT/doc-guide/sphinx.rst @@ -109,7 +109,7 @@ Sphinx. Se lo script riesce a riconoscere la vostra distribuzione, allora sarà in grado di darvi dei suggerimenti su come procedere per completare l'installazione:: - $ ./scripts/sphinx-pre-install + $ ./tools/docs/sphinx-pre-install Checking if the needed tools for Fedora release 26 (Twenty Six) are available Warning: better to also install "texlive-luatex85". You should run: @@ -119,7 +119,7 @@ l'installazione:: . sphinx_2.4.4/bin/activate pip install -r Documentation/sphinx/requirements.txt - Can't build as 1 mandatory dependency is missing at ./scripts/sphinx-pre-install line 468. + Can't build as 1 mandatory dependency is missing at ./tools/docs/sphinx-pre-install line 468. L'impostazione predefinita prevede il controllo dei requisiti per la generazione di documenti html e PDF, includendo anche il supporto per le immagini, le diff --git a/Documentation/translations/zh_CN/doc-guide/sphinx.rst b/Documentation/translations/zh_CN/doc-guide/sphinx.rst index 23eac67fbc30..3375c6f3a811 100644 --- a/Documentation/translations/zh_CN/doc-guide/sphinx.rst +++ b/Documentation/translations/zh_CN/doc-guide/sphinx.rst @@ -84,7 +84,7 @@ PDF和LaTeX构建 这有一个脚本可以自动检查Sphinx依赖项。如果它认得您的发行版，还会提示您所用发行 版的安装命令:: - $ ./scripts/sphinx-pre-install + $ ./tools/docs/sphinx-pre-install Checking if the needed tools for Fedora release 26 (Twenty Six) are available Warning: better to also install "texlive-luatex85". You should run: @@ -94,7 +94,7 @@ PDF和LaTeX构建 . sphinx_2.4.4/bin/activate pip install -r Documentation/sphinx/requirements.txt - Can't build as 1 mandatory dependency is missing at ./scripts/sphinx-pre-install line 468. + Can't build as 1 mandatory dependency is missing at ./tools/docs/sphinx-pre-install line 468. 默认情况下，它会检查html和PDF的所有依赖项，包括图像、数学表达式和LaTeX构建的 需求，并假设将使用虚拟Python环境。html构建所需的依赖项被认为是必需的，其他依 diff --git a/Documentation/translations/zh_CN/how-to.rst b/Documentation/translations/zh_CN/how-to.rst index ddd99c0f9b4d..714664fec308 100644 --- a/Documentation/translations/zh_CN/how-to.rst +++ b/Documentation/translations/zh_CN/how-to.rst @@ -64,7 +64,7 @@ Linux 发行版和简单地使用 Linux 命令行，那么可以迅速开始了 :: cd linux - ./scripts/sphinx-pre-install + ./tools/docs/sphinx-pre-install 以 Fedora 为例，它的输出是这样的:: diff --git a/MAINTAINERS b/MAINTAINERS index 46126ce2f968..5aa6d769b254 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -7411,7 +7411,6 @@ S: Maintained P: Documentation/doc-guide/maintainer-profile.rst T: git git://git.lwn.net/linux.git docs-next F: Documentation/ -F: scripts/check-variable-fonts.sh F: scripts/checktransupdate.py F: scripts/documentation-file-ref-check F: scripts/get_abi.py @@ -7420,7 +7419,6 @@ F: scripts/lib/abi/* F: scripts/lib/kdoc/* F: tools/docs/* F: tools/net/ynl/pyynl/lib/doc_generator.py -F: scripts/sphinx-pre-install X: Documentation/ABI/ X: Documentation/admin-guide/media/ X: Documentation/devicetree/ @@ -7455,7 +7453,7 @@ L: linux-doc@vger.kernel.org S: Maintained F: Documentation/sphinx/parse-headers.pl F: scripts/documentation-file-ref-check -F: scripts/sphinx-pre-install +F: tools/docs/sphinx-pre-install DOCUMENTATION/ITALIAN M: Federico Vaga diff --git a/Makefile b/Makefile index 17cfa11ca716..9ce6f6f52ea8 100644 --- a/Makefile +++ b/Makefile @@ -1797,9 +1797,10 @@ $(help-board-dirs): help-%: # Documentation targets # --------------------------------------------------------------------------- -DOC_TARGETS := xmldocs latexdocs pdfdocs htmldocs htmldocs-redirects \ - epubdocs cleandocs linkcheckdocs dochelp refcheckdocs \ - texinfodocs infodocs +DOC_TARGETS := xmldocs latexdocs pdfdocs htmldocs epubdocs cleandocs \ + linkcheckdocs dochelp refcheckdocs texinfodocs infodocs mandocs \ + htmldocs-redirects + PHONY += $(DOC_TARGETS) $(DOC_TARGETS): $(Q)$(MAKE) $(build)=Documentation $@ diff --git a/scripts/check-variable-fonts.sh b/scripts/check-variable-fonts.sh deleted file mode 100755 index ce63f0acea5f..000000000000 --- a/scripts/check-variable-fonts.sh +++ /dev/null @@ -1,115 +0,0 @@ -#!/bin/sh -# SPDX-License-Identifier: GPL-2.0-only -# Copyright (C) Akira Yokosawa, 2024 -# -# For "make pdfdocs", reports of build errors of translations.pdf started -# arriving early 2024 [1, 2]. It turned out that Fedora and openSUSE -# tumbleweed have started deploying variable-font [3] format of "Noto CJK" -# fonts [4, 5]. For PDF, a LaTeX package named xeCJK is used for CJK -# (Chinese, Japanese, Korean) pages. xeCJK requires XeLaTeX/XeTeX, which -# does not (and likely never will) understand variable fonts for historical -# reasons. -# -# The build error happens even when both of variable- and non-variable-format -# fonts are found on the build system. To make matters worse, Fedora enlists -# variable "Noto CJK" fonts in the requirements of langpacks-ja, -ko, -zh_CN, -# -zh_TW, etc. Hence developers who have interest in CJK pages are more -# likely to encounter the build errors. -# -# This script is invoked from the error path of "make pdfdocs" and emits -# suggestions if variable-font files of "Noto CJK" fonts are in the list of -# fonts accessible from XeTeX. -# -# References: -# [1]: https://lore.kernel.org/r/8734tqsrt7.fsf@meer.lwn.net/ -# [2]: https://lore.kernel.org/r/1708585803.600323099@f111.i.mail.ru/ -# [3]: https://en.wikipedia.org/wiki/Variable_font -# [4]: https://fedoraproject.org/wiki/Changes/Noto_CJK_Variable_Fonts -# [5]: https://build.opensuse.org/request/show/1157217 -# -#=========================================================================== -# Workarounds for building translations.pdf -#=========================================================================== -# -# * Denylist "variable font" Noto CJK fonts. -# - Create $HOME/deny-vf/fontconfig/fonts.conf from template below, with -# tweaks if necessary. Remove leading "# ". -# - Path of fontconfig/fonts.conf can be overridden by setting an env -# variable FONTS_CONF_DENY_VF. -# -# * Template: -# ----------------------------------------------------------------- -# -# -# -# -# -# -# -# /usr/share/fonts/google-noto-*-cjk-vf-fonts -# -# /usr/share/fonts/truetype/Noto*CJK*-VF.otf -# -# -# -# ----------------------------------------------------------------- -# -# The denylisting is activated for "make pdfdocs". -# -# * For skipping CJK pages in PDF -# - Uninstall texlive-xecjk. -# Denylisting is not needed in this case. -# -# * For printing CJK pages in PDF -# - Need non-variable "Noto CJK" fonts. -# * Fedora -# - google-noto-sans-cjk-fonts -# - google-noto-serif-cjk-fonts -# * openSUSE tumbleweed -# - Non-variable "Noto CJK" fonts are not available as distro packages -# as of April, 2024. Fetch a set of font files from upstream Noto -# CJK Font released at: -# https://github.com/notofonts/noto-cjk/tree/main/Sans#super-otc -# and at: -# https://github.com/notofonts/noto-cjk/tree/main/Serif#super-otc -# , then uncompress and deploy them. -# - Remember to update fontconfig cache by running fc-cache. -# -# !!! Caution !!! -# Uninstalling "variable font" packages can be dangerous. -# They might be depended upon by other packages important for your work. -# Denylisting should be less invasive, as it is effective only while -# XeLaTeX runs in "make pdfdocs". - -# Default per-user fontconfig path (overridden by env variable) -: ${FONTS_CONF_DENY_VF:=$HOME/deny-vf} - -export XDG_CONFIG_HOME=${FONTS_CONF_DENY_VF} - -notocjkvffonts=`fc-list : file family variable | \ - grep 'variable=True' | \ - grep -E -e 'Noto (Sans|Sans Mono|Serif) CJK' | \ - sed -e 's/^/ /' -e 's/: Noto S.*$//' | sort | uniq` - -if [ "x$notocjkvffonts" != "x" ] ; then - echo '=============================================================================' - echo 'XeTeX is confused by "variable font" files listed below:' - echo "$notocjkvffonts" - echo - echo 'For CJK pages in PDF, they need to be hidden from XeTeX by denylisting.' - echo 'Or, CJK pages can be skipped by uninstalling texlive-xecjk.' - echo - echo 'For more info on denylisting, other options, and variable font, see header' - echo 'comments of scripts/check-variable-fonts.sh.' - echo '=============================================================================' -fi - -# As this script is invoked from Makefile's error path, always error exit -# regardless of whether any variable font is discovered or not. -exit 1 diff --git a/scripts/jobserver-exec b/scripts/jobserver-exec index 7eca035472d3..ae23afd344ec 100755 --- a/scripts/jobserver-exec +++ b/scripts/jobserver-exec @@ -1,77 +1,35 @@ #!/usr/bin/env python3 # SPDX-License-Identifier: GPL-2.0+ -# -# This determines how many parallel tasks "make" is expecting, as it is -# not exposed via an special variables, reserves them all, runs a subprocess -# with PARALLELISM environment variable set, and releases the jobs back again. -# -# https://www.gnu.org/software/make/manual/html_node/POSIX-Jobserver.html#POSIX-Jobserver -from __future__ import print_function -import os, sys, errno -import subprocess -# Extract and prepare jobserver file descriptors from environment. -claim = 0 -jobs = b"" -try: - # Fetch the make environment options. - flags = os.environ['MAKEFLAGS'] +""" +Determines how many parallel tasks "make" is expecting, as it is +not exposed via any special variables, reserves them all, runs a subprocess +with PARALLELISM environment variable set, and releases the jobs back again. - # Look for "--jobserver=R,W" - # Note that GNU Make has used --jobserver-fds and --jobserver-auth - # so this handles all of them. - opts = [x for x in flags.split(" ") if x.startswith("--jobserver")] +See: + https://www.gnu.org/software/make/manual/html_node/POSIX-Jobserver.html#POSIX-Jobserver +""" - # Parse out R,W file descriptor numbers and set them nonblocking. - # If the MAKEFLAGS variable contains multiple instances of the - # --jobserver-auth= option, the last one is relevant. - fds = opts[-1].split("=", 1)[1] +import os +import sys - # Starting with GNU Make 4.4, named pipes are used for reader and writer. - # Example argument: --jobserver-auth=fifo:/tmp/GMfifo8134 - _, _, path = fds.partition('fifo:') +LIB_DIR = "lib" +SRC_DIR = os.path.dirname(os.path.realpath(__file__)) - if path: - reader = os.open(path, os.O_RDONLY | os.O_NONBLOCK) - writer = os.open(path, os.O_WRONLY) - else: - reader, writer = [int(x) for x in fds.split(",", 1)] - # Open a private copy of reader to avoid setting nonblocking - # on an unexpecting process with the same reader fd. - reader = os.open("/proc/self/fd/%d" % (reader), - os.O_RDONLY | os.O_NONBLOCK) +sys.path.insert(0, os.path.join(SRC_DIR, LIB_DIR)) - # Read out as many jobserver slots as possible. - while True: - try: - slot = os.read(reader, 8) - jobs += slot - except (OSError, IOError) as e: - if e.errno == errno.EWOULDBLOCK: - # Stop at the end of the jobserver queue. - break - # If something went wrong, give back the jobs. - if len(jobs): - os.write(writer, jobs) - raise e - # Add a bump for our caller's reserveration, since we're just going - # to sit here blocked on our child. - claim = len(jobs) + 1 -except (KeyError, IndexError, ValueError, OSError, IOError) as e: - # Any missing environment strings or bad fds should result in just - # not being parallel. - pass +from jobserver import JobserverExec # pylint: disable=C0415 -# We can only claim parallelism if there was a jobserver (i.e. a top-level -# "-jN" argument) and there were no other failures. Otherwise leave out the -# environment variable and let the child figure out what is best. -if claim > 0: - os.environ['PARALLELISM'] = '%d' % (claim) -rc = subprocess.call(sys.argv[1:]) +def main(): + """Main program""" + if len(sys.argv) < 2: + name = os.path.basename(__file__) + sys.exit("usage: " + name +" command [args ...]\n" + __doc__) -# Return all the reserved slots. -if len(jobs): - os.write(writer, jobs) + with JobserverExec() as jobserver: + jobserver.run(sys.argv[1:]) -sys.exit(rc) + +if __name__ == "__main__": + main() diff --git a/scripts/lib/jobserver.py b/scripts/lib/jobserver.py new file mode 100755 index 000000000000..a24f30ef4fa8 --- /dev/null +++ b/scripts/lib/jobserver.py @@ -0,0 +1,149 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-2.0+ +# +# pylint: disable=C0103,C0209 +# +# + +""" +Interacts with the POSIX jobserver during the Kernel build time. + +A "normal" jobserver task, like the one initiated by a make subrocess would do: + + - open read/write file descriptors to communicate with the job server; + - ask for one slot by calling: + claim = os.read(reader, 1) + - when the job finshes, call: + os.write(writer, b"+") # os.write(writer, claim) + +Here, the goal is different: This script aims to get the remaining number +of slots available, using all of them to run a command which handle tasks in +parallel. To to that, it has a loop that ends only after there are no +slots left. It then increments the number by one, in order to allow a +call equivalent to make -j$((claim+1)), e.g. having a parent make creating +$claim child to do the actual work. + +The end goal here is to keep the total number of build tasks under the +limit established by the initial make -j$n_proc call. + +See: + https://www.gnu.org/software/make/manual/html_node/POSIX-Jobserver.html#POSIX-Jobserver +""" + +import errno +import os +import subprocess +import sys + +class JobserverExec: + """ + Claim all slots from make using POSIX Jobserver. + + The main methods here are: + - open(): reserves all slots; + - close(): method returns all used slots back to make; + - run(): executes a command setting PARALLELISM= + """ + + def __init__(self): + """Initialize internal vars""" + self.claim = 0 + self.jobs = b"" + self.reader = None + self.writer = None + self.is_open = False + + def open(self): + """Reserve all available slots to be claimed later on""" + + if self.is_open: + return + + try: + # Fetch the make environment options. + flags = os.environ["MAKEFLAGS"] + # Look for "--jobserver=R,W" + # Note that GNU Make has used --jobserver-fds and --jobserver-auth + # so this handles all of them. + opts = [x for x in flags.split(" ") if x.startswith("--jobserver")] + + # Parse out R,W file descriptor numbers and set them nonblocking. + # If the MAKEFLAGS variable contains multiple instances of the + # --jobserver-auth= option, the last one is relevant. + fds = opts[-1].split("=", 1)[1] + + # Starting with GNU Make 4.4, named pipes are used for reader + # and writer. + # Example argument: --jobserver-auth=fifo:/tmp/GMfifo8134 + _, _, path = fds.partition("fifo:") + + if path: + self.reader = os.open(path, os.O_RDONLY | os.O_NONBLOCK) + self.writer = os.open(path, os.O_WRONLY) + else: + self.reader, self.writer = [int(x) for x in fds.split(",", 1)] + # Open a private copy of reader to avoid setting nonblocking + # on an unexpecting process with the same reader fd. + self.reader = os.open("/proc/self/fd/%d" % (self.reader), + os.O_RDONLY | os.O_NONBLOCK) + + # Read out as many jobserver slots as possible + while True: + try: + slot = os.read(self.reader, 8) + self.jobs += slot + except (OSError, IOError) as e: + if e.errno == errno.EWOULDBLOCK: + # Stop at the end of the jobserver queue. + break + # If something went wrong, give back the jobs. + if self.jobs: + os.write(self.writer, self.jobs) + raise e + + # Add a bump for our caller's reserveration, since we're just going + # to sit here blocked on our child. + self.claim = len(self.jobs) + 1 + + except (KeyError, IndexError, ValueError, OSError, IOError): + # Any missing environment strings or bad fds should result in just + # not being parallel. + self.claim = None + + self.is_open = True + + def close(self): + """Return all reserved slots to Jobserver""" + + if not self.is_open: + return + + # Return all the reserved slots. + if len(self.jobs): + os.write(self.writer, self.jobs) + + self.is_open = False + + def __enter__(self): + self.open() + return self + + def __exit__(self, exc_type, exc_value, exc_traceback): + self.close() + + def run(self, cmd, *args, **pwargs): + """ + Run a command setting PARALLELISM env variable to the number of + available job slots (claim) + 1, e.g. it will reserve claim slots + to do the actual build work, plus one to monitor its children. + """ + self.open() # Ensure that self.claim is set + + # We can only claim parallelism if there was a jobserver (i.e. a + # top-level "-jN" argument) and there were no other failures. Otherwise + # leave out the environment variable and let the child figure out what + # is best. + if self.claim: + os.environ["PARALLELISM"] = str(self.claim) + + return subprocess.call(cmd, *args, **pwargs) diff --git a/scripts/lib/kdoc/kdoc_files.py b/scripts/lib/kdoc/kdoc_files.py index 9e09b45b02fa..061c033f32da 100644 --- a/scripts/lib/kdoc/kdoc_files.py +++ b/scripts/lib/kdoc/kdoc_files.py @@ -275,7 +275,10 @@ class KernelFiles(): self.config.log.warning("No kernel-doc for file %s", fname) continue - for arg in self.results[fname]: + symbols = self.results[fname] + self.out_style.set_symbols(symbols) + + for arg in symbols: m = self.out_msg(fname, arg.name, arg) if m is None: diff --git a/scripts/lib/kdoc/kdoc_item.py b/scripts/lib/kdoc/kdoc_item.py index b3b225764550..19805301cb2c 100644 --- a/scripts/lib/kdoc/kdoc_item.py +++ b/scripts/lib/kdoc/kdoc_item.py @@ -5,8 +5,9 @@ # class KdocItem: - def __init__(self, name, type, start_line, **other_stuff): + def __init__(self, name, fname, type, start_line, **other_stuff): self.name = name + self.fname = fname self.type = type self.declaration_start_line = start_line self.sections = {} diff --git a/scripts/lib/kdoc/kdoc_output.py b/scripts/lib/kdoc/kdoc_output.py index ea8914537ba0..58f115059e93 100644 --- a/scripts/lib/kdoc/kdoc_output.py +++ b/scripts/lib/kdoc/kdoc_output.py @@ -215,6 +215,9 @@ class OutputFormat: # Virtual methods to be overridden by inherited classes # At the base class, those do nothing. + def set_symbols(self, symbols): + """Get a list of all symbols from kernel_doc""" + def out_doc(self, fname, name, args): """Outputs a DOC block""" @@ -577,6 +580,7 @@ class ManFormat(OutputFormat): super().__init__() self.modulename = modulename + self.symbols = [] dt = None tstamp = os.environ.get("KBUILD_BUILD_TIMESTAMP") @@ -593,6 +597,69 @@ class ManFormat(OutputFormat): self.man_date = dt.strftime("%B %Y") + def arg_name(self, args, name): + """ + Return the name that will be used for the man page. + + As we may have the same name on different namespaces, + prepend the data type for all types except functions and typedefs. + + The doc section is special: it uses the modulename. + """ + + dtype = args.type + + if dtype == "doc": + return self.modulename + + if dtype in ["function", "typedef"]: + return name + + return f"{dtype} {name}" + + def set_symbols(self, symbols): + """ + Get a list of all symbols from kernel_doc. + + Man pages will uses it to add a SEE ALSO section with other + symbols at the same file. + """ + self.symbols = symbols + + def out_tail(self, fname, name, args): + """Adds a tail for all man pages""" + + # SEE ALSO section + self.data += f'.SH "SEE ALSO"' + "\n.PP\n" + self.data += (f"Kernel file \\fB{args.fname}\\fR\n") + if len(self.symbols) >= 2: + cur_name = self.arg_name(args, name) + + related = [] + for arg in self.symbols: + out_name = self.arg_name(arg, arg.name) + + if cur_name == out_name: + continue + + related.append(f"\\fB{out_name}\\fR(9)") + + self.data += ",\n".join(related) + "\n" + + # TODO: does it make sense to add other sections? Maybe + # REPORTING ISSUES? LICENSE? + + def msg(self, fname, name, args): + """ + Handles a single entry from kernel-doc parser. + + Add a tail at the end of man pages output. + """ + super().msg(fname, name, args) + self.out_tail(fname, name, args) + + return self.data + def output_highlight(self, block): """ Outputs a C symbol that may require being highlighted with @@ -618,7 +685,9 @@ class ManFormat(OutputFormat): if not self.check_doc(name, args): return - self.data += f'.TH "{self.modulename}" 9 "{self.modulename}" "{self.man_date}" "API Manual" LINUX' + "\n" + out_name = self.arg_name(args, name) + + self.data += f'.TH "{self.modulename}" 9 "{out_name}" "{self.man_date}" "API Manual" LINUX' + "\n" for section, text in args.sections.items(): self.data += f'.SH "{section}"' + "\n" @@ -627,7 +696,9 @@ class ManFormat(OutputFormat): def out_function(self, fname, name, args): """output function in man""" - self.data += f'.TH "{name}" 9 "{name}" "{self.man_date}" "Kernel Hacker\'s Manual" LINUX' + "\n" + out_name = self.arg_name(args, name) + + self.data += f'.TH "{name}" 9 "{out_name}" "{self.man_date}" "Kernel Hacker\'s Manual" LINUX' + "\n" self.data += ".SH NAME\n" self.data += f"{name} \\- {args['purpose']}\n" @@ -671,7 +742,9 @@ class ManFormat(OutputFormat): self.output_highlight(text) def out_enum(self, fname, name, args): - self.data += f'.TH "{self.modulename}" 9 "enum {name}" "{self.man_date}" "API Manual" LINUX' + "\n" + out_name = self.arg_name(args, name) + + self.data += f'.TH "{self.modulename}" 9 "{out_name}" "{self.man_date}" "API Manual" LINUX' + "\n" self.data += ".SH NAME\n" self.data += f"enum {name} \\- {args['purpose']}\n" @@ -703,8 +776,9 @@ class ManFormat(OutputFormat): def out_typedef(self, fname, name, args): module = self.modulename purpose = args.get('purpose') + out_name = self.arg_name(args, name) - self.data += f'.TH "{module}" 9 "{name}" "{self.man_date}" "API Manual" LINUX' + "\n" + self.data += f'.TH "{module}" 9 "{out_name}" "{self.man_date}" "API Manual" LINUX' + "\n" self.data += ".SH NAME\n" self.data += f"typedef {name} \\- {purpose}\n" @@ -717,8 +791,9 @@ class ManFormat(OutputFormat): module = self.modulename purpose = args.get('purpose') definition = args.get('definition') + out_name = self.arg_name(args, name) - self.data += f'.TH "{module}" 9 "{args.type} {name}" "{self.man_date}" "API Manual" LINUX' + "\n" + self.data += f'.TH "{module}" 9 "{out_name}" "{self.man_date}" "API Manual" LINUX' + "\n" self.data += ".SH NAME\n" self.data += f"{args.type} {name} \\- {purpose}\n" diff --git a/scripts/lib/kdoc/kdoc_parser.py b/scripts/lib/kdoc/kdoc_parser.py index 2376f180b1fa..6e5c115cbdf3 100644 --- a/scripts/lib/kdoc/kdoc_parser.py +++ b/scripts/lib/kdoc/kdoc_parser.py @@ -254,8 +254,9 @@ SECTION_DEFAULT = "Description" # default section class KernelEntry: - def __init__(self, config, ln): + def __init__(self, config, fname, ln): self.config = config + self.fname = fname self._contents = [] self.prototype = "" @@ -350,6 +351,7 @@ class KernelEntry: self.section = SECTION_DEFAULT self._contents = [] +python_warning = False class KernelDoc: """ @@ -383,9 +385,13 @@ class KernelDoc: # We need Python 3.7 for its "dicts remember the insertion # order" guarantee # - if sys.version_info.major == 3 and sys.version_info.minor < 7: + global python_warning + if (not python_warning and + sys.version_info.major == 3 and sys.version_info.minor < 7): + self.emit_msg(0, 'Python 3.7 or later is required for correct results') + python_warning = True def emit_msg(self, ln, msg, warning=True): """Emit a message""" @@ -417,7 +423,8 @@ class KernelDoc: The actual output and output filters will be handled elsewhere """ - item = KdocItem(name, dtype, self.entry.declaration_start_line, **args) + item = KdocItem(name, self.fname, dtype, + self.entry.declaration_start_line, **args) item.warnings = self.entry.warnings # Drop empty sections @@ -440,7 +447,7 @@ class KernelDoc: variables used by the state machine. """ - self.entry = KernelEntry(self.config, ln) + self.entry = KernelEntry(self.config, self.fname, ln) # State flags self.state = state.NORMAL diff --git a/scripts/split-man.pl b/scripts/split-man.pl deleted file mode 100755 index 96bd99dc977a..000000000000 --- a/scripts/split-man.pl +++ /dev/null @@ -1,28 +0,0 @@ -#!/usr/bin/env perl -# SPDX-License-Identifier: GPL-2.0 -# -# Author: Mauro Carvalho Chehab -# -# Produce manpages from kernel-doc. -# See Documentation/doc-guide/kernel-doc.rst for instructions - -if ($#ARGV < 0) { - die "where do I put the results?\n"; -} - -mkdir $ARGV[0],0777; -$state = 0; -while () { - if (/^\.TH \"[^\"]*\" 9 \"([^\"]*)\"/) { - if ($state == 1) { close OUT } - $state = 1; - $fn = "$ARGV[0]/$1.9"; - print STDERR "Creating $fn\n"; - open OUT, ">$fn" or die "can't open $fn: $!\n"; - print OUT $_; - } elsif ($state != 0) { - print OUT $_; - } -} - -close OUT; diff --git a/tools/docs/check-variable-fonts.py b/tools/docs/check-variable-fonts.py new file mode 100755 index 000000000000..c0997d6861dc --- /dev/null +++ b/tools/docs/check-variable-fonts.py @@ -0,0 +1,33 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-2.0-only +# Copyright (C) Akira Yokosawa, 2024 +# +# Ported to Python by (c) Mauro Carvalho Chehab, 2025 +# +# pylint: disable=C0103 + +""" +Detect problematic Noto CJK variable fonts. + +or more details, see lib/latex_fonts.py. +""" + +import argparse +import sys + +from lib.latex_fonts import LatexFontChecker + +checker = LatexFontChecker() + +parser=argparse.ArgumentParser(description=checker.description(), + formatter_class=argparse.RawTextHelpFormatter) +parser.add_argument("--deny-vf", + help="XDG_CONFIG_HOME dir containing fontconfig/fonts.conf file") + +args=parser.parse_args() + +msg = LatexFontChecker(args.deny_vf).check() +if msg: + print(msg) + +sys.exit(1) diff --git a/tools/docs/lib/latex_fonts.py b/tools/docs/lib/latex_fonts.py new file mode 100755 index 000000000000..29317f8006ea --- /dev/null +++ b/tools/docs/lib/latex_fonts.py @@ -0,0 +1,167 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-2.0-only +# Copyright (C) Akira Yokosawa, 2024 +# +# Ported to Python by (c) Mauro Carvalho Chehab, 2025 + +""" +Detect problematic Noto CJK variable fonts. + +For "make pdfdocs", reports of build errors of translations.pdf started +arriving early 2024 [1, 2]. It turned out that Fedora and openSUSE +tumbleweed have started deploying variable-font [3] format of "Noto CJK" +fonts [4, 5]. For PDF, a LaTeX package named xeCJK is used for CJK +(Chinese, Japanese, Korean) pages. xeCJK requires XeLaTeX/XeTeX, which +does not (and likely never will) understand variable fonts for historical +reasons. + +The build error happens even when both of variable- and non-variable-format +fonts are found on the build system. To make matters worse, Fedora enlists +variable "Noto CJK" fonts in the requirements of langpacks-ja, -ko, -zh_CN, +-zh_TW, etc. Hence developers who have interest in CJK pages are more +likely to encounter the build errors. + +This script is invoked from the error path of "make pdfdocs" and emits +suggestions if variable-font files of "Noto CJK" fonts are in the list of +fonts accessible from XeTeX. + +References: +[1]: https://lore.kernel.org/r/8734tqsrt7.fsf@meer.lwn.net/ +[2]: https://lore.kernel.org/r/1708585803.600323099@f111.i.mail.ru/ +[3]: https://en.wikipedia.org/wiki/Variable_font +[4]: https://fedoraproject.org/wiki/Changes/Noto_CJK_Variable_Fonts +[5]: https://build.opensuse.org/request/show/1157217 + +#=========================================================================== +Workarounds for building translations.pdf +#=========================================================================== + +* Denylist "variable font" Noto CJK fonts. + - Create $HOME/deny-vf/fontconfig/fonts.conf from template below, with + tweaks if necessary. Remove leading "". + - Path of fontconfig/fonts.conf can be overridden by setting an env + variable FONTS_CONF_DENY_VF. + + * Template: +----------------------------------------------------------------- + + + + + + + + /usr/share/fonts/google-noto-*-cjk-vf-fonts + + /usr/share/fonts/truetype/Noto*CJK*-VF.otf + + + +----------------------------------------------------------------- + + The denylisting is activated for "make pdfdocs". + +* For skipping CJK pages in PDF + - Uninstall texlive-xecjk. + Denylisting is not needed in this case. + +* For printing CJK pages in PDF + - Need non-variable "Noto CJK" fonts. + * Fedora + - google-noto-sans-cjk-fonts + - google-noto-serif-cjk-fonts + * openSUSE tumbleweed + - Non-variable "Noto CJK" fonts are not available as distro packages + as of April, 2024. Fetch a set of font files from upstream Noto + CJK Font released at: + https://github.com/notofonts/noto-cjk/tree/main/Sans#super-otc + and at: + https://github.com/notofonts/noto-cjk/tree/main/Serif#super-otc + , then uncompress and deploy them. + - Remember to update fontconfig cache by running fc-cache. + +!!! Caution !!! + Uninstalling "variable font" packages can be dangerous. + They might be depended upon by other packages important for your work. + Denylisting should be less invasive, as it is effective only while + XeLaTeX runs in "make pdfdocs". +""" + +import os +import re +import subprocess +import textwrap +import sys + +class LatexFontChecker: + """ + Detect problems with CJK variable fonts that affect PDF builds for + translations. + """ + + def __init__(self, deny_vf=None): + if not deny_vf: + deny_vf = os.environ.get('FONTS_CONF_DENY_VF', "~/deny-vf") + + self.environ = os.environ.copy() + self.environ['XDG_CONFIG_HOME'] = os.path.expanduser(deny_vf) + + self.re_cjk = re.compile(r"([^:]+):\s*Noto\s+(Sans|Sans Mono|Serif) CJK") + + def description(self): + return __doc__ + + def get_noto_cjk_vf_fonts(self): + """Get Noto CJK fonts""" + + cjk_fonts = set() + cmd = ["fc-list", ":", "file", "family", "variable"] + try: + result = subprocess.run(cmd,stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + universal_newlines=True, + env=self.environ, + check=True) + + except subprocess.CalledProcessError as exc: + sys.exit(f"Error running fc-list: {repr(exc)}") + + for line in result.stdout.splitlines(): + if 'variable=True' not in line: + continue + + match = self.re_cjk.search(line) + if match: + cjk_fonts.add(match.group(1)) + + return sorted(cjk_fonts) + + def check(self): + """Check for problems with CJK fonts""" + + fonts = textwrap.indent("\n".join(self.get_noto_cjk_vf_fonts()), " ") + if not fonts: + return None + + rel_file = os.path.relpath(__file__, os.getcwd()) + + msg = "=" * 77 + "\n" + msg += 'XeTeX is confused by "variable font" files listed below:\n' + msg += fonts + "\n" + msg += textwrap.dedent(f""" + For CJK pages in PDF, they need to be hidden from XeTeX by denylisting. + Or, CJK pages can be skipped by uninstalling texlive-xecjk. + + For more info on denylisting, other options, and variable font, run: + + tools/docs/check-variable-fonts.py -h + """) + msg += "=" * 77 + + return msg diff --git a/tools/docs/lib/python_version.py b/tools/docs/lib/python_version.py new file mode 100644 index 000000000000..4fde1b882164 --- /dev/null +++ b/tools/docs/lib/python_version.py @@ -0,0 +1,178 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-2.0-or-later +# Copyright (c) 2017-2025 Mauro Carvalho Chehab + +""" +Handle Python version check logic. + +Not all Python versions are supported by scripts. Yet, on some cases, +like during documentation build, a newer version of python could be +available. + +This class allows checking if the minimal requirements are followed. + +Better than that, PythonVersion.check_python() not only checks the minimal +requirements, but it automatically switches to a the newest available +Python version if present. + +""" + +import os +import re +import subprocess +import shlex +import sys + +from glob import glob +from textwrap import indent + +class PythonVersion: + """ + Ancillary methods that checks for missing dependencies for different + types of types, like binaries, python modules, rpm deps, etc. + """ + + def __init__(self, version): + """Ïnitialize self.version tuple from a version string""" + self.version = self.parse_version(version) + + @staticmethod + def parse_version(version): + """Convert a major.minor.patch version into a tuple""" + return tuple(int(x) for x in version.split(".")) + + @staticmethod + def ver_str(version): + """Returns a version tuple as major.minor.patch""" + return ".".join([str(x) for x in version]) + + @staticmethod + def cmd_print(cmd, max_len=80): + cmd_line = [] + + for w in cmd: + w = shlex.quote(w) + + if cmd_line: + if not max_len or len(cmd_line[-1]) + len(w) < max_len: + cmd_line[-1] += " " + w + continue + else: + cmd_line[-1] += " \\" + cmd_line.append(w) + else: + cmd_line.append(w) + + return "\n ".join(cmd_line) + + def __str__(self): + """Returns a version tuple as major.minor.patch from self.version""" + return self.ver_str(self.version) + + @staticmethod + def get_python_version(cmd): + """ + Get python version from a Python binary. As we need to detect if + are out there newer python binaries, we can't rely on sys.release here. + """ + + kwargs = {} + if sys.version_info < (3, 7): + kwargs['universal_newlines'] = True + else: + kwargs['text'] = True + + result = subprocess.run([cmd, "--version"], + stdout = subprocess.PIPE, + stderr = subprocess.PIPE, + **kwargs, check=False) + + version = result.stdout.strip() + + match = re.search(r"(\d+\.\d+\.\d+)", version) + if match: + return PythonVersion.parse_version(match.group(1)) + + print(f"Can't parse version {version}") + return (0, 0, 0) + + @staticmethod + def find_python(min_version): + """ + Detect if are out there any python 3.xy version newer than the + current one. + + Note: this routine is limited to up to 2 digits for python3. We + may need to update it one day, hopefully on a distant future. + """ + patterns = [ + "python3.[0-9][0-9]", + "python3.[0-9]", + ] + + python_cmd = [] + + # Seek for a python binary newer than min_version + for path in os.getenv("PATH", "").split(":"): + for pattern in patterns: + for cmd in glob(os.path.join(path, pattern)): + if os.path.isfile(cmd) and os.access(cmd, os.X_OK): + version = PythonVersion.get_python_version(cmd) + if version >= min_version: + python_cmd.append((version, cmd)) + + return sorted(python_cmd, reverse=True) + + @staticmethod + def check_python(min_version, show_alternatives=False, bail_out=False, + success_on_error=False): + """ + Check if the current python binary satisfies our minimal requirement + for Sphinx build. If not, re-run with a newer version if found. + """ + cur_ver = sys.version_info[:3] + if cur_ver >= min_version: + ver = PythonVersion.ver_str(cur_ver) + return + + python_ver = PythonVersion.ver_str(cur_ver) + + available_versions = PythonVersion.find_python(min_version) + if not available_versions: + print(f"ERROR: Python version {python_ver} is not spported anymore\n") + print(" Can't find a new version. This script may fail") + return + + script_path = os.path.abspath(sys.argv[0]) + + # Check possible alternatives + if available_versions: + new_python_cmd = available_versions[0][1] + else: + new_python_cmd = None + + if show_alternatives and available_versions: + print("You could run, instead:") + for _, cmd in available_versions: + args = [cmd, script_path] + sys.argv[1:] + + cmd_str = indent(PythonVersion.cmd_print(args), " ") + print(f"{cmd_str}\n") + + if bail_out: + msg = f"Python {python_ver} not supported. Bailing out" + if success_on_error: + print(msg, file=sys.stderr) + sys.exit(0) + else: + sys.exit(msg) + + print(f"Python {python_ver} not supported. Changing to {new_python_cmd}") + + # Restart script using the newer version + args = [new_python_cmd, script_path] + sys.argv[1:] + + try: + os.execv(new_python_cmd, args) + except OSError as e: + sys.exit(f"Failed to restart with {new_python_cmd}: {e}") diff --git a/scripts/sphinx-build-wrapper b/tools/docs/sphinx-build-wrapper similarity index 50% rename from scripts/sphinx-build-wrapper rename to tools/docs/sphinx-build-wrapper index abe8c26ae137..3e6d166d4102 100755 --- a/scripts/sphinx-build-wrapper +++ b/tools/docs/sphinx-build-wrapper @@ -56,68 +56,40 @@ import sys from concurrent import futures from glob import glob -LIB_DIR = "lib" +from lib.python_version import PythonVersion +from lib.latex_fonts import LatexFontChecker + +LIB_DIR = "../../scripts/lib" SRC_DIR = os.path.dirname(os.path.realpath(__file__)) sys.path.insert(0, os.path.join(SRC_DIR, LIB_DIR)) -from jobserver import JobserverExec # pylint: disable=C0413 +from jobserver import JobserverExec # pylint: disable=C0413,C0411,E0401 - -def parse_version(version): - """Convert a major.minor.patch version into a tuple""" - return tuple(int(x) for x in version.split(".")) - -def ver_str(version): - """Returns a version tuple as major.minor.patch""" - - return ".".join([str(x) for x in version]) - -# Minimal supported Python version needed by Sphinx and its extensions -MIN_PYTHON_VERSION = parse_version("3.7") - -# Default value for --venv parameter +# +# Some constants +# VENV_DEFAULT = "sphinx_latest" +MIN_PYTHON_VERSION = PythonVersion("3.7").version +PAPER = ["", "a4", "letter"] -# List of make targets and its corresponding builder and output directory TARGETS = { - "cleandocs": { - "builder": "clean", - }, - "htmldocs": { - "builder": "html", - }, - "epubdocs": { - "builder": "epub", - "out_dir": "epub", - }, - "texinfodocs": { - "builder": "texinfo", - "out_dir": "texinfo", - }, - "infodocs": { - "builder": "texinfo", - "out_dir": "texinfo", - }, - "latexdocs": { - "builder": "latex", - "out_dir": "latex", - }, - "pdfdocs": { - "builder": "latex", - "out_dir": "latex", - }, - "xmldocs": { - "builder": "xml", - "out_dir": "xml", - }, - "linkcheckdocs": { - "builder": "linkcheck" - }, + "cleandocs": { "builder": "clean" }, + "linkcheckdocs": { "builder": "linkcheck" }, + "htmldocs": { "builder": "html" }, + "epubdocs": { "builder": "epub", "out_dir": "epub" }, + "texinfodocs": { "builder": "texinfo", "out_dir": "texinfo" }, + "infodocs": { "builder": "texinfo", "out_dir": "texinfo" }, + "mandocs": { "builder": "man", "out_dir": "man" }, + "latexdocs": { "builder": "latex", "out_dir": "latex" }, + "pdfdocs": { "builder": "latex", "out_dir": "latex" }, + "xmldocs": { "builder": "xml", "out_dir": "xml" }, } -# Paper sizes. An empty value will pick the default -PAPER = ["", "a4", "letter"] + +# +# SphinxBuilder class +# class SphinxBuilder: """ @@ -125,15 +97,7 @@ class SphinxBuilder: with the Kernel. """ - def is_rust_enabled(self): - """Check if rust is enabled at .config""" - config_path = os.path.join(self.srctree, ".config") - if os.path.isfile(config_path): - with open(config_path, "r", encoding="utf-8") as f: - return "CONFIG_RUST=y" in f.read() - return False - - def get_path(self, path, abs_path=False): + def get_path(self, path, use_cwd=False, abs_path=False): """ Ancillary routine to handle patches the right way, as shell does. @@ -143,23 +107,97 @@ class SphinxBuilder: path = os.path.expanduser(path) if not path.startswith("/"): - path = os.path.join(self.srctree, path) + if use_cwd: + base = os.getcwd() + else: + base = self.srctree + + path = os.path.join(base, path) if abs_path: return os.path.abspath(path) return path - def __init__(self, venv=None, verbose=False, n_jobs=None, interactive=None): + def get_sphinx_extra_opts(self, n_jobs): + """ + Get the number of jobs to be used for docs build passed via command + line and desired sphinx verbosity. + + The number of jobs can be on different places: + + 1) It can be passed via "-j" argument; + 2) The SPHINXOPTS="-j8" env var may have "-j"; + 3) if called via GNU make, -j specifies the desired number of jobs. + with GNU makefile, this number is available via POSIX jobserver; + 4) if none of the above is available, it should default to "-jauto", + and let sphinx decide the best value. + """ + + # + # SPHINXOPTS env var, if used, contains extra arguments to be used + # by sphinx-build time. Among them, it may contain sphinx verbosity + # and desired number of parallel jobs. + # + parser = argparse.ArgumentParser() + parser.add_argument('-j', '--jobs', type=int) + parser.add_argument('-q', '--quiet', action='store_true') + + # + # Other sphinx-build arguments go as-is, so place them + # at self.sphinxopts, using shell parser + # + sphinxopts = shlex.split(os.environ.get("SPHINXOPTS", "")) + + # + # Build a list of sphinx args, honoring verbosity here if specified + # + + verbose = self.verbose + sphinx_args, self.sphinxopts = parser.parse_known_args(sphinxopts) + if sphinx_args.quiet is True: + verbose = False + + # + # If the user explicitly sets "-j" at command line, use it. + # Otherwise, pick it from SPHINXOPTS args + # + if n_jobs: + self.n_jobs = n_jobs + elif sphinx_args.jobs: + self.n_jobs = sphinx_args.jobs + else: + self.n_jobs = None + + if not verbose: + self.sphinxopts += ["-q"] + + def __init__(self, builddir, venv=None, verbose=False, n_jobs=None, + interactive=None): """Initialize internal variables""" self.venv = venv self.verbose = None + # # Normal variables passed from Kernel's makefile + # self.kernelversion = os.environ.get("KERNELVERSION", "unknown") self.kernelrelease = os.environ.get("KERNELRELEASE", "unknown") self.pdflatex = os.environ.get("PDFLATEX", "xelatex") + # + # Kernel main Makefile defines a PYTHON3 variable whose default is + # "python3". When set to a different value, it allows running a + # diferent version than the default official python3 package. + # Several distros package python3xx-sphinx packages with newer + # versions of Python and sphinx-build. + # + # Honor such variable different than default + # + self.python = os.environ.get("PYTHON3") + if self.python == "python3": + self.python = None + if not interactive: self.latexopts = os.environ.get("LATEXOPTS", "-interaction=batchmode -no-shell-escape") else: @@ -168,112 +206,107 @@ class SphinxBuilder: if not verbose: verbose = bool(os.environ.get("KBUILD_VERBOSE", "") != "") - # Handle SPHINXOPTS evironment - sphinxopts = shlex.split(os.environ.get("SPHINXOPTS", "")) - - # As we handle number of jobs and quiet in separate, we need to pick - # it the same way as sphinx-build would pick, so let's use argparse - # do to the right argument expansion - parser = argparse.ArgumentParser() - parser.add_argument('-j', '--jobs', type=int) - parser.add_argument('-q', '--quiet', type=int) - - # Other sphinx-build arguments go as-is, so place them - # at self.sphinxopts - sphinx_args, self.sphinxopts = parser.parse_known_args(sphinxopts) - if sphinx_args.quiet == True: - self.verbose = False - - if sphinx_args.jobs: - self.n_jobs = sphinx_args.jobs - - # Command line arguments was passed, override SPHINXOPTS if verbose is not None: self.verbose = verbose - self.n_jobs = n_jobs - + # # Source tree directory. This needs to be at os.environ, as - # Sphinx extensions and media uAPI makefile needs it + # Sphinx extensions use it + # self.srctree = os.environ.get("srctree") if not self.srctree: self.srctree = "." os.environ["srctree"] = self.srctree + # # Now that we can expand srctree, get other directories as well + # self.sphinxbuild = os.environ.get("SPHINXBUILD", "sphinx-build") self.kerneldoc = self.get_path(os.environ.get("KERNELDOC", "scripts/kernel-doc.py")) - self.obj = os.environ.get("obj", "Documentation") - self.builddir = self.get_path(os.path.join(self.obj, "output"), - abs_path=True) - - # Media uAPI needs it - os.environ["BUILDDIR"] = self.builddir - - # Detect if rust is enabled - self.config_rust = self.is_rust_enabled() + self.builddir = self.get_path(builddir, use_cwd=True, abs_path=True) + # # Get directory locations for LaTeX build toolchain + # self.pdflatex_cmd = shutil.which(self.pdflatex) self.latexmk_cmd = shutil.which("latexmk") self.env = os.environ.copy() - # If venv parameter is specified, run Sphinx from venv + self.get_sphinx_extra_opts(n_jobs) + + # + # If venv command line argument is specified, run Sphinx from venv + # if venv: bin_dir = os.path.join(venv, "bin") - if os.path.isfile(os.path.join(bin_dir, "activate")): - # "activate" virtual env - self.env["PATH"] = bin_dir + ":" + self.env["PATH"] - self.env["VIRTUAL_ENV"] = venv - if "PYTHONHOME" in self.env: - del self.env["PYTHONHOME"] - print(f"Setting venv to {venv}") - else: + if not os.path.isfile(os.path.join(bin_dir, "activate")): sys.exit(f"Venv {venv} not found.") + # "activate" virtual env + self.env["PATH"] = bin_dir + ":" + self.env["PATH"] + self.env["VIRTUAL_ENV"] = venv + if "PYTHONHOME" in self.env: + del self.env["PYTHONHOME"] + print(f"Setting venv to {venv}") + def run_sphinx(self, sphinx_build, build_args, *args, **pwargs): """ - Executes sphinx-build using current python3 command and setting - -j parameter if possible to run the build in parallel. + Executes sphinx-build using current python3 command. + + When calling via GNU make, POSIX jobserver is used to tell how + many jobs are still available from a job pool. claim all remaining + jobs, as we don't want sphinx-build to run in parallel with other + jobs. + + Despite that, the user may actually force a different value than + the number of available jobs via command line. + + The "with" logic here is used to ensure that the claimed jobs will + be freed once subprocess finishes """ with JobserverExec() as jobserver: if jobserver.claim: + # + # when GNU make is used, claim available jobs from jobserver + # n_jobs = str(jobserver.claim) else: - n_jobs = "auto" # Supported since Sphinx 1.7 + # + # Otherwise, let sphinx decide by default + # + n_jobs = "auto" - cmd = [] - - if self.venv: - cmd.append("python") - else: - cmd.append(sys.executable) - - cmd.append(sphinx_build) - - # if present, SPHINXOPTS or command line --jobs overrides default + # + # If explicitly requested via command line, override default + # if self.n_jobs: n_jobs = str(self.n_jobs) - if n_jobs: - cmd += [f"-j{n_jobs}"] - - if not self.verbose: - cmd.append("-q") + # + # We can't simply call python3 sphinx-build, as OpenSUSE + # Tumbleweed uses an ELF binary file (/usr/bin/alts) to switch + # between different versions of sphinx-build. So, only call it + # prepending "python3.xx" when PYTHON3 variable is not default. + # + if self.python: + cmd = [self.python] + else: + cmd = [] + cmd += [sphinx_build] + cmd += [f"-j{n_jobs}"] cmd += self.sphinxopts - cmd += build_args if self.verbose: print(" ".join(cmd)) - rc = subprocess.call(cmd, *args, **pwargs) + return subprocess.call(cmd, *args, **pwargs) - def handle_html(self, css, output_dir): + def handle_html(self, css, output_dir, rustdoc): """ Extra steps for HTML and epub output. @@ -281,32 +314,43 @@ class SphinxBuilder: copied to the output _static directory """ - if not css: - return + if css: + css = os.path.expanduser(css) + if not css.startswith("/"): + css = os.path.join(self.srctree, css) - css = os.path.expanduser(css) - if not css.startswith("/"): - css = os.path.join(self.srctree, css) + static_dir = os.path.join(output_dir, "_static") + os.makedirs(static_dir, exist_ok=True) - static_dir = os.path.join(output_dir, "_static") - os.makedirs(static_dir, exist_ok=True) + try: + shutil.copy2(css, static_dir) + except (OSError, IOError) as e: + print(f"Warning: Failed to copy CSS: {e}", file=sys.stderr) - try: - shutil.copy2(css, static_dir) - except (OSError, IOError) as e: - print(f"Warning: Failed to copy CSS: {e}", file=sys.stderr) + if rustdoc: + if "MAKE" in self.env: + cmd = [self.env["MAKE"]] + else: + cmd = ["make", "LLVM=1"] + + cmd += [ "rustdoc"] + if self.verbose: + print(" ".join(cmd)) + + try: + subprocess.run(cmd, check=True) + except subprocess.CalledProcessError as e: + print(f"Ignored errors when building rustdoc: {e}. Is RUST enabled?", + file=sys.stderr) def build_pdf_file(self, latex_cmd, from_dir, path): """Builds a single pdf file using latex_cmd""" try: subprocess.run(latex_cmd + [path], - cwd=from_dir, check=True) + cwd=from_dir, check=True, env=self.env) return True except subprocess.CalledProcessError: - # LaTeX PDF error code is almost useless: it returns - # error codes even when build succeeds but has warnings. - # So, we'll ignore the results return False def pdf_parallel_build(self, tex_suffix, latex_cmd, tex_files, n_jobs): @@ -316,7 +360,14 @@ class SphinxBuilder: max_len = 0 has_tex = False - # Process files in parallel + # + # LaTeX PDF error code is almost useless for us: + # any warning makes it non-zero. For kernel doc builds it always return + # non-zero even when build succeeds. So, let's do the best next thing: + # Ignore build errors. At the end, check if all PDF files were built, + # printing a summary with the built ones and returning 0 if all of + # them were actually built. + # with futures.ThreadPoolExecutor(max_workers=n_jobs) as executor: jobs = {} @@ -327,46 +378,51 @@ class SphinxBuilder: continue name = name[:-len(tex_suffix)] - - max_len = max(max_len, len(name)) - has_tex = True future = executor.submit(self.build_pdf_file, latex_cmd, from_dir, entry.path) - jobs[future] = (from_dir, name, entry.path) + jobs[future] = (from_dir, pdf_dir, name) for future in futures.as_completed(jobs): - from_dir, name, path = jobs[future] + from_dir, pdf_dir, name = jobs[future] pdf_name = name + ".pdf" pdf_from = os.path.join(from_dir, pdf_name) + pdf_to = os.path.join(pdf_dir, pdf_name) + out_name = os.path.relpath(pdf_to, self.builddir) + max_len = max(max_len, len(out_name)) try: success = future.result() if success and os.path.exists(pdf_from): - pdf_to = os.path.join(pdf_dir, pdf_name) - os.rename(pdf_from, pdf_to) - builds[name] = os.path.relpath(pdf_to, self.builddir) + + # + # if verbose, get the name of built PDF file + # + if self.verbose: + builds[out_name] = "SUCCESS" else: - builds[name] = "FAILED" + builds[out_name] = "FAILED" build_failed = True - except Exception as e: - builds[name] = f"FAILED ({str(e)})" + except futures.Error as e: + builds[out_name] = f"FAILED ({repr(e)})" build_failed = True + # # Handle case where no .tex files were found + # if not has_tex: - name = "Sphinx LaTeX builder" - max_len = max(max_len, len(name)) - builds[name] = "FAILED (no .tex file was generated)" + out_name = "LaTeX files" + max_len = max(max_len, len(out_name)) + builds[out_name] = "FAILED: no .tex files were generated" build_failed = True return builds, build_failed, max_len - def handle_pdf(self, output_dirs): + def handle_pdf(self, output_dirs, deny_vf): """ Extra steps for PDF output. @@ -377,9 +433,22 @@ class SphinxBuilder: builds = {} max_len = 0 tex_suffix = ".tex" - - # Get all tex files that will be used for PDF build tex_files = [] + + # + # Since early 2024, Fedora and openSUSE tumbleweed have started + # deploying variable-font format of "Noto CJK", causing LaTeX + # to break with CJK. Work around it, by denying the variable font + # usage during xelatex build by passing the location of a config + # file with a deny list. + # + # See tools/docs/lib/latex_fonts.py for more details. + # + if deny_vf: + deny_vf = os.path.expanduser(deny_vf) + if os.path.isdir(deny_vf): + self.env["XDG_CONFIG_HOME"] = deny_vf + for from_dir in output_dirs: pdf_dir = os.path.join(from_dir, "../pdf") os.makedirs(pdf_dir, exist_ok=True) @@ -397,10 +466,12 @@ class SphinxBuilder: if entry.name.endswith(tex_suffix): tex_files.append((from_dir, pdf_dir, entry)) + # # When using make, this won't be used, as the number of jobs comes # from POSIX jobserver. So, this covers the case where build comes # from command line. On such case, serialize by default, except if # the user explicitly sets the number of jobs. + # n_jobs = 1 # n_jobs is either an integer or "auto". Only use it if it is a number @@ -410,13 +481,17 @@ class SphinxBuilder: except ValueError: pass + # # When using make, jobserver.claim is the number of jobs that were # used with "-j" and that aren't used by other make targets + # with JobserverExec() as jobserver: n_jobs = 1 + # # Handle the case when a parameter is passed via command line, # using it as default, if jobserver doesn't claim anything + # if self.n_jobs: try: n_jobs = int(self.n_jobs) @@ -426,28 +501,42 @@ class SphinxBuilder: if jobserver.claim: n_jobs = jobserver.claim - # Build files in parallel builds, build_failed, max_len = self.pdf_parallel_build(tex_suffix, latex_cmd, tex_files, n_jobs) - msg = "Summary" - msg += "\n" + "=" * len(msg) - print() - print(msg) + # + # In verbose mode, print a summary with the build results per file. + # Otherwise, print a single line with all failures, if any. + # On both cases, return code 1 indicates build failures, + # + if self.verbose: + msg = "Summary" + msg += "\n" + "=" * len(msg) + print() + print(msg) - for pdf_name, pdf_file in builds.items(): - print(f"{pdf_name:<{max_len}}: {pdf_file}") + for pdf_name, pdf_file in builds.items(): + print(f"{pdf_name:<{max_len}}: {pdf_file}") - print() + print() + if build_failed: + msg = LatexFontChecker().check() + if msg: + print(msg) - # return an error if a PDF file is missing + sys.exit("Error: not all PDF files were created.") - if build_failed: - sys.exit(f"PDF build failed: not all PDF files were created.") - else: - print("All PDF files were built.") + elif build_failed: + n_failures = len(builds) + failures = ", ".join(builds.keys()) + + msg = LatexFontChecker().check() + if msg: + print(msg) + + sys.exit(f"Error: Can't build {n_failures} PDF file(s): {failures}") def handle_info(self, output_dirs): """ @@ -463,12 +552,78 @@ class SphinxBuilder: except subprocess.CalledProcessError as e: sys.exit(f"Error generating info docs: {e}") - def cleandocs(self, builder): + def handle_man(self, kerneldoc, docs_dir, src_dir, output_dir): + """ + Create man pages from kernel-doc output + """ + re_kernel_doc = re.compile(r"^\.\.\s+kernel-doc::\s*(\S+)") + re_man = re.compile(r'^\.TH "[^"]*" (\d+) "([^"]*)"') + + if docs_dir == src_dir: + # + # Pick the entire set of kernel-doc markups from the entire tree + # + kdoc_files = set([self.srctree]) + else: + kdoc_files = set() + + for fname in glob(os.path.join(src_dir, "**"), recursive=True): + if os.path.isfile(fname) and fname.endswith(".rst"): + with open(fname, "r", encoding="utf-8") as in_fp: + data = in_fp.read() + + for line in data.split("\n"): + match = re_kernel_doc.match(line) + if match: + if os.path.isfile(match.group(1)): + kdoc_files.add(match.group(1)) + + if not kdoc_files: + sys.exit(f"Directory {src_dir} doesn't contain kernel-doc tags") + + cmd = [ kerneldoc, "-m" ] + sorted(kdoc_files) + try: + if self.verbose: + print(" ".join(cmd)) + + result = subprocess.run(cmd, stdout=subprocess.PIPE, text= True) + + if result.returncode: + print(f"Warning: kernel-doc returned {result.returncode} warnings") + + except (OSError, ValueError, subprocess.SubprocessError) as e: + sys.exit(f"Failed to create man pages for {src_dir}: {repr(e)}") + + fp = None + try: + for line in result.stdout.split("\n"): + match = re_man.match(line) + if not match: + if fp: + fp.write(line + '\n') + continue + + if fp: + fp.close() + + fname = f"{output_dir}/{match.group(2)}.{match.group(1)}" + + if self.verbose: + print(f"Creating {fname}") + fp = open(fname, "w", encoding="utf-8") + fp.write(line + '\n') + finally: + if fp: + fp.close() + + def cleandocs(self, builder): # pylint: disable=W0613 + """Remove documentation output directory""" shutil.rmtree(self.builddir, ignore_errors=True) - def build(self, target, sphinxdirs=None, conf="conf.py", - theme=None, css=None, paper=None): + def build(self, target, sphinxdirs=None, + theme=None, css=None, paper=None, deny_vf=None, rustdoc=False, + skip_sphinx=False): """ Build documentation using Sphinx. This is the core function of this module. It prepares all arguments required by sphinx-build. @@ -477,32 +632,38 @@ class SphinxBuilder: builder = TARGETS[target]["builder"] out_dir = TARGETS[target].get("out_dir", "") + # # Cleandocs doesn't require sphinx-build + # if target == "cleandocs": self.cleandocs(builder) return - # Other targets require sphinx-build - sphinxbuild = shutil.which(self.sphinxbuild, path=self.env["PATH"]) - if not sphinxbuild: - sys.exit(f"Error: {self.sphinxbuild} not found in PATH.\n") + if theme: + os.environ["DOCS_THEME"] = theme - if builder == "latex": + # + # Other targets require sphinx-build, so check if it exists + # + if not skip_sphinx: + sphinxbuild = shutil.which(self.sphinxbuild, path=self.env["PATH"]) + if not sphinxbuild and target != "mandocs": + sys.exit(f"Error: {self.sphinxbuild} not found in PATH.\n") + + if target == "pdfdocs": if not self.pdflatex_cmd and not self.latexmk_cmd: sys.exit("Error: pdflatex or latexmk required for PDF generation") docs_dir = os.path.abspath(os.path.join(self.srctree, "Documentation")) - # Prepare base arguments for Sphinx build + # + # Fill in base arguments for Sphinx build + # kerneldoc = self.kerneldoc if kerneldoc.startswith(self.srctree): kerneldoc = os.path.relpath(kerneldoc, self.srctree) - # Prepare common Sphinx options - args = [ - "-b", builder, - "-c", docs_dir, - ] + args = [ "-b", builder, "-c", docs_dir ] if builder == "latex": if not paper: @@ -510,41 +671,48 @@ class SphinxBuilder: args.extend(["-D", f"latex_elements.papersize={paper}paper"]) - if self.config_rust: + if rustdoc: args.extend(["-t", "rustdoc"]) - if conf: - self.env["SPHINX_CONF"] = self.get_path(conf, abs_path=True) - if not sphinxdirs: sphinxdirs = os.environ.get("SPHINXDIRS", ".") + # # The sphinx-build tool has a bug: internally, it tries to set # locale with locale.setlocale(locale.LC_ALL, ''). This causes a # crash if language is not set. Detect and fix it. + # try: locale.setlocale(locale.LC_ALL, '') - except Exception: + except locale.Error: self.env["LC_ALL"] = "C" - self.env["LANG"] = "C" + # # sphinxdirs can be a list or a whitespace-separated string + # sphinxdirs_list = [] for sphinxdir in sphinxdirs: if isinstance(sphinxdir, list): sphinxdirs_list += sphinxdir else: - for name in sphinxdir.split(" "): - sphinxdirs_list.append(name) + sphinxdirs_list += sphinxdir.split() - # Build each directory + # + # Step 1: Build each directory in separate. + # + # This is not the best way of handling it, as cross-references between + # them will be broken, but this is what we've been doing since + # the beginning. + # output_dirs = [] for sphinxdir in sphinxdirs_list: src_dir = os.path.join(docs_dir, sphinxdir) doctree_dir = os.path.join(self.builddir, ".doctrees") output_dir = os.path.join(self.builddir, sphinxdir, out_dir) + # # Make directory names canonical + # src_dir = os.path.normpath(src_dir) doctree_dir = os.path.normpath(doctree_dir) output_dir = os.path.normpath(output_dir) @@ -564,93 +732,34 @@ class SphinxBuilder: output_dir, ] - # Execute sphinx-build - try: - self.run_sphinx(sphinxbuild, build_args, env=self.env) - except Exception as e: - sys.exit(f"Build failed: {e}") + if target == "mandocs": + self.handle_man(kerneldoc, docs_dir, src_dir, output_dir) + elif not skip_sphinx: + try: + result = self.run_sphinx(sphinxbuild, build_args, + env=self.env) - # Ensure that html/epub will have needed static files + if result: + sys.exit(f"Build failed: return code: {result}") + + except (OSError, ValueError, subprocess.SubprocessError) as e: + sys.exit(f"Build failed: {repr(e)}") + + # + # Ensure that each html/epub output will have needed static files + # if target in ["htmldocs", "epubdocs"]: - self.handle_html(css, output_dir) + self.handle_html(css, output_dir, rustdoc) - # PDF and Info require a second build step + # + # Step 2: Some targets (PDF and info) require an extra step once + # sphinx-build finishes + # if target == "pdfdocs": - self.handle_pdf(output_dirs) + self.handle_pdf(output_dirs, deny_vf) elif target == "infodocs": self.handle_info(output_dirs) - @staticmethod - def get_python_version(cmd): - """ - Get python version from a Python binary. As we need to detect if - are out there newer python binaries, we can't rely on sys.release here. - """ - - result = subprocess.run([cmd, "--version"], check=True, - stdout=subprocess.PIPE, stderr=subprocess.PIPE, - universal_newlines=True) - version = result.stdout.strip() - - match = re.search(r"(\d+\.\d+\.\d+)", version) - if match: - return parse_version(match.group(1)) - - print(f"Can't parse version {version}") - return (0, 0, 0) - - @staticmethod - def find_python(): - """ - Detect if are out there any python 3.xy version newer than the - current one. - - Note: this routine is limited to up to 2 digits for python3. We - may need to update it one day, hopefully on a distant future. - """ - patterns = [ - "python3.[0-9]", - "python3.[0-9][0-9]", - ] - - # Seek for a python binary newer than MIN_PYTHON_VERSION - for path in os.getenv("PATH", "").split(":"): - for pattern in patterns: - for cmd in glob(os.path.join(path, pattern)): - if os.path.isfile(cmd) and os.access(cmd, os.X_OK): - version = SphinxBuilder.get_python_version(cmd) - if version >= MIN_PYTHON_VERSION: - return cmd - - return None - - @staticmethod - def check_python(): - """ - Check if the current python binary satisfies our minimal requirement - for Sphinx build. If not, re-run with a newer version if found. - """ - cur_ver = sys.version_info[:3] - if cur_ver >= MIN_PYTHON_VERSION: - return - - python_ver = ver_str(cur_ver) - - new_python_cmd = SphinxBuilder.find_python() - if not new_python_cmd: - sys.exit(f"Python version {python_ver} is not supported anymore.") - - # Restart script using the newer version - script_path = os.path.abspath(sys.argv[0]) - args = [new_python_cmd, script_path] + sys.argv[1:] - - print(f"Python {python_ver} not supported. Changing to {new_python_cmd}") - - try: - os.execv(new_python_cmd, args) - except OSError as e: - sys.exit(f"Failed to restart with {new_python_cmd}: {e}") - def jobs_type(value): """ Handle valid values for -j. Accepts Sphinx "-jauto", plus a number @@ -668,7 +777,7 @@ def jobs_type(value): raise argparse.ArgumentTypeError(f"Minimum jobs is 1, got {value}") except ValueError: - raise argparse.ArgumentTypeError(f"Must be 'auto' or positive integer, got {value}") + raise argparse.ArgumentTypeError(f"Must be 'auto' or positive integer, got {value}") # pylint: disable=W0707 def main(): """ @@ -682,7 +791,7 @@ def main(): help="Documentation target to build") parser.add_argument("--sphinxdirs", nargs="+", help="Specific directories to build") - parser.add_argument("--conf", default="conf.py", + parser.add_argument("--builddir", default="output", help="Sphinx configuration file") parser.add_argument("--theme", help="Sphinx theme to use") @@ -692,6 +801,12 @@ def main(): parser.add_argument("--paper", choices=PAPER, default=PAPER[0], help="Paper size for LaTeX/PDF output") + parser.add_argument('--deny-vf', + help="Configuration to deny variable fonts on pdf builds") + + parser.add_argument('--rustdoc', action="store_true", + help="Enable rustdoc build. Requires CONFIG_RUST") + parser.add_argument("-v", "--verbose", action='store_true', help="place build in verbose mode") @@ -701,19 +816,26 @@ def main(): parser.add_argument('-i', '--interactive', action='store_true', help="Change latex default to run in interactive mode") + parser.add_argument('-s', '--skip-sphinx-build', action='store_true', + help="Skip sphinx-build step") + parser.add_argument("-V", "--venv", nargs='?', const=f'{VENV_DEFAULT}', default=None, help=f'If used, run Sphinx from a venv dir (default dir: {VENV_DEFAULT})') args = parser.parse_args() - SphinxBuilder.check_python() + PythonVersion.check_python(MIN_PYTHON_VERSION, show_alternatives=True, + bail_out=True) - builder = SphinxBuilder(venv=args.venv, verbose=args.verbose, - n_jobs=args.jobs, interactive=args.interactive) + builder = SphinxBuilder(builddir=args.builddir, venv=args.venv, + verbose=args.verbose, n_jobs=args.jobs, + interactive=args.interactive) - builder.build(args.target, sphinxdirs=args.sphinxdirs, conf=args.conf, - theme=args.theme, css=args.css, paper=args.paper) + builder.build(args.target, sphinxdirs=args.sphinxdirs, + theme=args.theme, css=args.css, paper=args.paper, + rustdoc=args.rustdoc, deny_vf=args.deny_vf, + skip_sphinx=args.skip_sphinx_build) if __name__ == "__main__": main() diff --git a/scripts/sphinx-pre-install b/tools/docs/sphinx-pre-install similarity index 93% rename from scripts/sphinx-pre-install rename to tools/docs/sphinx-pre-install index 954ed3dc0645..698989584b6a 100755 --- a/scripts/sphinx-pre-install +++ b/tools/docs/sphinx-pre-install @@ -26,26 +26,17 @@ system pacage install is recommended. """ import argparse +import locale import os import re import subprocess import sys from glob import glob +from lib.python_version import PythonVersion -def parse_version(version): - """Convert a major.minor.patch version into a tuple""" - return tuple(int(x) for x in version.split(".")) - - -def ver_str(version): - """Returns a version tuple as major.minor.patch""" - - return ".".join([str(x) for x in version]) - - -RECOMMENDED_VERSION = parse_version("3.4.3") -MIN_PYTHON_VERSION = parse_version("3.7") +RECOMMENDED_VERSION = PythonVersion("3.4.3").version +MIN_PYTHON_VERSION = PythonVersion("3.7").version class DepManager: @@ -235,95 +226,11 @@ class AncillaryMethods: return None - @staticmethod - def get_python_version(cmd): - """ - Get python version from a Python binary. As we need to detect if - are out there newer python binaries, we can't rely on sys.release here. - """ - - result = SphinxDependencyChecker.run([cmd, "--version"], - capture_output=True, text=True) - version = result.stdout.strip() - - match = re.search(r"(\d+\.\d+\.\d+)", version) - if match: - return parse_version(match.group(1)) - - print(f"Can't parse version {version}") - return (0, 0, 0) - - @staticmethod - def find_python(): - """ - Detect if are out there any python 3.xy version newer than the - current one. - - Note: this routine is limited to up to 2 digits for python3. We - may need to update it one day, hopefully on a distant future. - """ - patterns = [ - "python3.[0-9]", - "python3.[0-9][0-9]", - ] - - # Seek for a python binary newer than MIN_PYTHON_VERSION - for path in os.getenv("PATH", "").split(":"): - for pattern in patterns: - for cmd in glob(os.path.join(path, pattern)): - if os.path.isfile(cmd) and os.access(cmd, os.X_OK): - version = SphinxDependencyChecker.get_python_version(cmd) - if version >= MIN_PYTHON_VERSION: - return cmd - - @staticmethod - def check_python(): - """ - Check if the current python binary satisfies our minimal requirement - for Sphinx build. If not, re-run with a newer version if found. - """ - cur_ver = sys.version_info[:3] - if cur_ver >= MIN_PYTHON_VERSION: - ver = ver_str(cur_ver) - print(f"Python version: {ver}") - - # This could be useful for debugging purposes - if SphinxDependencyChecker.which("docutils"): - result = SphinxDependencyChecker.run(["docutils", "--version"], - capture_output=True, text=True) - ver = result.stdout.strip() - match = re.search(r"(\d+\.\d+\.\d+)", ver) - if match: - ver = match.group(1) - - print(f"Docutils version: {ver}") - - return - - python_ver = ver_str(cur_ver) - - new_python_cmd = SphinxDependencyChecker.find_python() - if not new_python_cmd: - print(f"ERROR: Python version {python_ver} is not spported anymore\n") - print(" Can't find a new version. This script may fail") - return - - # Restart script using the newer version - script_path = os.path.abspath(sys.argv[0]) - args = [new_python_cmd, script_path] + sys.argv[1:] - - print(f"Python {python_ver} not supported. Changing to {new_python_cmd}") - - try: - os.execv(new_python_cmd, args) - except OSError as e: - sys.exit(f"Failed to restart with {new_python_cmd}: {e}") - @staticmethod def run(*args, **kwargs): """ Excecute a command, hiding its output by default. - Preserve comatibility with older Python versions. + Preserve compatibility with older Python versions. """ capture_output = kwargs.pop('capture_output', False) @@ -516,8 +423,19 @@ class MissingCheckers(AncillaryMethods): """ Gets sphinx-build version. """ + env = os.environ.copy() + + # The sphinx-build tool has a bug: internally, it tries to set + # locale with locale.setlocale(locale.LC_ALL, ''). This causes a + # crash if language is not set. Detect and fix it. try: - result = self.run([cmd, "--version"], + locale.setlocale(locale.LC_ALL, '') + except Exception: + env["LC_ALL"] = "C" + env["LANG"] = "C" + + try: + result = self.run([cmd, "--version"], env=env, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True, check=True) @@ -527,11 +445,11 @@ class MissingCheckers(AncillaryMethods): for line in result.stdout.split("\n"): match = re.match(r"^sphinx-build\s+([\d\.]+)(?:\+(?:/[\da-f]+)|b\d+)?\s*$", line) if match: - return parse_version(match.group(1)) + return PythonVersion.parse_version(match.group(1)) match = re.match(r"^Sphinx.*\s+([\d\.]+)\s*$", line) if match: - return parse_version(match.group(1)) + return PythonVersion.parse_version(match.group(1)) def check_sphinx(self, conf): """ @@ -542,7 +460,7 @@ class MissingCheckers(AncillaryMethods): for line in f: match = re.match(r"^\s*needs_sphinx\s*=\s*[\'\"]([\d\.]+)[\'\"]", line) if match: - self.min_version = parse_version(match.group(1)) + self.min_version = PythonVersion.parse_version(match.group(1)) break except IOError: sys.exit(f"Can't open {conf}") @@ -562,8 +480,8 @@ class MissingCheckers(AncillaryMethods): sys.exit(f"{sphinx} didn't return its version") if self.cur_version < self.min_version: - curver = ver_str(self.cur_version) - minver = ver_str(self.min_version) + curver = PythonVersion.ver_str(self.cur_version) + minver = PythonVersion.ver_str(self.min_version) print(f"ERROR: Sphinx version is {curver}. It should be >= {minver}") self.need_sphinx = 1 @@ -1304,7 +1222,7 @@ class SphinxDependencyChecker(MissingCheckers): else: if self.need_sphinx and ver >= self.min_version: return (f, ver) - elif parse_version(ver) > self.cur_version: + elif PythonVersion.parse_version(ver) > self.cur_version: return (f, ver) return ("", ver) @@ -1411,7 +1329,7 @@ class SphinxDependencyChecker(MissingCheckers): return if self.latest_avail_ver: - latest_avail_ver = ver_str(self.latest_avail_ver) + latest_avail_ver = PythonVersion.ver_str(self.latest_avail_ver) if not self.need_sphinx: # sphinx-build is present and its version is >= $min_version @@ -1507,7 +1425,7 @@ class SphinxDependencyChecker(MissingCheckers): else: print("Unknown OS") if self.cur_version != (0, 0, 0): - ver = ver_str(self.cur_version) + ver = PythonVersion.ver_str(self.cur_version) print(f"Sphinx version: {ver}\n") # Check the type of virtual env, depending on Python version @@ -1613,7 +1531,8 @@ def main(): checker = SphinxDependencyChecker(args) - checker.check_python() + PythonVersion.check_python(MIN_PYTHON_VERSION, + bail_out=True, success_on_error=True) checker.check_needs() # Call main if not used as module