| 1 | #!/usr/bin/env python3 |
| 2 | # SPDX-License-Identifier: GPL-2.0 |
| 3 | # Copyright (C) 2025 Mauro Carvalho Chehab <[email protected]> |
| 4 | # |
| 5 | # pylint: disable=R0902, R0912, R0913, R0914, R0915, R0917, C0103 |
| 6 | # |
| 7 | # Converted from docs Makefile and parallel-wrapper.sh, both under |
| 8 | # GPLv2, copyrighted since 2008 by the following authors: |
| 9 | # |
| 10 | # Akira Yokosawa <[email protected]> |
| 11 | # Arnd Bergmann <[email protected]> |
| 12 | # Breno Leitao <[email protected]> |
| 13 | # Carlos Bilbao <[email protected]> |
| 14 | # Dave Young <[email protected]> |
| 15 | # Donald Hunter <[email protected]> |
| 16 | # Geert Uytterhoeven <[email protected]> |
| 17 | # Jani Nikula <[email protected]> |
| 18 | # Jan Stancek <[email protected]> |
| 19 | # Jonathan Corbet <[email protected]> |
| 20 | # Joshua Clayton <[email protected]> |
| 21 | # Kees Cook <[email protected]> |
| 22 | # Linus Torvalds <[email protected]> |
| 23 | # Magnus Damm <[email protected]> |
| 24 | # Masahiro Yamada <[email protected]> |
| 25 | # Mauro Carvalho Chehab <[email protected]> |
| 26 | # Maxim Cournoyer <[email protected]> |
| 27 | # Peter Foley <[email protected]> |
| 28 | # Randy Dunlap <[email protected]> |
| 29 | # Rob Herring <[email protected]> |
| 30 | # Shuah Khan <[email protected]> |
| 31 | # Thorsten Blum <[email protected]> |
| 32 | # Tomas Winkler <[email protected]> |
| 33 | |
| 34 | |
| 35 | """ |
| 36 | Sphinx build wrapper that handles Kernel-specific business rules: |
| 37 | |
| 38 | - it gets the Kernel build environment vars; |
| 39 | - it determines what's the best parallelism; |
| 40 | - it handles SPHINXDIRS |
| 41 | |
| 42 | This tool ensures that MIN_PYTHON_VERSION is satisfied. If version is |
| 43 | below that, it seeks for a new Python version. If found, it re-runs using |
| 44 | the newer version. |
| 45 | """ |
| 46 | |
| 47 | import argparse |
| 48 | import locale |
| 49 | import os |
| 50 | import re |
| 51 | import shlex |
| 52 | import shutil |
| 53 | import subprocess |
| 54 | import sys |
| 55 | |
| 56 | from concurrent import futures |
| 57 | from glob import glob |
| 58 | |
| 59 | |
| 60 | LIB_DIR = "../lib/python" |
| 61 | SRC_DIR = os.path.dirname(os.path.realpath(__file__)) |
| 62 | |
| 63 | sys.path.insert(0, os.path.join(SRC_DIR, LIB_DIR)) |
| 64 | |
| 65 | from kdoc.python_version import PythonVersion |
| 66 | from kdoc.latex_fonts import LatexFontChecker |
| 67 | from jobserver import JobserverExec # pylint: disable=C0413,C0411,E0401 |
| 68 | |
| 69 | # |
| 70 | # Some constants |
| 71 | # |
| 72 | VENV_DEFAULT = "sphinx_latest" |
| 73 | MIN_PYTHON_VERSION = PythonVersion("3.7").version |
| 74 | PAPER = ["", "a4", "letter"] |
| 75 | |
| 76 | TARGETS = { |
| 77 | "cleandocs": { "builder": "clean" }, |
| 78 | "linkcheckdocs": { "builder": "linkcheck" }, |
| 79 | "htmldocs": { "builder": "html" }, |
| 80 | "epubdocs": { "builder": "epub", "out_dir": "epub" }, |
| 81 | "texinfodocs": { "builder": "texinfo", "out_dir": "texinfo" }, |
| 82 | "infodocs": { "builder": "texinfo", "out_dir": "texinfo" }, |
| 83 | "mandocs": { "builder": "man", "out_dir": "man" }, |
| 84 | "latexdocs": { "builder": "latex", "out_dir": "latex" }, |
| 85 | "pdfdocs": { "builder": "latex", "out_dir": "latex" }, |
| 86 | "xmldocs": { "builder": "xml", "out_dir": "xml" }, |
| 87 | } |
| 88 | |
| 89 | |
| 90 | # |
| 91 | # SphinxBuilder class |
| 92 | # |
| 93 | |
| 94 | class SphinxBuilder: |
| 95 | """ |
| 96 | Handles a sphinx-build target, adding needed arguments to build |
| 97 | with the Kernel. |
| 98 | """ |
| 99 | |
| 100 | def get_path(self, path, use_cwd=False, abs_path=False): |
| 101 | """ |
| 102 | Ancillary routine to handle paths the right way, as shell does. |
| 103 | |
| 104 | It first expands "~" and "~user". Then, if path is not absolute, |
| 105 | join self.srctree. Finally, if requested, convert to abspath. |
| 106 | """ |
| 107 | |
| 108 | path = os.path.expanduser(path) |
| 109 | if not path.startswith("/"): |
| 110 | if use_cwd: |
| 111 | base = os.getcwd() |
| 112 | else: |
| 113 | base = self.srctree |
| 114 | |
| 115 | path = os.path.join(base, path) |
| 116 | |
| 117 | if abs_path: |
| 118 | return os.path.abspath(path) |
| 119 | |
| 120 | return path |
| 121 | |
| 122 | def check_rust(self, sphinxdirs): |
| 123 | """ |
| 124 | Checks if Rust is enabled |
| 125 | """ |
| 126 | config = os.path.join(self.srctree, ".config") |
| 127 | |
| 128 | if not {'.', 'rust'}.intersection(sphinxdirs): |
| 129 | return False |
| 130 | |
| 131 | if not os.path.isfile(config): |
| 132 | return False |
| 133 | |
| 134 | re_rust = re.compile(r"CONFIG_RUST=(m|y)") |
| 135 | |
| 136 | try: |
| 137 | with open(config, "r", encoding="utf-8") as fp: |
| 138 | for line in fp: |
| 139 | if re_rust.match(line): |
| 140 | return True |
| 141 | |
| 142 | except OSError as e: |
| 143 | print(f"Failed to open {config}", file=sys.stderr) |
| 144 | return False |
| 145 | |
| 146 | return False |
| 147 | |
| 148 | def get_sphinx_extra_opts(self, n_jobs): |
| 149 | """ |
| 150 | Get the number of jobs to be used for docs build passed via command |
| 151 | line and desired sphinx verbosity. |
| 152 | |
| 153 | The number of jobs can be on different places: |
| 154 | |
| 155 | 1) It can be passed via "-j" argument; |
| 156 | 2) The SPHINXOPTS="-j8" env var may have "-j"; |
| 157 | 3) if called via GNU make, -j specifies the desired number of jobs. |
| 158 | with GNU makefile, this number is available via POSIX jobserver; |
| 159 | 4) if none of the above is available, it should default to "-jauto", |
| 160 | and let sphinx decide the best value. |
| 161 | """ |
| 162 | |
| 163 | # |
| 164 | # SPHINXOPTS env var, if used, contains extra arguments to be used |
| 165 | # by sphinx-build time. Among them, it may contain sphinx verbosity |
| 166 | # and desired number of parallel jobs. |
| 167 | # |
| 168 | parser = argparse.ArgumentParser() |
| 169 | parser.add_argument('-j', '--jobs', type=int) |
| 170 | parser.add_argument('-q', '--quiet', action='store_true') |
| 171 | parser.add_argument('-v', '--verbose', default=0, action='count') |
| 172 | |
| 173 | # |
| 174 | # Other sphinx-build arguments go as-is, so place them |
| 175 | # at self.sphinxopts, using shell parser |
| 176 | # |
| 177 | sphinxopts = shlex.split(os.environ.get("SPHINXOPTS", "")) |
| 178 | |
| 179 | # |
| 180 | # Build a list of sphinx args, honoring verbosity here if specified |
| 181 | # |
| 182 | |
| 183 | sphinx_args, self.sphinxopts = parser.parse_known_args(sphinxopts) |
| 184 | |
| 185 | verbose = sphinx_args.verbose |
| 186 | if self.verbose: |
| 187 | verbose += 1 |
| 188 | |
| 189 | if sphinx_args.quiet is True: |
| 190 | verbose = 0 |
| 191 | |
| 192 | # |
| 193 | # If the user explicitly sets "-j" at command line, use it. |
| 194 | # Otherwise, pick it from SPHINXOPTS args |
| 195 | # |
| 196 | if n_jobs: |
| 197 | self.n_jobs = n_jobs |
| 198 | elif sphinx_args.jobs: |
| 199 | self.n_jobs = sphinx_args.jobs |
| 200 | else: |
| 201 | self.n_jobs = None |
| 202 | |
| 203 | if verbose < 1: |
| 204 | self.sphinxopts += ["-q"] |
| 205 | else: |
| 206 | for i in range(1, sphinx_args.verbose): |
| 207 | self.sphinxopts += ["-v"] |
| 208 | |
| 209 | def __init__(self, builddir, venv=None, verbose=False, n_jobs=None, |
| 210 | interactive=None): |
| 211 | """Initialize internal variables""" |
| 212 | self.venv = venv |
| 213 | self.verbose = None |
| 214 | |
| 215 | # |
| 216 | # Normal variables passed from Kernel's makefile |
| 217 | # |
| 218 | self.kernelversion = os.environ.get("KERNELVERSION", "unknown") |
| 219 | self.kernelrelease = os.environ.get("KERNELRELEASE", "unknown") |
| 220 | self.pdflatex = os.environ.get("PDFLATEX", "xelatex") |
| 221 | |
| 222 | # |
| 223 | # Add localversion* to kernelversion if present |
| 224 | # |
| 225 | for file in glob(os.environ["srctree"] + "/localversion*"): |
| 226 | if not file.endswith(".orig"): |
| 227 | with open(file, 'r', encoding='utf-8') as f: |
| 228 | text = f.read() |
| 229 | self.kernelversion += text |
| 230 | |
| 231 | # |
| 232 | # Kernel main Makefile defines a PYTHON3 variable whose default is |
| 233 | # "python3". When set to a different value, it allows running a |
| 234 | # diferent version than the default official python3 package. |
| 235 | # Several distros package python3xx-sphinx packages with newer |
| 236 | # versions of Python and sphinx-build. |
| 237 | # |
| 238 | # Honor such variable different than default |
| 239 | # |
| 240 | self.python = os.environ.get("PYTHON3") |
| 241 | if self.python == "python3": |
| 242 | self.python = None |
| 243 | |
| 244 | if not interactive: |
| 245 | self.latexopts = os.environ.get("LATEXOPTS", "-interaction=batchmode -no-shell-escape") |
| 246 | else: |
| 247 | self.latexopts = os.environ.get("LATEXOPTS", "") |
| 248 | |
| 249 | if not verbose: |
| 250 | try: |
| 251 | verbose = bool(int(os.environ.get("KBUILD_VERBOSE", 0))) |
| 252 | except ValueError: |
| 253 | # Handles an eventual case where verbosity is not a number |
| 254 | # like KBUILD_VERBOSE="" |
| 255 | verbose = False |
| 256 | |
| 257 | if verbose is not None: |
| 258 | self.verbose = verbose |
| 259 | |
| 260 | # |
| 261 | # Source tree directory. This needs to be at os.environ, as |
| 262 | # Sphinx extensions use it |
| 263 | # |
| 264 | self.srctree = os.environ.get("srctree") |
| 265 | if not self.srctree: |
| 266 | self.srctree = "." |
| 267 | os.environ["srctree"] = self.srctree |
| 268 | |
| 269 | # |
| 270 | # Now that we can expand srctree, get other directories as well |
| 271 | # |
| 272 | self.sphinxbuild = os.environ.get("SPHINXBUILD", "sphinx-build") |
| 273 | self.kerneldoc = self.get_path(os.environ.get("KERNELDOC", |
| 274 | "tools/docs/kernel-doc")) |
| 275 | self.builddir = self.get_path(builddir, use_cwd=True, abs_path=True) |
| 276 | |
| 277 | # |
| 278 | # Get directory locations for LaTeX build toolchain |
| 279 | # |
| 280 | self.pdflatex_cmd = shutil.which(self.pdflatex) |
| 281 | self.latexmk_cmd = shutil.which("latexmk") |
| 282 | |
| 283 | self.env = os.environ.copy() |
| 284 | |
| 285 | self.get_sphinx_extra_opts(n_jobs) |
| 286 | |
| 287 | # |
| 288 | # If venv command line argument is specified, run Sphinx from venv |
| 289 | # |
| 290 | if venv: |
| 291 | bin_dir = os.path.join(venv, "bin") |
| 292 | if not os.path.isfile(os.path.join(bin_dir, "activate")): |
| 293 | sys.exit(f"Venv {venv} not found.") |
| 294 | |
| 295 | # "activate" virtual env |
| 296 | self.env["PATH"] = bin_dir + ":" + self.env["PATH"] |
| 297 | self.env["VIRTUAL_ENV"] = venv |
| 298 | if "PYTHONHOME" in self.env: |
| 299 | del self.env["PYTHONHOME"] |
| 300 | print(f"Setting venv to {venv}") |
| 301 | |
| 302 | def run_sphinx(self, sphinx_build, build_args, *args, **pwargs): |
| 303 | """ |
| 304 | Executes sphinx-build using current python3 command. |
| 305 | |
| 306 | When calling via GNU make, POSIX jobserver is used to tell how |
| 307 | many jobs are still available from a job pool. claim all remaining |
| 308 | jobs, as we don't want sphinx-build to run in parallel with other |
| 309 | jobs. |
| 310 | |
| 311 | Despite that, the user may actually force a different value than |
| 312 | the number of available jobs via command line. |
| 313 | |
| 314 | The "with" logic here is used to ensure that the claimed jobs will |
| 315 | be freed once subprocess finishes |
| 316 | """ |
| 317 | |
| 318 | with JobserverExec() as jobserver: |
| 319 | if jobserver.claim: |
| 320 | # |
| 321 | # when GNU make is used, claim available jobs from jobserver |
| 322 | # |
| 323 | n_jobs = str(jobserver.claim) |
| 324 | else: |
| 325 | # |
| 326 | # Otherwise, let sphinx decide by default |
| 327 | # |
| 328 | n_jobs = "auto" |
| 329 | |
| 330 | # |
| 331 | # If explicitly requested via command line, override default |
| 332 | # |
| 333 | if self.n_jobs: |
| 334 | n_jobs = str(self.n_jobs) |
| 335 | |
| 336 | # |
| 337 | # We can't simply call python3 sphinx-build, as OpenSUSE |
| 338 | # Tumbleweed uses an ELF binary file (/usr/bin/alts) to switch |
| 339 | # between different versions of sphinx-build. So, only call it |
| 340 | # prepending "python3.xx" when PYTHON3 variable is not default. |
| 341 | # |
| 342 | if self.python: |
| 343 | cmd = [self.python] |
| 344 | else: |
| 345 | cmd = [] |
| 346 | |
| 347 | cmd += [sphinx_build] |
| 348 | cmd += [f"-j{n_jobs}"] |
| 349 | cmd += build_args |
| 350 | cmd += self.sphinxopts |
| 351 | |
| 352 | if self.verbose: |
| 353 | print(" ".join(cmd)) |
| 354 | |
| 355 | return subprocess.call(cmd, *args, **pwargs) |
| 356 | |
| 357 | def handle_html(self, css, output_dir): |
| 358 | """ |
| 359 | Extra steps for HTML and epub output. |
| 360 | |
| 361 | For such targets, we need to ensure that CSS will be properly |
| 362 | copied to the output _static directory |
| 363 | """ |
| 364 | |
| 365 | if css: |
| 366 | css = os.path.expanduser(css) |
| 367 | if not css.startswith("/"): |
| 368 | css = os.path.join(self.srctree, css) |
| 369 | |
| 370 | static_dir = os.path.join(output_dir, "_static") |
| 371 | os.makedirs(static_dir, exist_ok=True) |
| 372 | |
| 373 | try: |
| 374 | shutil.copy2(css, static_dir) |
| 375 | except (OSError, IOError) as e: |
| 376 | print(f"Warning: Failed to copy CSS: {e}", file=sys.stderr) |
| 377 | |
| 378 | def build_pdf_file(self, latex_cmd, from_dir, path): |
| 379 | """Builds a single pdf file using latex_cmd""" |
| 380 | try: |
| 381 | subprocess.run(latex_cmd + [path], |
| 382 | cwd=from_dir, check=True, env=self.env) |
| 383 | |
| 384 | return True |
| 385 | except subprocess.CalledProcessError: |
| 386 | return False |
| 387 | |
| 388 | def pdf_parallel_build(self, tex_suffix, latex_cmd, tex_files, n_jobs): |
| 389 | """Build PDF files in parallel if possible""" |
| 390 | builds = {} |
| 391 | build_failed = False |
| 392 | max_len = 0 |
| 393 | has_tex = False |
| 394 | |
| 395 | # |
| 396 | # LaTeX PDF error code is almost useless for us: |
| 397 | # any warning makes it non-zero. For kernel doc builds it always return |
| 398 | # non-zero even when build succeeds. So, let's do the best next thing: |
| 399 | # Ignore build errors. At the end, check if all PDF files were built, |
| 400 | # printing a summary with the built ones and returning 0 if all of |
| 401 | # them were actually built. |
| 402 | # |
| 403 | with futures.ThreadPoolExecutor(max_workers=n_jobs) as executor: |
| 404 | jobs = {} |
| 405 | |
| 406 | for from_dir, pdf_dir, entry in tex_files: |
| 407 | name = entry.name |
| 408 | |
| 409 | if not name.endswith(tex_suffix): |
| 410 | continue |
| 411 | |
| 412 | name = name[:-len(tex_suffix)] |
| 413 | has_tex = True |
| 414 | |
| 415 | future = executor.submit(self.build_pdf_file, latex_cmd, |
| 416 | from_dir, entry.path) |
| 417 | jobs[future] = (from_dir, pdf_dir, name) |
| 418 | |
| 419 | for future in futures.as_completed(jobs): |
| 420 | from_dir, pdf_dir, name = jobs[future] |
| 421 | |
| 422 | pdf_name = name + ".pdf" |
| 423 | pdf_from = os.path.join(from_dir, pdf_name) |
| 424 | pdf_to = os.path.join(pdf_dir, pdf_name) |
| 425 | out_name = os.path.relpath(pdf_to, self.builddir) |
| 426 | max_len = max(max_len, len(out_name)) |
| 427 | |
| 428 | try: |
| 429 | success = future.result() |
| 430 | |
| 431 | if success and os.path.exists(pdf_from): |
| 432 | os.rename(pdf_from, pdf_to) |
| 433 | |
| 434 | # |
| 435 | # if verbose, get the name of built PDF file |
| 436 | # |
| 437 | if self.verbose: |
| 438 | builds[out_name] = "SUCCESS" |
| 439 | else: |
| 440 | builds[out_name] = "FAILED" |
| 441 | build_failed = True |
| 442 | except futures.Error as e: |
| 443 | builds[out_name] = f"FAILED ({repr(e)})" |
| 444 | build_failed = True |
| 445 | |
| 446 | # |
| 447 | # Handle case where no .tex files were found |
| 448 | # |
| 449 | if not has_tex: |
| 450 | out_name = "LaTeX files" |
| 451 | max_len = max(max_len, len(out_name)) |
| 452 | builds[out_name] = "FAILED: no .tex files were generated" |
| 453 | build_failed = True |
| 454 | |
| 455 | return builds, build_failed, max_len |
| 456 | |
| 457 | def handle_pdf(self, output_dirs, deny_vf): |
| 458 | """ |
| 459 | Extra steps for PDF output. |
| 460 | |
| 461 | As PDF is handled via a LaTeX output, after building the .tex file, |
| 462 | a new build is needed to create the PDF output from the latex |
| 463 | directory. |
| 464 | """ |
| 465 | builds = {} |
| 466 | max_len = 0 |
| 467 | tex_suffix = ".tex" |
| 468 | tex_files = [] |
| 469 | |
| 470 | # |
| 471 | # Since early 2024, Fedora and openSUSE tumbleweed have started |
| 472 | # deploying variable-font format of "Noto CJK", causing LaTeX |
| 473 | # to break with CJK. Work around it, by denying the variable font |
| 474 | # usage during xelatex build by passing the location of a config |
| 475 | # file with a deny list. |
| 476 | # |
| 477 | # See tools/docs/lib/latex_fonts.py for more details. |
| 478 | # |
| 479 | if deny_vf: |
| 480 | deny_vf = os.path.expanduser(deny_vf) |
| 481 | if os.path.isdir(deny_vf): |
| 482 | self.env["XDG_CONFIG_HOME"] = deny_vf |
| 483 | |
| 484 | for from_dir in output_dirs: |
| 485 | pdf_dir = os.path.join(from_dir, "../pdf") |
| 486 | os.makedirs(pdf_dir, exist_ok=True) |
| 487 | |
| 488 | if self.latexmk_cmd: |
| 489 | latex_cmd = [self.latexmk_cmd, f"-{self.pdflatex}"] |
| 490 | else: |
| 491 | latex_cmd = [self.pdflatex] |
| 492 | |
| 493 | latex_cmd.extend(shlex.split(self.latexopts)) |
| 494 | |
| 495 | # Get a list of tex files to process |
| 496 | with os.scandir(from_dir) as it: |
| 497 | for entry in it: |
| 498 | if entry.name.endswith(tex_suffix): |
| 499 | tex_files.append((from_dir, pdf_dir, entry)) |
| 500 | |
| 501 | # |
| 502 | # When using make, this won't be used, as the number of jobs comes |
| 503 | # from POSIX jobserver. So, this covers the case where build comes |
| 504 | # from command line. On such case, serialize by default, except if |
| 505 | # the user explicitly sets the number of jobs. |
| 506 | # |
| 507 | n_jobs = 1 |
| 508 | |
| 509 | # n_jobs is either an integer or "auto". Only use it if it is a number |
| 510 | if self.n_jobs: |
| 511 | try: |
| 512 | n_jobs = int(self.n_jobs) |
| 513 | except ValueError: |
| 514 | pass |
| 515 | |
| 516 | # |
| 517 | # When using make, jobserver.claim is the number of jobs that were |
| 518 | # used with "-j" and that aren't used by other make targets |
| 519 | # |
| 520 | with JobserverExec() as jobserver: |
| 521 | n_jobs = 1 |
| 522 | |
| 523 | # |
| 524 | # Handle the case when a parameter is passed via command line, |
| 525 | # using it as default, if jobserver doesn't claim anything |
| 526 | # |
| 527 | if self.n_jobs: |
| 528 | try: |
| 529 | n_jobs = int(self.n_jobs) |
| 530 | except ValueError: |
| 531 | pass |
| 532 | |
| 533 | if jobserver.claim: |
| 534 | n_jobs = jobserver.claim |
| 535 | |
| 536 | builds, build_failed, max_len = self.pdf_parallel_build(tex_suffix, |
| 537 | latex_cmd, |
| 538 | tex_files, |
| 539 | n_jobs) |
| 540 | |
| 541 | # |
| 542 | # In verbose mode, print a summary with the build results per file. |
| 543 | # Otherwise, print a single line with all failures, if any. |
| 544 | # On both cases, return code 1 indicates build failures, |
| 545 | # |
| 546 | if self.verbose: |
| 547 | msg = "Summary" |
| 548 | msg += "\n" + "=" * len(msg) |
| 549 | print() |
| 550 | print(msg) |
| 551 | |
| 552 | for pdf_name, pdf_file in builds.items(): |
| 553 | print(f"{pdf_name:<{max_len}}: {pdf_file}") |
| 554 | |
| 555 | print() |
| 556 | if build_failed: |
| 557 | msg = LatexFontChecker().check() |
| 558 | if msg: |
| 559 | print(msg) |
| 560 | |
| 561 | sys.exit("Error: not all PDF files were created.") |
| 562 | |
| 563 | elif build_failed: |
| 564 | n_failures = len(builds) |
| 565 | failures = ", ".join(builds.keys()) |
| 566 | |
| 567 | msg = LatexFontChecker().check() |
| 568 | if msg: |
| 569 | print(msg) |
| 570 | |
| 571 | sys.exit(f"Error: Can't build {n_failures} PDF file(s): {failures}") |
| 572 | |
| 573 | def handle_info(self, output_dirs): |
| 574 | """ |
| 575 | Extra steps for Info output. |
| 576 | |
| 577 | For texinfo generation, an additional make is needed from the |
| 578 | texinfo directory. |
| 579 | """ |
| 580 | |
| 581 | for output_dir in output_dirs: |
| 582 | try: |
| 583 | subprocess.run(["make", "info"], cwd=output_dir, check=True) |
| 584 | except subprocess.CalledProcessError as e: |
| 585 | sys.exit(f"Error generating info docs: {e}") |
| 586 | |
| 587 | def handle_man(self, kerneldoc, docs_dir, src_dir, output_dir): |
| 588 | """ |
| 589 | Create man pages from kernel-doc output |
| 590 | """ |
| 591 | |
| 592 | re_kernel_doc = re.compile(r"^\.\.\s+kernel-doc::\s*(\S+)") |
| 593 | |
| 594 | if docs_dir == src_dir: |
| 595 | # |
| 596 | # Pick the entire set of kernel-doc markups from the entire tree |
| 597 | # |
| 598 | kdoc_files = set([self.srctree]) |
| 599 | else: |
| 600 | kdoc_files = set() |
| 601 | |
| 602 | for fname in glob(os.path.join(src_dir, "**"), recursive=True): |
| 603 | if os.path.isfile(fname) and fname.endswith(".rst"): |
| 604 | with open(fname, "r", encoding="utf-8") as in_fp: |
| 605 | data = in_fp.read() |
| 606 | |
| 607 | for line in data.split("\n"): |
| 608 | match = re_kernel_doc.match(line) |
| 609 | if match: |
| 610 | if os.path.isfile(match.group(1)): |
| 611 | kdoc_files.add(match.group(1)) |
| 612 | |
| 613 | if not kdoc_files: |
| 614 | sys.exit(f"Directory {src_dir} doesn't contain kernel-doc tags") |
| 615 | |
| 616 | cmd = [ kerneldoc, "-m" ] + sorted(kdoc_files) |
| 617 | try: |
| 618 | if self.verbose: |
| 619 | print(" ".join(cmd)) |
| 620 | |
| 621 | result = subprocess.run(cmd, stdout=subprocess.PIPE, text= True) |
| 622 | |
| 623 | if result.returncode: |
| 624 | print(f"Warning: kernel-doc returned {result.returncode} warnings") |
| 625 | |
| 626 | except (OSError, ValueError, subprocess.SubprocessError) as e: |
| 627 | sys.exit(f"Failed to create man pages for {src_dir}: {repr(e)}") |
| 628 | |
| 629 | fp = None |
| 630 | try: |
| 631 | for line in result.stdout.split("\n"): |
| 632 | if not line.startswith(".TH"): |
| 633 | if fp: |
| 634 | fp.write(line + '\n') |
| 635 | continue |
| 636 | |
| 637 | if fp: |
| 638 | fp.close() |
| 639 | |
| 640 | # Use shlex here, as it handles well parameters with commas |
| 641 | args = shlex.split(line) |
| 642 | fname = f"{args[1]}.{args[2]}" |
| 643 | fname = fname.replace("/", " ") |
| 644 | fname = f"{output_dir}/{fname}" |
| 645 | |
| 646 | if self.verbose: |
| 647 | print(f"Creating {fname}") |
| 648 | fp = open(fname, "w", encoding="utf-8") |
| 649 | fp.write(line + '\n') |
| 650 | finally: |
| 651 | if fp: |
| 652 | fp.close() |
| 653 | |
| 654 | def cleandocs(self, builder): # pylint: disable=W0613 |
| 655 | """Remove documentation output directory""" |
| 656 | shutil.rmtree(self.builddir, ignore_errors=True) |
| 657 | |
| 658 | def build(self, target, sphinxdirs=None, |
| 659 | theme=None, css=None, paper=None, deny_vf=None, |
| 660 | skip_sphinx=False): |
| 661 | """ |
| 662 | Build documentation using Sphinx. This is the core function of this |
| 663 | module. It prepares all arguments required by sphinx-build. |
| 664 | """ |
| 665 | |
| 666 | builder = TARGETS[target]["builder"] |
| 667 | out_dir = TARGETS[target].get("out_dir", "") |
| 668 | |
| 669 | # |
| 670 | # Cleandocs doesn't require sphinx-build |
| 671 | # |
| 672 | if target == "cleandocs": |
| 673 | self.cleandocs(builder) |
| 674 | return |
| 675 | |
| 676 | if theme: |
| 677 | os.environ["DOCS_THEME"] = theme |
| 678 | |
| 679 | # |
| 680 | # Other targets require sphinx-build, so check if it exists |
| 681 | # |
| 682 | if not skip_sphinx: |
| 683 | sphinxbuild = shutil.which(self.sphinxbuild, path=self.env["PATH"]) |
| 684 | if not sphinxbuild and target != "mandocs": |
| 685 | sys.exit(f"Error: {self.sphinxbuild} not found in PATH.\n") |
| 686 | |
| 687 | if target == "pdfdocs": |
| 688 | if not self.pdflatex_cmd and not self.latexmk_cmd: |
| 689 | sys.exit("Error: pdflatex or latexmk required for PDF generation") |
| 690 | |
| 691 | docs_dir = os.path.abspath(os.path.join(self.srctree, "Documentation")) |
| 692 | |
| 693 | # |
| 694 | # Fill in base arguments for Sphinx build |
| 695 | # |
| 696 | kerneldoc = self.kerneldoc |
| 697 | if kerneldoc.startswith(self.srctree): |
| 698 | kerneldoc = os.path.relpath(kerneldoc, self.srctree) |
| 699 | |
| 700 | if not sphinxdirs: |
| 701 | sphinxdirs = os.environ.get("SPHINXDIRS", ".") |
| 702 | |
| 703 | # |
| 704 | # sphinxdirs can be a list or a whitespace-separated string |
| 705 | # |
| 706 | sphinxdirs_list = [] |
| 707 | for sphinxdir in sphinxdirs: |
| 708 | if isinstance(sphinxdir, list): |
| 709 | sphinxdirs_list += sphinxdir |
| 710 | else: |
| 711 | sphinxdirs_list += sphinxdir.split() |
| 712 | |
| 713 | args = [ "-b", builder, "-c", docs_dir ] |
| 714 | |
| 715 | if builder == "latex": |
| 716 | if not paper: |
| 717 | paper = PAPER[1] |
| 718 | |
| 719 | args.extend(["-D", f"latex_elements.papersize={paper}paper"]) |
| 720 | |
| 721 | rustdoc = self.check_rust(sphinxdirs_list) |
| 722 | if rustdoc: |
| 723 | args.extend(["-t", "rustdoc"]) |
| 724 | |
| 725 | # |
| 726 | # The sphinx-build tool has a bug: internally, it tries to set |
| 727 | # locale with locale.setlocale(locale.LC_ALL, ''). This causes a |
| 728 | # crash if language is not set. Detect and fix it. |
| 729 | # |
| 730 | try: |
| 731 | locale.setlocale(locale.LC_ALL, '') |
| 732 | except locale.Error: |
| 733 | self.env["LC_ALL"] = "C" |
| 734 | |
| 735 | # |
| 736 | # Step 1: Build each directory in separate. |
| 737 | # |
| 738 | # This is not the best way of handling it, as cross-references between |
| 739 | # them will be broken, but this is what we've been doing since |
| 740 | # the beginning. |
| 741 | # |
| 742 | output_dirs = [] |
| 743 | for sphinxdir in sphinxdirs_list: |
| 744 | src_dir = os.path.join(docs_dir, sphinxdir) |
| 745 | doctree_dir = os.path.join(self.builddir, ".doctrees") |
| 746 | output_dir = os.path.join(self.builddir, sphinxdir, out_dir) |
| 747 | |
| 748 | # |
| 749 | # Make directory names canonical |
| 750 | # |
| 751 | src_dir = os.path.normpath(src_dir) |
| 752 | doctree_dir = os.path.normpath(doctree_dir) |
| 753 | output_dir = os.path.normpath(output_dir) |
| 754 | |
| 755 | os.makedirs(doctree_dir, exist_ok=True) |
| 756 | os.makedirs(output_dir, exist_ok=True) |
| 757 | |
| 758 | output_dirs.append(output_dir) |
| 759 | |
| 760 | build_args = args + [ |
| 761 | "-d", doctree_dir, |
| 762 | "-D", f"version={self.kernelversion}", |
| 763 | "-D", f"release={self.kernelrelease}", |
| 764 | "-D", f"kerneldoc_srctree={self.srctree}", |
| 765 | src_dir, |
| 766 | output_dir, |
| 767 | ] |
| 768 | |
| 769 | if target == "mandocs": |
| 770 | self.handle_man(kerneldoc, docs_dir, src_dir, output_dir) |
| 771 | elif not skip_sphinx: |
| 772 | try: |
| 773 | result = self.run_sphinx(sphinxbuild, build_args, |
| 774 | env=self.env) |
| 775 | |
| 776 | if result: |
| 777 | sys.exit(f"Build failed: return code: {result}") |
| 778 | |
| 779 | except (OSError, ValueError, subprocess.SubprocessError) as e: |
| 780 | sys.exit(f"Build failed: {repr(e)}") |
| 781 | |
| 782 | # |
| 783 | # Ensure that each html/epub output will have needed static files |
| 784 | # |
| 785 | if target in ["htmldocs", "epubdocs"]: |
| 786 | self.handle_html(css, output_dir) |
| 787 | |
| 788 | # |
| 789 | # Step 2: Some targets (PDF and info) require an extra step once |
| 790 | # sphinx-build finishes |
| 791 | # |
| 792 | if target == "pdfdocs": |
| 793 | self.handle_pdf(output_dirs, deny_vf) |
| 794 | elif target == "infodocs": |
| 795 | self.handle_info(output_dirs) |
| 796 | |
| 797 | if rustdoc and target in ["htmldocs", "epubdocs"]: |
| 798 | print("Building rust docs") |
| 799 | if "MAKE" in self.env: |
| 800 | cmd = [self.env["MAKE"]] |
| 801 | else: |
| 802 | cmd = ["make", "LLVM=1"] |
| 803 | |
| 804 | cmd += [ "rustdoc"] |
| 805 | if self.verbose: |
| 806 | print(" ".join(cmd)) |
| 807 | |
| 808 | try: |
| 809 | subprocess.run(cmd, check=True) |
| 810 | except subprocess.CalledProcessError as e: |
| 811 | print(f"Ignored errors when building rustdoc: {e}. Is RUST enabled?", |
| 812 | file=sys.stderr) |
| 813 | |
| 814 | def jobs_type(value): |
| 815 | """ |
| 816 | Handle valid values for -j. Accepts Sphinx "-jauto", plus a number |
| 817 | equal or bigger than one. |
| 818 | """ |
| 819 | if value is None: |
| 820 | return None |
| 821 | |
| 822 | if value.lower() == 'auto': |
| 823 | return value.lower() |
| 824 | |
| 825 | try: |
| 826 | if int(value) >= 1: |
| 827 | return value |
| 828 | |
| 829 | raise argparse.ArgumentTypeError(f"Minimum jobs is 1, got {value}") |
| 830 | except ValueError: |
| 831 | raise argparse.ArgumentTypeError(f"Must be 'auto' or positive integer, got {value}") # pylint: disable=W0707 |
| 832 | |
| 833 | EPILOG=""" |
| 834 | Besides the command line arguments, several environment variables affect its |
| 835 | default behavior, meant to be used when called via Kernel Makefile: |
| 836 | |
| 837 | - KERNELVERSION: Kernel major version |
| 838 | - KERNELRELEASE: Kernel release |
| 839 | - KBUILD_VERBOSE: Contains the value of "make V=[0|1] variable. |
| 840 | When V=0 (KBUILD_VERBOSE=0), sets verbose level to "-q". |
| 841 | - SPHINXBUILD: Documentation build tool (default: "sphinx-build"). |
| 842 | - SPHINXOPTS: Extra options pased to SPHINXBUILD |
| 843 | (default: "-j auto" and "-q" if KBUILD_VERBOSE=0). |
| 844 | The "-v" flag can be used to increase verbosity. |
| 845 | If V=0, the first "-v" will drop "-q". |
| 846 | - PYTHON3: Python command to run SPHINXBUILD |
| 847 | - PDFLATEX: LaTeX PDF engine. (default: "xelatex") |
| 848 | - LATEXOPTS: Optional set of command line arguments to the LaTeX engine |
| 849 | - srctree: Location of the Kernel root directory (default: "."). |
| 850 | |
| 851 | """ |
| 852 | |
| 853 | def main(): |
| 854 | """ |
| 855 | Main function. The only mandatory argument is the target. If not |
| 856 | specified, the other arguments will use default values if not |
| 857 | specified at os.environ. |
| 858 | """ |
| 859 | parser = argparse.ArgumentParser(formatter_class=argparse.RawTextHelpFormatter, |
| 860 | description=__doc__, |
| 861 | epilog=EPILOG) |
| 862 | |
| 863 | parser.add_argument("target", choices=list(TARGETS.keys()), |
| 864 | help="Documentation target to build") |
| 865 | parser.add_argument("--sphinxdirs", nargs="+", |
| 866 | help="Specific directories to build") |
| 867 | parser.add_argument("--builddir", default="output", |
| 868 | help="Sphinx configuration file (default: %(default)s)") |
| 869 | |
| 870 | parser.add_argument("--theme", help="Sphinx theme to use") |
| 871 | |
| 872 | parser.add_argument("--css", help="Custom CSS file for HTML/EPUB") |
| 873 | |
| 874 | parser.add_argument("--paper", choices=PAPER, default=PAPER[0], |
| 875 | help="Paper size for LaTeX/PDF output") |
| 876 | |
| 877 | parser.add_argument('--deny-vf', |
| 878 | help="Configuration to deny variable fonts on pdf builds") |
| 879 | |
| 880 | parser.add_argument("-v", "--verbose", action='store_true', |
| 881 | help="place build in verbose mode") |
| 882 | |
| 883 | parser.add_argument('-j', '--jobs', type=jobs_type, |
| 884 | help="Sets number of jobs to use with sphinx-build(default: auto)") |
| 885 | |
| 886 | parser.add_argument('-i', '--interactive', action='store_true', |
| 887 | help="Change latex default to run in interactive mode") |
| 888 | |
| 889 | parser.add_argument('-s', '--skip-sphinx-build', action='store_true', |
| 890 | help="Skip sphinx-build step") |
| 891 | |
| 892 | parser.add_argument("-V", "--venv", nargs='?', const=f'{VENV_DEFAULT}', |
| 893 | default=None, |
| 894 | help=f'If used, run Sphinx from a venv dir (default dir: {VENV_DEFAULT})') |
| 895 | |
| 896 | args = parser.parse_args() |
| 897 | |
| 898 | PythonVersion.check_python(MIN_PYTHON_VERSION, show_alternatives=True, |
| 899 | bail_out=True) |
| 900 | |
| 901 | builder = SphinxBuilder(builddir=args.builddir, venv=args.venv, |
| 902 | verbose=args.verbose, n_jobs=args.jobs, |
| 903 | interactive=args.interactive) |
| 904 | |
| 905 | builder.build(args.target, sphinxdirs=args.sphinxdirs, |
| 906 | theme=args.theme, css=args.css, paper=args.paper, |
| 907 | deny_vf=args.deny_vf, |
| 908 | skip_sphinx=args.skip_sphinx_build) |
| 909 | |
| 910 | if __name__ == "__main__": |
| 911 | main() |