projects/linux·blob master · refs

513 lines · 15.6 KiB

1#!/usr/bin/env python3
2# SPDX-License-Identifier: GPL-2.0
3# Copyright(c) 2025: Mauro Carvalho Chehab <[email protected]>
4#
5# pylint: disable=R0903,R0912,R0913,R0914,R0917,C0301
6
7"""
8Install minimal supported requirements for different Sphinx versions
9and optionally test the build.
10"""
11
12import argparse
13import asyncio
14import os.path
15import shutil
16import sys
17import time
18import subprocess
19
20# Minimal python version supported by the building system.
21
22PYTHON = os.path.basename(sys.executable)
23
24min_python_bin = None
25
26for i in range(9, 13):
27 p = f"python3.{i}"
28 if shutil.which(p):
29 min_python_bin = p
30 break
31
32if not min_python_bin:
33 min_python_bin = PYTHON
34
35# Starting from 8.0, Python 3.9 is not supported anymore.
36PYTHON_VER_CHANGES = {(8, 0, 0): PYTHON}
37
38DEFAULT_VERSIONS_TO_TEST = [
39 (3, 4, 3), # Minimal supported version
40 (5, 3, 0), # CentOS Stream 9 / AlmaLinux 9
41 (6, 1, 1), # Debian 12
42 (7, 2, 1), # openSUSE Leap 15.6
43 (7, 2, 6), # Ubuntu 24.04 LTS
44 (7, 4, 7), # Ubuntu 24.10
45 (7, 3, 0), # openSUSE Tumbleweed
46 (8, 1, 3), # Fedora 42
47 (8, 2, 3) # Latest version - covers rolling distros
48]
49
50# Sphinx versions to be installed and their incremental requirements
51SPHINX_REQUIREMENTS = {
52 # Oldest versions we support for each package required by Sphinx 3.4.3
53 (3, 4, 3): {
54 "docutils": "0.16",
55 "alabaster": "0.7.12",
56 "babel": "2.8.0",
57 "certifi": "2020.6.20",
58 "docutils": "0.16",
59 "idna": "2.10",
60 "imagesize": "1.2.0",
61 "Jinja2": "2.11.2",
62 "MarkupSafe": "1.1.1",
63 "packaging": "20.4",
64 "Pygments": "2.6.1",
65 "PyYAML": "5.1",
66 "requests": "2.24.0",
67 "snowballstemmer": "2.0.0",
68 "sphinxcontrib-applehelp": "1.0.2",
69 "sphinxcontrib-devhelp": "1.0.2",
70 "sphinxcontrib-htmlhelp": "1.0.3",
71 "sphinxcontrib-jsmath": "1.0.1",
72 "sphinxcontrib-qthelp": "1.0.3",
73 "sphinxcontrib-serializinghtml": "1.1.4",
74 "urllib3": "1.25.9",
75 },
76
77 # Update package dependencies to a more modern base. The goal here
78 # is to avoid to many incremental changes for the next entries
79 (3, 5, 0): {
80 "alabaster": "0.7.13",
81 "babel": "2.17.0",
82 "certifi": "2025.6.15",
83 "idna": "3.10",
84 "imagesize": "1.4.1",
85 "packaging": "25.0",
86 "Pygments": "2.8.1",
87 "requests": "2.32.4",
88 "snowballstemmer": "3.0.1",
89 "sphinxcontrib-applehelp": "1.0.4",
90 "sphinxcontrib-htmlhelp": "2.0.1",
91 "sphinxcontrib-serializinghtml": "1.1.5",
92 "urllib3": "2.0.0",
93 },
94
95 # Starting from here, ensure all docutils versions are covered with
96 # supported Sphinx versions. Other packages are upgraded only when
97 # required by pip
98 (4, 0, 0): {
99 "PyYAML": "5.1",
100 },
101 (4, 1, 0): {
102 "docutils": "0.17",
103 "Pygments": "2.19.1",
104 "Jinja2": "3.0.3",
105 "MarkupSafe": "2.0",
106 },
107 (4, 3, 0): {},
108 (4, 4, 0): {},
109 (4, 5, 0): {
110 "docutils": "0.17.1",
111 },
112 (5, 0, 0): {},
113 (5, 1, 0): {},
114 (5, 2, 0): {
115 "docutils": "0.18",
116 "Jinja2": "3.1.2",
117 "MarkupSafe": "2.0",
118 "PyYAML": "5.3.1",
119 },
120 (5, 3, 0): {
121 "docutils": "0.18.1",
122 },
123 (6, 0, 0): {},
124 (6, 1, 0): {},
125 (6, 2, 0): {
126 "PyYAML": "5.4.1",
127 },
128 (7, 0, 0): {},
129 (7, 1, 0): {},
130 (7, 2, 0): {
131 "docutils": "0.19",
132 "PyYAML": "6.0.1",
133 "sphinxcontrib-serializinghtml": "1.1.9",
134 },
135 (7, 2, 6): {
136 "docutils": "0.20",
137 },
138 (7, 3, 0): {
139 "alabaster": "0.7.14",
140 "PyYAML": "6.0.1",
141 "tomli": "2.0.1",
142 },
143 (7, 4, 0): {
144 "docutils": "0.20.1",
145 "PyYAML": "6.0.1",
146 },
147 (8, 0, 0): {
148 "docutils": "0.21",
149 },
150 (8, 1, 0): {
151 "docutils": "0.21.1",
152 "PyYAML": "6.0.1",
153 "sphinxcontrib-applehelp": "1.0.7",
154 "sphinxcontrib-devhelp": "1.0.6",
155 "sphinxcontrib-htmlhelp": "2.0.6",
156 "sphinxcontrib-qthelp": "1.0.6",
157 },
158 (8, 2, 0): {
159 "docutils": "0.21.2",
160 "PyYAML": "6.0.1",
161 "sphinxcontrib-serializinghtml": "1.1.9",
162 },
163}
164
165
166class AsyncCommands:
167 """Excecute command synchronously"""
168
169 def __init__(self, fp=None):
170
171 self.stdout = None
172 self.stderr = None
173 self.output = None
174 self.fp = fp
175
176 def log(self, out, verbose, is_info=True):
177 out = out.removesuffix('\n')
178
179 if verbose:
180 if is_info:
181 print(out)
182 else:
183 print(out, file=sys.stderr)
184
185 if self.fp:
186 self.fp.write(out + "\n")
187
188 async def _read(self, stream, verbose, is_info):
189 """Ancillary routine to capture while displaying"""
190
191 while stream is not None:
192 line = await stream.readline()
193 if line:
194 out = line.decode("utf-8", errors="backslashreplace")
195 self.log(out, verbose, is_info)
196 if is_info:
197 self.stdout += out
198 else:
199 self.stderr += out
200 else:
201 break
202
203 async def run(self, cmd, capture_output=False, check=False,
204 env=None, verbose=True):
205
206 """
207 Execute an arbitrary command, handling errors.
208
209 Please notice that this class is not thread safe
210 """
211
212 self.stdout = ""
213 self.stderr = ""
214
215 self.log("$ " + " ".join(cmd), verbose)
216
217 proc = await asyncio.create_subprocess_exec(cmd[0],
218 *cmd[1:],
219 env=env,
220 stdout=asyncio.subprocess.PIPE,
221 stderr=asyncio.subprocess.PIPE)
222
223 # Handle input and output in realtime
224 await asyncio.gather(
225 self._read(proc.stdout, verbose, True),
226 self._read(proc.stderr, verbose, False),
227 )
228
229 await proc.wait()
230
231 if check and proc.returncode > 0:
232 raise subprocess.CalledProcessError(returncode=proc.returncode,
233 cmd=" ".join(cmd),
234 output=self.stdout,
235 stderr=self.stderr)
236
237 if capture_output:
238 if proc.returncode > 0:
239 self.log(f"Error {proc.returncode}", verbose=True, is_info=False)
240 return ""
241
242 return self.output
243
244 ret = subprocess.CompletedProcess(args=cmd,
245 returncode=proc.returncode,
246 stdout=self.stdout,
247 stderr=self.stderr)
248
249 return ret
250
251
252class SphinxVenv:
253 """
254 Installs Sphinx on one virtual env per Sphinx version with a minimal
255 set of dependencies, adjusting them to each specific version.
256 """
257
258 def __init__(self):
259 """Initialize instance variables"""
260
261 self.built_time = {}
262 self.first_run = True
263
264 async def _handle_version(self, args, fp,
265 cur_ver, cur_requirements, python_bin):
266 """Handle a single Sphinx version"""
267
268 cmd = AsyncCommands(fp)
269
270 ver = ".".join(map(str, cur_ver))
271
272 if not self.first_run and args.wait_input and args.build:
273 ret = input("Press Enter to continue or 'a' to abort: ").strip().lower()
274 if ret == "a":
275 print("Aborted.")
276 sys.exit()
277 else:
278 self.first_run = False
279
280 venv_dir = f"Sphinx_{ver}"
281 req_file = f"requirements_{ver}.txt"
282
283 cmd.log(f"\nSphinx {ver} with {python_bin}", verbose=True)
284
285 # Create venv
286 await cmd.run([python_bin, "-m", "venv", venv_dir],
287 verbose=args.verbose, check=True)
288 pip = os.path.join(venv_dir, "bin/pip")
289
290 # Create install list
291 reqs = []
292 for pkg, verstr in cur_requirements.items():
293 reqs.append(f"{pkg}=={verstr}")
294
295 reqs.append(f"Sphinx=={ver}")
296
297 await cmd.run([pip, "install"] + reqs, check=True, verbose=args.verbose)
298
299 # Freeze environment
300 result = await cmd.run([pip, "freeze"], verbose=False, check=True)
301
302 # Pip install succeeded. Write requirements file
303 if args.req_file:
304 with open(req_file, "w", encoding="utf-8") as fp:
305 fp.write(result.stdout)
306
307 if args.build:
308 start_time = time.time()
309
310 # Prepare a venv environment
311 env = os.environ.copy()
312 bin_dir = os.path.join(venv_dir, "bin")
313 env["PATH"] = bin_dir + ":" + env["PATH"]
314 env["VIRTUAL_ENV"] = venv_dir
315 if "PYTHONHOME" in env:
316 del env["PYTHONHOME"]
317
318 # Test doc build
319 await cmd.run(["make", "cleandocs"], env=env, check=True)
320 make = ["make"]
321
322 if args.output:
323 sphinx_build = os.path.realpath(f"{bin_dir}/sphinx-build")
324 make += [f"O={args.output}", f"SPHINXBUILD={sphinx_build}"]
325
326 if args.make_args:
327 make += args.make_args
328
329 make += args.targets
330
331 if args.verbose:
332 cmd.log(f". {bin_dir}/activate", verbose=True)
333 await cmd.run(make, env=env, check=True, verbose=True)
334 if args.verbose:
335 cmd.log("deactivate", verbose=True)
336
337 end_time = time.time()
338 elapsed_time = end_time - start_time
339 hours, minutes = divmod(elapsed_time, 3600)
340 minutes, seconds = divmod(minutes, 60)
341
342 hours = int(hours)
343 minutes = int(minutes)
344 seconds = int(seconds)
345
346 self.built_time[ver] = f"{hours:02d}:{minutes:02d}:{seconds:02d}"
347
348 cmd.log(f"Finished doc build for Sphinx {ver}. Elapsed time: {self.built_time[ver]}", verbose=True)
349
350 async def run(self, args):
351 """
352 Navigate though multiple Sphinx versions, handling each of them
353 on a loop.
354 """
355
356 if args.log:
357 fp = open(args.log, "w", encoding="utf-8")
358 if not args.verbose:
359 args.verbose = False
360 else:
361 fp = None
362 if not args.verbose:
363 args.verbose = True
364
365 cur_requirements = {}
366 python_bin = min_python_bin
367
368 vers = set(SPHINX_REQUIREMENTS.keys()) | set(args.versions)
369
370 for cur_ver in sorted(vers):
371 if cur_ver in SPHINX_REQUIREMENTS:
372 new_reqs = SPHINX_REQUIREMENTS[cur_ver]
373 cur_requirements.update(new_reqs)
374
375 if cur_ver in PYTHON_VER_CHANGES: # pylint: disable=R1715
376 python_bin = PYTHON_VER_CHANGES[cur_ver]
377
378 if cur_ver not in args.versions:
379 continue
380
381 if args.min_version:
382 if cur_ver < args.min_version:
383 continue
384
385 if args.max_version:
386 if cur_ver > args.max_version:
387 break
388
389 await self._handle_version(args, fp, cur_ver, cur_requirements,
390 python_bin)
391
392 if args.build:
393 cmd = AsyncCommands(fp)
394 cmd.log("\nSummary:", verbose=True)
395 for ver, elapsed_time in sorted(self.built_time.items()):
396 cmd.log(f"\tSphinx {ver} elapsed time: {elapsed_time}",
397 verbose=True)
398
399 if fp:
400 fp.close()
401
402def parse_version(ver_str):
403 """Convert a version string into a tuple."""
404
405 return tuple(map(int, ver_str.split(".")))
406
407
408DEFAULT_VERS = " - "
409DEFAULT_VERS += "\n - ".join(map(lambda v: f"{v[0]}.{v[1]}.{v[2]}",
410 DEFAULT_VERSIONS_TO_TEST))
411
412SCRIPT = os.path.relpath(__file__)
413
414DESCRIPTION = f"""
415This tool allows creating Python virtual environments for different
416Sphinx versions that are supported by the Linux Kernel build system.
417
418Besides creating the virtual environment, it can also test building
419the documentation using "make htmldocs" (and/or other doc targets).
420
421If called without "--versions" argument, it covers the versions shipped
422on major distros, plus the lowest supported version:
423
424{DEFAULT_VERS}
425
426A typical usage is to run:
427
428 {SCRIPT} -m -l sphinx_builds.log
429
430This will create one virtual env for the default version set and run
431"make htmldocs" for each version, creating a log file with the
432excecuted commands on it.
433
434NOTE: The build time can be very long, specially on old versions. Also, there
435is a known bug with Sphinx version 6.0.x: each subprocess uses a lot of
436memory. That, together with "-jauto" may cause OOM killer to cause
437failures at the doc generation. To minimize the risk, you may use the
438"-a" command line parameter to constrain the built directories and/or
439reduce the number of threads from "-jauto" to, for instance, "-j4":
440
441 {SCRIPT} -m -V 6.0.1 -a "SPHINXDIRS=process" "SPHINXOPTS='-j4'"
442
443"""
444
445MAKE_TARGETS = [
446 "htmldocs",
447 "texinfodocs",
448 "infodocs",
449 "latexdocs",
450 "pdfdocs",
451 "epubdocs",
452 "xmldocs",
453]
454
455async def main():
456 """Main program"""
457
458 parser = argparse.ArgumentParser(description=DESCRIPTION,
459 formatter_class=argparse.RawDescriptionHelpFormatter)
460
461 ver_group = parser.add_argument_group("Version range options")
462
463 ver_group.add_argument('-V', '--versions', nargs="*",
464 default=DEFAULT_VERSIONS_TO_TEST,type=parse_version,
465 help='Sphinx versions to test')
466 ver_group.add_argument('--min-version', "--min", type=parse_version,
467 help='Sphinx minimal version')
468 ver_group.add_argument('--max-version', "--max", type=parse_version,
469 help='Sphinx maximum version')
470 ver_group.add_argument('-f', '--full', action='store_true',
471 help='Add all Sphinx (major,minor) supported versions to the version range')
472
473 build_group = parser.add_argument_group("Build options")
474
475 build_group.add_argument('-b', '--build', action='store_true',
476 help='Build documentation')
477 build_group.add_argument('-a', '--make-args', nargs="*",
478 help='extra arguments for make, like SPHINXDIRS=netlink/specs',
479 )
480 build_group.add_argument('-t', '--targets', nargs="+", choices=MAKE_TARGETS,
481 default=[MAKE_TARGETS[0]],
482 help="make build targets. Default: htmldocs.")
483 build_group.add_argument("-o", '--output',
484 help="output directory for the make O=OUTPUT")
485
486 other_group = parser.add_argument_group("Other options")
487
488 other_group.add_argument('-r', '--req-file', action='store_true',
489 help='write a requirements.txt file')
490 other_group.add_argument('-l', '--log',
491 help='Log command output on a file')
492 other_group.add_argument('-v', '--verbose', action='store_true',
493 help='Verbose all commands')
494 other_group.add_argument('-i', '--wait-input', action='store_true',
495 help='Wait for an enter before going to the next version')
496
497 args = parser.parse_args()
498
499 if not args.make_args:
500 args.make_args = []
501
502 sphinx_versions = sorted(list(SPHINX_REQUIREMENTS.keys()))
503
504 if args.full:
505 args.versions += list(SPHINX_REQUIREMENTS.keys())
506
507 venv = SphinxVenv()
508 await venv.run(args)
509
510
511# Call main method
512if __name__ == "__main__":
513 asyncio.run(main())