projects/linux·blob 50c94de6 · refs

380 lines · 13.4 KiB

1# SPDX-License-Identifier: GPL-2.0
2#
3# Runs UML kernel, collects output, and handles errors.
4#
5# Copyright (C) 2019, Google LLC.
6# Author: Felix Guo <[email protected]>
7# Author: Brendan Higgins <[email protected]>
8
9import importlib.abc
10import importlib.util
11import logging
12import subprocess
13import os
14import shutil
15import signal
16import threading
17from typing import Iterator, List, Optional, Tuple
18
19import kunit_config
20import kunit_parser
21import qemu_config
22
23KCONFIG_PATH = '.config'
24KUNITCONFIG_PATH = '.kunitconfig'
25OLD_KUNITCONFIG_PATH = 'last_used_kunitconfig'
26DEFAULT_KUNITCONFIG_PATH = 'tools/testing/kunit/configs/default.config'
27BROKEN_ALLCONFIG_PATH = 'tools/testing/kunit/configs/broken_on_uml.config'
28OUTFILE_PATH = 'test.log'
29ABS_TOOL_PATH = os.path.abspath(os.path.dirname(__file__))
30QEMU_CONFIGS_DIR = os.path.join(ABS_TOOL_PATH, 'qemu_configs')
31
32def get_file_path(build_dir, default):
33 if build_dir:
34 default = os.path.join(build_dir, default)
35 return default
36
37class ConfigError(Exception):
38 """Represents an error trying to configure the Linux kernel."""
39
40
41class BuildError(Exception):
42 """Represents an error trying to build the Linux kernel."""
43
44
45class LinuxSourceTreeOperations(object):
46 """An abstraction over command line operations performed on a source tree."""
47
48 def __init__(self, linux_arch: str, cross_compile: Optional[str]):
49 self._linux_arch = linux_arch
50 self._cross_compile = cross_compile
51
52 def make_mrproper(self) -> None:
53 try:
54 subprocess.check_output(['make', 'mrproper'], stderr=subprocess.STDOUT)
55 except OSError as e:
56 raise ConfigError('Could not call make command: ' + str(e))
57 except subprocess.CalledProcessError as e:
58 raise ConfigError(e.output.decode())
59
60 def make_arch_qemuconfig(self, kconfig: kunit_config.Kconfig) -> None:
61 pass
62
63 def make_allyesconfig(self, build_dir, make_options) -> None:
64 raise ConfigError('Only the "um" arch is supported for alltests')
65
66 def make_olddefconfig(self, build_dir, make_options) -> None:
67 command = ['make', 'ARCH=' + self._linux_arch, 'olddefconfig']
68 if self._cross_compile:
69 command += ['CROSS_COMPILE=' + self._cross_compile]
70 if make_options:
71 command.extend(make_options)
72 if build_dir:
73 command += ['O=' + build_dir]
74 print('Populating config with:\n$', ' '.join(command))
75 try:
76 subprocess.check_output(command, stderr=subprocess.STDOUT)
77 except OSError as e:
78 raise ConfigError('Could not call make command: ' + str(e))
79 except subprocess.CalledProcessError as e:
80 raise ConfigError(e.output.decode())
81
82 def make(self, jobs, build_dir, make_options) -> None:
83 command = ['make', 'ARCH=' + self._linux_arch, '--jobs=' + str(jobs)]
84 if make_options:
85 command.extend(make_options)
86 if self._cross_compile:
87 command += ['CROSS_COMPILE=' + self._cross_compile]
88 if build_dir:
89 command += ['O=' + build_dir]
90 print('Building with:\n$', ' '.join(command))
91 try:
92 proc = subprocess.Popen(command,
93 stderr=subprocess.PIPE,
94 stdout=subprocess.DEVNULL)
95 except OSError as e:
96 raise BuildError('Could not call execute make: ' + str(e))
97 except subprocess.CalledProcessError as e:
98 raise BuildError(e.output)
99 _, stderr = proc.communicate()
100 if proc.returncode != 0:
101 raise BuildError(stderr.decode())
102 if stderr: # likely only due to build warnings
103 print(stderr.decode())
104
105 def start(self, params: List[str], build_dir: str) -> subprocess.Popen:
106 raise RuntimeError('not implemented!')
107
108
109class LinuxSourceTreeOperationsQemu(LinuxSourceTreeOperations):
110
111 def __init__(self, qemu_arch_params: qemu_config.QemuArchParams, cross_compile: Optional[str]):
112 super().__init__(linux_arch=qemu_arch_params.linux_arch,
113 cross_compile=cross_compile)
114 self._kconfig = qemu_arch_params.kconfig
115 self._qemu_arch = qemu_arch_params.qemu_arch
116 self._kernel_path = qemu_arch_params.kernel_path
117 self._kernel_command_line = qemu_arch_params.kernel_command_line + ' kunit_shutdown=reboot'
118 self._extra_qemu_params = qemu_arch_params.extra_qemu_params
119
120 def make_arch_qemuconfig(self, base_kunitconfig: kunit_config.Kconfig) -> None:
121 kconfig = kunit_config.parse_from_string(self._kconfig)
122 base_kunitconfig.merge_in_entries(kconfig)
123
124 def start(self, params: List[str], build_dir: str) -> subprocess.Popen:
125 kernel_path = os.path.join(build_dir, self._kernel_path)
126 qemu_command = ['qemu-system-' + self._qemu_arch,
127 '-nodefaults',
128 '-m', '1024',
129 '-kernel', kernel_path,
130 '-append', '\'' + ' '.join(params + [self._kernel_command_line]) + '\'',
131 '-no-reboot',
132 '-nographic',
133 '-serial stdio'] + self._extra_qemu_params
134 print('Running tests with:\n$', ' '.join(qemu_command))
135 return subprocess.Popen(' '.join(qemu_command),
136 stdin=subprocess.PIPE,
137 stdout=subprocess.PIPE,
138 stderr=subprocess.STDOUT,
139 text=True, shell=True, errors='backslashreplace')
140
141class LinuxSourceTreeOperationsUml(LinuxSourceTreeOperations):
142 """An abstraction over command line operations performed on a source tree."""
143
144 def __init__(self, cross_compile=None):
145 super().__init__(linux_arch='um', cross_compile=cross_compile)
146
147 def make_allyesconfig(self, build_dir, make_options) -> None:
148 kunit_parser.print_with_timestamp(
149 'Enabling all CONFIGs for UML...')
150 command = ['make', 'ARCH=um', 'allyesconfig']
151 if make_options:
152 command.extend(make_options)
153 if build_dir:
154 command += ['O=' + build_dir]
155 process = subprocess.Popen(
156 command,
157 stdout=subprocess.DEVNULL,
158 stderr=subprocess.STDOUT)
159 process.wait()
160 kunit_parser.print_with_timestamp(
161 'Disabling broken configs to run KUnit tests...')
162
163 with open(get_kconfig_path(build_dir), 'a') as config:
164 with open(BROKEN_ALLCONFIG_PATH, 'r') as disable:
165 config.write(disable.read())
166 kunit_parser.print_with_timestamp(
167 'Starting Kernel with all configs takes a few minutes...')
168
169 def start(self, params: List[str], build_dir: str) -> subprocess.Popen:
170 """Runs the Linux UML binary. Must be named 'linux'."""
171 linux_bin = get_file_path(build_dir, 'linux')
172 return subprocess.Popen([linux_bin] + params,
173 stdin=subprocess.PIPE,
174 stdout=subprocess.PIPE,
175 stderr=subprocess.STDOUT,
176 text=True, errors='backslashreplace')
177
178def get_kconfig_path(build_dir) -> str:
179 return get_file_path(build_dir, KCONFIG_PATH)
180
181def get_kunitconfig_path(build_dir) -> str:
182 return get_file_path(build_dir, KUNITCONFIG_PATH)
183
184def get_old_kunitconfig_path(build_dir) -> str:
185 return get_file_path(build_dir, OLD_KUNITCONFIG_PATH)
186
187def get_outfile_path(build_dir) -> str:
188 return get_file_path(build_dir, OUTFILE_PATH)
189
190def get_source_tree_ops(arch: str, cross_compile: Optional[str]) -> LinuxSourceTreeOperations:
191 config_path = os.path.join(QEMU_CONFIGS_DIR, arch + '.py')
192 if arch == 'um':
193 return LinuxSourceTreeOperationsUml(cross_compile=cross_compile)
194 elif os.path.isfile(config_path):
195 return get_source_tree_ops_from_qemu_config(config_path, cross_compile)[1]
196
197 options = [f[:-3] for f in os.listdir(QEMU_CONFIGS_DIR) if f.endswith('.py')]
198 raise ConfigError(arch + ' is not a valid arch, options are ' + str(sorted(options)))
199
200def get_source_tree_ops_from_qemu_config(config_path: str,
201 cross_compile: Optional[str]) -> Tuple[
202 str, LinuxSourceTreeOperations]:
203 # The module name/path has very little to do with where the actual file
204 # exists (I learned this through experimentation and could not find it
205 # anywhere in the Python documentation).
206 #
207 # Bascially, we completely ignore the actual file location of the config
208 # we are loading and just tell Python that the module lives in the
209 # QEMU_CONFIGS_DIR for import purposes regardless of where it actually
210 # exists as a file.
211 module_path = '.' + os.path.join(os.path.basename(QEMU_CONFIGS_DIR), os.path.basename(config_path))
212 spec = importlib.util.spec_from_file_location(module_path, config_path)
213 assert spec is not None
214 config = importlib.util.module_from_spec(spec)
215 # See https://github.com/python/typeshed/pull/2626 for context.
216 assert isinstance(spec.loader, importlib.abc.Loader)
217 spec.loader.exec_module(config)
218
219 if not hasattr(config, 'QEMU_ARCH'):
220 raise ValueError('qemu_config module missing "QEMU_ARCH": ' + config_path)
221 params: qemu_config.QemuArchParams = config.QEMU_ARCH # type: ignore
222 return params.linux_arch, LinuxSourceTreeOperationsQemu(
223 params, cross_compile=cross_compile)
224
225class LinuxSourceTree(object):
226 """Represents a Linux kernel source tree with KUnit tests."""
227
228 def __init__(
229 self,
230 build_dir: str,
231 load_config=True,
232 kunitconfig_path='',
233 kconfig_add: Optional[List[str]]=None,
234 arch=None,
235 cross_compile=None,
236 qemu_config_path=None) -> None:
237 signal.signal(signal.SIGINT, self.signal_handler)
238 if qemu_config_path:
239 self._arch, self._ops = get_source_tree_ops_from_qemu_config(
240 qemu_config_path, cross_compile)
241 else:
242 self._arch = 'um' if arch is None else arch
243 self._ops = get_source_tree_ops(self._arch, cross_compile)
244
245 if not load_config:
246 return
247
248 if kunitconfig_path:
249 if os.path.isdir(kunitconfig_path):
250 kunitconfig_path = os.path.join(kunitconfig_path, KUNITCONFIG_PATH)
251 if not os.path.exists(kunitconfig_path):
252 raise ConfigError(f'Specified kunitconfig ({kunitconfig_path}) does not exist')
253 else:
254 kunitconfig_path = get_kunitconfig_path(build_dir)
255 if not os.path.exists(kunitconfig_path):
256 shutil.copyfile(DEFAULT_KUNITCONFIG_PATH, kunitconfig_path)
257
258 self._kconfig = kunit_config.parse_file(kunitconfig_path)
259 if kconfig_add:
260 kconfig = kunit_config.parse_from_string('\n'.join(kconfig_add))
261 self._kconfig.merge_in_entries(kconfig)
262
263
264 def clean(self) -> bool:
265 try:
266 self._ops.make_mrproper()
267 except ConfigError as e:
268 logging.error(e)
269 return False
270 return True
271
272 def validate_config(self, build_dir) -> bool:
273 kconfig_path = get_kconfig_path(build_dir)
274 validated_kconfig = kunit_config.parse_file(kconfig_path)
275 if self._kconfig.is_subset_of(validated_kconfig):
276 return True
277 invalid = self._kconfig.entries() - validated_kconfig.entries()
278 message = 'Not all Kconfig options selected in kunitconfig were in the generated .config.\n' \
279 'This is probably due to unsatisfied dependencies.\n' \
280 'Missing: ' + ', '.join([str(e) for e in invalid])
281 if self._arch == 'um':
282 message += '\nNote: many Kconfig options aren\'t available on UML. You can try running ' \
283 'on a different architecture with something like "--arch=x86_64".'
284 logging.error(message)
285 return False
286
287 def build_config(self, build_dir, make_options) -> bool:
288 kconfig_path = get_kconfig_path(build_dir)
289 if build_dir and not os.path.exists(build_dir):
290 os.mkdir(build_dir)
291 try:
292 self._ops.make_arch_qemuconfig(self._kconfig)
293 self._kconfig.write_to_file(kconfig_path)
294 self._ops.make_olddefconfig(build_dir, make_options)
295 except ConfigError as e:
296 logging.error(e)
297 return False
298 if not self.validate_config(build_dir):
299 return False
300
301 old_path = get_old_kunitconfig_path(build_dir)
302 if os.path.exists(old_path):
303 os.remove(old_path) # write_to_file appends to the file
304 self._kconfig.write_to_file(old_path)
305 return True
306
307 def _kunitconfig_changed(self, build_dir: str) -> bool:
308 old_path = get_old_kunitconfig_path(build_dir)
309 if not os.path.exists(old_path):
310 return True
311
312 old_kconfig = kunit_config.parse_file(old_path)
313 return old_kconfig.entries() != self._kconfig.entries()
314
315 def build_reconfig(self, build_dir, make_options) -> bool:
316 """Creates a new .config if it is not a subset of the .kunitconfig."""
317 kconfig_path = get_kconfig_path(build_dir)
318 if not os.path.exists(kconfig_path):
319 print('Generating .config ...')
320 return self.build_config(build_dir, make_options)
321
322 existing_kconfig = kunit_config.parse_file(kconfig_path)
323 self._ops.make_arch_qemuconfig(self._kconfig)
324 if self._kconfig.is_subset_of(existing_kconfig) and not self._kunitconfig_changed(build_dir):
325 return True
326 print('Regenerating .config ...')
327 os.remove(kconfig_path)
328 return self.build_config(build_dir, make_options)
329
330 def build_kernel(self, alltests, jobs, build_dir, make_options) -> bool:
331 try:
332 if alltests:
333 self._ops.make_allyesconfig(build_dir, make_options)
334 self._ops.make_olddefconfig(build_dir, make_options)
335 self._ops.make(jobs, build_dir, make_options)
336 except (ConfigError, BuildError) as e:
337 logging.error(e)
338 return False
339 return self.validate_config(build_dir)
340
341 def run_kernel(self, args=None, build_dir='', filter_glob='', timeout=None) -> Iterator[str]:
342 if not args:
343 args = []
344 args.extend(['mem=1G', 'console=tty', 'kunit_shutdown=halt'])
345 if filter_glob:
346 args.append('kunit.filter_glob='+filter_glob)
347
348 process = self._ops.start(args, build_dir)
349 assert process.stdout is not None # tell mypy it's set
350
351 # Enforce the timeout in a background thread.
352 def _wait_proc():
353 try:
354 process.wait(timeout=timeout)
355 except Exception as e:
356 print(e)
357 process.terminate()
358 process.wait()
359 waiter = threading.Thread(target=_wait_proc)
360 waiter.start()
361
362 output = open(get_outfile_path(build_dir), 'w')
363 try:
364 # Tee the output to the file and to our caller in real time.
365 for line in process.stdout:
366 output.write(line)
367 yield line
368 # This runs even if our caller doesn't consume every line.
369 finally:
370 # Flush any leftover output to the file
371 output.write(process.stdout.read())
372 output.close()
373 process.stdout.close()
374
375 waiter.join()
376 subprocess.call(['stty', 'sane'])
377
378 def signal_handler(self, sig, frame) -> None:
379 logging.error('Build interruption occurred. Cleaning console.')
380 subprocess.call(['stty', 'sane'])