projects/linux·blob master · refs

239 lines · 8.2 KiB

1#!/bin/bash
2# SPDX-License-Identifier: GPL-2.0
3#
4# Run the objtool klp tests. Each test-*.sh prints one TAP result line.
5#
6# Tests live in generic/ and in a directory per architecture. A run executes
7# generic/ plus the one matching this architecture, so a test which cannot
8# apply here is not run rather than reporting a skip; what was left out is
9# reported once, as a comment, so differing coverage is still visible.
10#
11# A run covers one compiler and one architecture; CI runs the combinations.
12# The harness checks the environment once up front and fails the run if the
13# suite cannot execute, rather than letting every test skip and exit 0.
14
15set -u
16
17# Determinism: the order test-*.sh expands in, and how grep's character ranges
18# and sort's collation behave inside the tests, are all locale-dependent. A
19# suite whose results depend on the invoking shell's locale is a suite whose
20# failures cannot be reproduced.
21export LC_ALL=C
22
23usage()
24{
25 cat <<EOF
26usage: $(basename "$0") [-k|--keep] [test...]
27
28Run the objtool klp tests for this architecture: everything in generic/, plus
29everything in the directory named for it. With no arguments, runs all of them.
30A test may be named with or without its "test-" prefix and ".sh" suffix, and is
31looked for in both directories.
32
33Options:
34 -k, --keep same as KEEP=all (see below)
35
36Environment:
37 OBJTOOL objtool binary to test (default ../objtool)
38 CC compiler used to build fixtures (default gcc)
39 ARCH architecture the tests are for (default: uname -m)
40 KEEP failed keep only failing tests (default)
41 all keep every test's working directory
42 none remove all working directories
43
44A test which needs something of its own says so in its skip message.
45EOF
46 exit "${1:-0}"
47}
48
49cd "$(dirname "$0")" || exit 1
50
51keep_from_args=
52while [ $# -gt 0 ]; do
53 case "$1" in
54 -h|--help) usage ;;
55 -k|--keep) keep_from_args=all; shift ;;
56 --) shift; break ;;
57 -*) echo "unknown option: $1" >&2; usage 1 ;;
58 *) break ;;
59 esac
60done
61
62KLP_TEST_KEEP="${KEEP:-failed}"
63[ -n "$keep_from_args" ] && KLP_TEST_KEEP="$keep_from_args"
64case "$KLP_TEST_KEEP" in
65all|none|failed) ;;
66*)
67 echo "invalid KEEP=$KLP_TEST_KEEP (want failed, all, or none)" >&2
68 exit 1
69 ;;
70esac
71export KLP_TEST_KEEP
72
73echo "TAP version 13"
74
75# Sourcing the harness runs its preflight, which decides which architecture
76# this run is for -- so the test list cannot be built before it has, and the
77# tests inherit the answers rather than working them out again.
78. ./lib.sh
79
80dirs=( generic )
81[ -d "$KLP_TEST_ARCH" ] && dirs+=( "$KLP_TEST_ARCH" )
82
83if [ $# -gt 0 ]; then
84 tests=()
85 for arg in "$@"; do
86 name="test-${arg#test-}"; name="${name%.sh}.sh"
87 found=
88 for d in "${dirs[@]}"; do
89 [ -f "$d/$name" ] || continue
90 [ -x "$d/$name" ] ||
91 { echo "not executable: $d/$name" >&2; exit 1; }
92 tests+=( "$d/$name" ); found=y
93 done
94 [ -n "$found" ] ||
95 { echo "no such test for $KLP_TEST_ARCH: $arg" >&2; exit 1; }
96 done
97else
98 tests=()
99 for d in "${dirs[@]}"; do
100 for t in "$d"/test-*.sh; do
101 [ -f "$t" ] && tests+=( "$t" )
102 done
103 done
104 [ "${#tests[@]}" -gt 0 ] ||
105 { echo "1..0 # SKIP no tests found"; exit 0; }
106
107 # Tests for another architecture are absent from this run entirely. Say
108 # how many, so a run which covers less than the tree holds does not look
109 # like one that covers all of it.
110 for d in */; do
111 d="${d%/}"
112 case "$d" in generic|"$KLP_TEST_ARCH") continue ;; esac
113 n=$(ls "$d"/test-*.sh 2>/dev/null | wc -l)
114 [ "$n" -gt 0 ] || continue
115 echo "# not run: $n test$( [ "$n" = 1 ] || echo s ) in $d/" \
116 "(this run is $KLP_TEST_ARCH)"
117 done
118fi
119
120# One directory for the whole run, one per test inside it, mirroring the
121# source layout. A run then leaves a single thing behind instead of 39
122# scattered among everything else using mktemp.
123rundir="$(mktemp -d "${TMPDIR:-/tmp}/klp-tests.XXXXXXXX")" ||
124 { echo "Bail out! cannot create a working directory" >&2; exit 1; }
125
126# The run's own two levels: each test's directory, and the one per source
127# directory holding them. Take them away if the tests left them empty, and
128# say so if they did not. Either rmdir may fail -- the glob stays unexpanded
129# when nothing was created -- so ask the directory itself rather than trusting
130# the status. Never rm -rf: what to keep is the tests' decision, made in
131# cleanup() as each one exits, and this must not overrule it.
132reap_rundir()
133{
134 rmdir "$rundir"/*/ 2>/dev/null
135 rmdir "$rundir" 2>/dev/null
136 [ -d "$rundir" ]
137}
138
139# An interrupted run has the same directory to answer for, and the tests it
140# never reached will not clean up on their way out. The one it was running
141# has, and under the default it kept what it had built, so say where.
142interrupted()
143{
144 reap_rundir && echo "# interrupted; what was built is in $rundir"
145 exit 130
146}
147trap interrupted INT TERM HUP
148
149echo "1..${#tests[@]}"
150
151pass=0 fail=0 static_skip=0 probe_skip=0 xfail=0 xpass=0
152failed_dirs=()
153
154for t in "${tests[@]}"; do
155 out="$(KLP_TEST_WORKDIR="$rundir/${t%.sh}" ./"$t" 2>&1)"
156 rc=$?
157
158 # A test prints one result line, but it is not necessarily the only
159 # thing it prints: objtool warns on stderr, and the runner captures
160 # that. Classify the result line itself rather than the whole of the
161 # output, or a stray line ahead of it makes every pattern below miss and
162 # the exit status decide -- which would count an expected failure, which
163 # exits 0, as a pass.
164 result="$(printf '%s\n' "$out" | grep -E '^(ok|not ok)' | tail -1)"
165 rest="$(printf '%s\n' "$out" | grep -Ev '^(ok|not ok)')"
166
167 # Classify from the result line, not the exit status: a skip and a pass
168 # both exit 0, and telling them apart is the point of counting.
169 #
170 # The two skip kinds differ in what they promise. A static skip was
171 # declared before the test ran ("clang does not do this"), so it is
172 # expected indefinitely. A probe skip means the construct did not turn
173 # up this time, which is weaker and worth watching: one that becomes
174 # permanent is a fixture that quietly stopped testing anything.
175 case "$result" in
176 *"# SKIP (declared)"*) static_skip=$((static_skip + 1)) ;;
177 *"# SKIP (probe)"*) probe_skip=$((probe_skip + 1)) ;;
178 *"# SKIP"*)
179 # An undeclared skip: the test gave up for a reason it never
180 # said it might. That is a hole, not an expected outcome.
181 #
182 # Replace the line rather than adding one. Every test owes the
183 # plan exactly one result, and a consumer counting them is
184 # entitled to say so when the totals disagree.
185 rest="$rest${rest:+$'\n'}was: $result"
186 result="not ok - $(basename "$t" .sh): undeclared skip"
187 result="$result (use gcc_only/clang_only or require_input_*)"
188 fail=$((fail + 1)); failed_dirs+=( "$rundir/${t%.sh}" ) ;;
189 "not ok"*"# TODO"*) xfail=$((xfail + 1)) ;;
190 "ok"*"# TODO"*) xpass=$((xpass + 1)); failed_dirs+=( "$rundir/${t%.sh}" ) ;;
191 "not ok"*) fail=$((fail + 1)); failed_dirs+=( "$rundir/${t%.sh}" ) ;;
192 "ok"*) pass=$((pass + 1)) ;;
193 *)
194 # No result line at all: the test died before reporting.
195 rest="$rest${rest:+$'\n'}exited $rc without a result line"
196 result="not ok - $(basename "$t" .sh): no TAP result"
197 fail=$((fail + 1)); failed_dirs+=( "$rundir/${t%.sh}" ) ;;
198 esac
199
200 echo "$result"
201 [ -n "$rest" ] && printf '%s\n' "$rest" | sed 's/^[^#]/# &/'
202
203done
204
205echo "# pass:$pass fail:$fail static-skip:$static_skip" \
206 "probe-skip:$probe_skip xfail:$xfail xpass:$xpass"
207
208case "$KLP_TEST_KEEP" in
209all)
210 echo "# keep=all: workdirs kept in $rundir"
211 echo "# inspect: diff.log, readelf -S out.o under each test-* subdirectory"
212 echo "# cleanup: rm -rf $rundir"
213 ;;
214failed)
215 if [ "${#failed_dirs[@]}" -gt 0 ]; then
216 echo "# keep=failed: ${#failed_dirs[@]} failing test(s) kept under $rundir:"
217 for d in "${failed_dirs[@]}"; do
218 echo "# ${d#"$rundir"/}/"
219 done
220 echo "# inspect: diff.log readelf -S out.o"
221 echo "# one test: $PWD/run-tests.sh <name>"
222 echo "# cleanup: rm -rf $rundir"
223 elif reap_rundir; then
224 echo "# $rundir was not empty;" \
225 "a test did not clean up after itself"
226 fi
227 ;;
228none)
229 if reap_rundir; then
230 echo "# $rundir was not empty;" \
231 "a test did not clean up after itself"
232 elif [ "$fail" != 0 ] || [ "$xpass" != 0 ]; then
233 echo "# keep=none: artifacts were removed" \
234 "(re-run with KEEP=failed or KEEP=all)"
235 fi
236 ;;
237esac
238
239[ "$fail" = 0 ] && [ "$xpass" = 0 ]