projects/linux·blob master · refs

116 lines · 3.0 KiB

1/* SPDX-License-Identifier: GPL-2.0 */
2#ifndef __VDSO_HELPERS_H
3#define __VDSO_HELPERS_H
4
5#ifndef __ASSEMBLER__
6
7#include <asm/barrier.h>
8#include <vdso/datapage.h>
9#include <vdso/processor.h>
10#include <vdso/clocksource.h>
11
12static __always_inline bool vdso_is_timens_clock(const struct vdso_clock *vc)
13{
14 return IS_ENABLED(CONFIG_TIME_NS) && vc->clock_mode == VDSO_CLOCKMODE_TIMENS;
15}
16
17static __always_inline u32 vdso_read_begin(const struct vdso_clock *vc)
18{
19 u32 seq;
20
21 while (unlikely((seq = READ_ONCE(vc->seq)) & 1))
22 cpu_relax();
23
24 smp_rmb();
25 return seq;
26}
27
28/*
29 * Variant of vdso_read_begin() to handle VDSO_CLOCKMODE_TIMENS.
30 *
31 * Time namespace enabled tasks have a special VVAR page installed which has
32 * vc->seq set to 1 and vc->clock_mode set to VDSO_CLOCKMODE_TIMENS. For non
33 * time namespace affected tasks this does not affect performance because if
34 * vc->seq is odd, i.e. a concurrent update is in progress the extra check for
35 * vc->clock_mode is just a few extra instructions while spin waiting for
36 * vc->seq to become even again.
37 */
38static __always_inline bool vdso_read_begin_timens(const struct vdso_clock *vc, u32 *seq)
39{
40 while (unlikely((*seq = READ_ONCE(vc->seq)) & 1)) {
41 if (vdso_is_timens_clock(vc))
42 return true;
43 cpu_relax();
44 }
45 smp_rmb();
46
47 return false;
48}
49
50static __always_inline u32 vdso_read_retry(const struct vdso_clock *vc,
51 u32 start)
52{
53 u32 seq;
54
55 smp_rmb();
56 seq = READ_ONCE(vc->seq);
57 return unlikely(seq != start);
58}
59
60static __always_inline void vdso_write_seq_begin(struct vdso_clock *vc)
61{
62 /*
63 * WRITE_ONCE() is required otherwise the compiler can validly tear
64 * updates to vc->seq and it is possible that the value seen by the
65 * reader is inconsistent.
66 */
67 WRITE_ONCE(vc->seq, vc->seq + 1);
68}
69
70static __always_inline void vdso_write_seq_end(struct vdso_clock *vc)
71{
72 /*
73 * WRITE_ONCE() is required otherwise the compiler can validly tear
74 * updates to vc->seq and it is possible that the value seen by the
75 * reader is inconsistent.
76 */
77 WRITE_ONCE(vc->seq, vc->seq + 1);
78}
79
80static __always_inline void vdso_write_begin_clock(struct vdso_clock *vc)
81{
82 vdso_write_seq_begin(vc);
83 /* Ensure the sequence invalidation is visible before data is modified */
84 smp_wmb();
85}
86
87static __always_inline void vdso_write_end_clock(struct vdso_clock *vc)
88{
89 /* Ensure the data update is visible before the sequence is set valid again */
90 smp_wmb();
91 vdso_write_seq_end(vc);
92}
93
94static __always_inline void vdso_write_begin(struct vdso_time_data *vd)
95{
96 struct vdso_clock *vc = vd->clock_data;
97
98 vdso_write_seq_begin(&vc[CS_HRES_COARSE]);
99 vdso_write_seq_begin(&vc[CS_RAW]);
100 /* Ensure the sequence invalidation is visible before data is modified */
101 smp_wmb();
102}
103
104static __always_inline void vdso_write_end(struct vdso_time_data *vd)
105{
106 struct vdso_clock *vc = vd->clock_data;
107
108 /* Ensure the data update is visible before the sequence is set valid again */
109 smp_wmb();
110 vdso_write_seq_end(&vc[CS_HRES_COARSE]);
111 vdso_write_seq_end(&vc[CS_RAW]);
112}
113
114#endif /* !__ASSEMBLER__ */
115
116#endif /* __VDSO_HELPERS_H */