projects/linux·blob master · refs

208 lines · 4.7 KiB

1/* SPDX-License-Identifier: GPL-2.0-only */
2/*
3 * Copyright (C) 2016 Red Hat, Inc.
4 * Author: Michael S. Tsirkin <[email protected]>
5 *
6 * Common macros and functions for ring benchmarking.
7 */
8#ifndef MAIN_H
9#define MAIN_H
10
11#include <assert.h>
12#include <stdbool.h>
13
14extern int param;
15
16extern bool do_exit;
17
18#if defined(__x86_64__) || defined(__i386__)
19#include "x86intrin.h"
20
21static inline void wait_cycles(unsigned long long cycles)
22{
23 unsigned long long t;
24
25 t = __rdtsc();
26 while (__rdtsc() - t < cycles) {}
27}
28
29#define VMEXIT_CYCLES 500
30#define VMENTRY_CYCLES 500
31
32#elif defined(__s390x__)
33static inline void wait_cycles(unsigned long long cycles)
34{
35 asm volatile("0: brctg %0,0b" : : "d" (cycles));
36}
37
38/* tweak me */
39#define VMEXIT_CYCLES 200
40#define VMENTRY_CYCLES 200
41
42#else
43static inline void wait_cycles(unsigned long long cycles)
44{
45 _Exit(5);
46}
47#define VMEXIT_CYCLES 0
48#define VMENTRY_CYCLES 0
49#endif
50
51static inline void vmexit(void)
52{
53 if (!do_exit)
54 return;
55
56 wait_cycles(VMEXIT_CYCLES);
57}
58static inline void vmentry(void)
59{
60 if (!do_exit)
61 return;
62
63 wait_cycles(VMENTRY_CYCLES);
64}
65
66/* implemented by ring */
67void alloc_ring(void);
68/* guest side */
69int add_inbuf(unsigned, void *, void *);
70void *get_buf(unsigned *, void **);
71void disable_call();
72bool used_empty();
73bool enable_call();
74void kick_available();
75/* host side */
76void disable_kick();
77bool avail_empty();
78bool enable_kick();
79bool use_buf(unsigned *, void **);
80void call_used();
81
82/* implemented by main */
83extern bool do_sleep;
84void kick(void);
85void wait_for_kick(void);
86void call(void);
87void wait_for_call(void);
88
89extern unsigned ring_size;
90
91/* Compiler barrier - similar to what Linux uses */
92#define barrier() asm volatile("" ::: "memory")
93
94/* Is there a portable way to do this? */
95#if defined(__x86_64__) || defined(__i386__)
96#define cpu_relax() asm ("rep; nop" ::: "memory")
97#elif defined(__s390x__)
98#define cpu_relax() barrier()
99#elif defined(__aarch64__)
100#define cpu_relax() asm ("yield" ::: "memory")
101#else
102#define cpu_relax() assert(0)
103#endif
104
105extern bool do_relax;
106
107static inline void busy_wait(void)
108{
109 if (do_relax)
110 cpu_relax();
111 else
112 /* prevent compiler from removing busy loops */
113 barrier();
114}
115
116#if defined(__x86_64__) || defined(__i386__)
117#define smp_mb() asm volatile("lock; addl $0,-132(%%rsp)" ::: "memory", "cc")
118#elif defined(__aarch64__)
119#define smp_mb() asm volatile("dmb ish" ::: "memory")
120#else
121/*
122 * Not using __ATOMIC_SEQ_CST since gcc docs say they are only synchronized
123 * with other __ATOMIC_SEQ_CST calls.
124 */
125#define smp_mb() __sync_synchronize()
126#endif
127
128/*
129 * This abuses the atomic builtins for thread fences, and
130 * adds a compiler barrier.
131 */
132#define smp_release() do { \
133 barrier(); \
134 __atomic_thread_fence(__ATOMIC_RELEASE); \
135} while (0)
136
137#define smp_acquire() do { \
138 __atomic_thread_fence(__ATOMIC_ACQUIRE); \
139 barrier(); \
140} while (0)
141
142#if defined(__i386__) || defined(__x86_64__) || defined(__s390x__)
143#define smp_wmb() barrier()
144#elif defined(__aarch64__)
145#define smp_wmb() asm volatile("dmb ishst" ::: "memory")
146#else
147#define smp_wmb() smp_release()
148#endif
149
150#ifndef __always_inline
151#define __always_inline inline __attribute__((always_inline))
152#endif
153
154static __always_inline
155void __read_once_size(const volatile void *p, void *res, int size)
156{
157 switch (size) {
158 case 1: *(unsigned char *)res = *(volatile unsigned char *)p; break;
159 case 2: *(unsigned short *)res = *(volatile unsigned short *)p; break;
160 case 4: *(unsigned int *)res = *(volatile unsigned int *)p; break;
161 case 8: *(unsigned long long *)res = *(volatile unsigned long long *)p; break;
162 default:
163 barrier();
164 __builtin_memcpy((void *)res, (const void *)p, size);
165 barrier();
166 }
167}
168
169static __always_inline void __write_once_size(volatile void *p, void *res, int size)
170{
171 switch (size) {
172 case 1: *(volatile unsigned char *)p = *(unsigned char *)res; break;
173 case 2: *(volatile unsigned short *)p = *(unsigned short *)res; break;
174 case 4: *(volatile unsigned int *)p = *(unsigned int *)res; break;
175 case 8: *(volatile unsigned long long *)p = *(unsigned long long *)res; break;
176 default:
177 barrier();
178 __builtin_memcpy((void *)p, (const void *)res, size);
179 barrier();
180 }
181}
182
183#ifdef __alpha__
184#define READ_ONCE(x) \
185({ \
186 union { typeof(x) __val; char __c[1]; } __u; \
187 __read_once_size(&(x), __u.__c, sizeof(x)); \
188 smp_mb(); /* Enforce dependency ordering from x */ \
189 __u.__val; \
190})
191#else
192#define READ_ONCE(x) \
193({ \
194 union { typeof(x) __val; char __c[1]; } __u; \
195 __read_once_size(&(x), __u.__c, sizeof(x)); \
196 __u.__val; \
197})
198#endif
199
200#define WRITE_ONCE(x, val) \
201({ \
202 union { typeof(x) __val; char __c[1]; } __u = \
203 { .__val = (typeof(x)) (val) }; \
204 __write_once_size(&(x), __u.__c, sizeof(x)); \
205 __u.__val; \
206})
207
208#endif