projects/linux·blob master · refs

432 lines · 9.2 KiB

1// SPDX-License-Identifier: GPL-2.0-only
2/* MSG_ZEROCOPY feature tests for vsock
3 *
4 * Copyright (C) 2023 SberDevices.
5 *
6 * Author: Arseniy Krasnov <[email protected]>
7 */
8
9#include <stdio.h>
10#include <stdlib.h>
11#include <string.h>
12#include <sys/ioctl.h>
13#include <sys/mman.h>
14#include <unistd.h>
15#include <poll.h>
16#include <linux/errqueue.h>
17#include <linux/kernel.h>
18#include <linux/sockios.h>
19#include <linux/time64.h>
20#include <errno.h>
21
22#include "control.h"
23#include "timeout.h"
24#include "vsock_test_zerocopy.h"
25#include "msg_zerocopy_common.h"
26
27#ifndef PAGE_SIZE
28#define PAGE_SIZE 4096
29#endif
30
31#define VSOCK_TEST_DATA_MAX_IOV 3
32
33struct vsock_test_data {
34 /* This test case if for SOCK_STREAM only. */
35 bool stream_only;
36 /* Data must be zerocopied. This field is checked against
37 * field 'ee_code' of the 'struct sock_extended_err', which
38 * contains bit to detect that zerocopy transmission was
39 * fallbacked to copy mode.
40 */
41 bool zerocopied;
42 /* Enable SO_ZEROCOPY option on the socket. Without enabled
43 * SO_ZEROCOPY, every MSG_ZEROCOPY transmission will behave
44 * like without MSG_ZEROCOPY flag.
45 */
46 bool so_zerocopy;
47 /* 'errno' after 'sendmsg()' call. */
48 int sendmsg_errno;
49 /* Number of valid elements in 'vecs'. */
50 int vecs_cnt;
51 struct iovec vecs[VSOCK_TEST_DATA_MAX_IOV];
52};
53
54static struct vsock_test_data test_data_array[] = {
55 /* Last element has non-page aligned size. */
56 {
57 .zerocopied = true,
58 .so_zerocopy = true,
59 .sendmsg_errno = 0,
60 .vecs_cnt = 3,
61 {
62 { NULL, PAGE_SIZE },
63 { NULL, PAGE_SIZE },
64 { NULL, 200 }
65 }
66 },
67 /* All elements have page aligned base and size. */
68 {
69 .zerocopied = true,
70 .so_zerocopy = true,
71 .sendmsg_errno = 0,
72 .vecs_cnt = 3,
73 {
74 { NULL, PAGE_SIZE },
75 { NULL, PAGE_SIZE * 2 },
76 { NULL, PAGE_SIZE * 3 }
77 }
78 },
79 /* All elements have page aligned base and size. But
80 * data length is bigger than 64Kb.
81 */
82 {
83 .zerocopied = true,
84 .so_zerocopy = true,
85 .sendmsg_errno = 0,
86 .vecs_cnt = 3,
87 {
88 { NULL, PAGE_SIZE * 16 },
89 { NULL, PAGE_SIZE * 16 },
90 { NULL, PAGE_SIZE * 16 }
91 }
92 },
93 /* Middle element has both non-page aligned base and size. */
94 {
95 .zerocopied = true,
96 .so_zerocopy = true,
97 .sendmsg_errno = 0,
98 .vecs_cnt = 3,
99 {
100 { NULL, PAGE_SIZE },
101 { (void *)1, 100 },
102 { NULL, PAGE_SIZE }
103 }
104 },
105 /* Middle element is unmapped. */
106 {
107 .zerocopied = false,
108 .so_zerocopy = true,
109 .sendmsg_errno = ENOMEM,
110 .vecs_cnt = 3,
111 {
112 { NULL, PAGE_SIZE },
113 { MAP_FAILED, PAGE_SIZE },
114 { NULL, PAGE_SIZE }
115 }
116 },
117 /* Valid data, but SO_ZEROCOPY is off. This
118 * will trigger fallback to copy.
119 */
120 {
121 .zerocopied = false,
122 .so_zerocopy = false,
123 .sendmsg_errno = 0,
124 .vecs_cnt = 1,
125 {
126 { NULL, PAGE_SIZE }
127 }
128 },
129 /* Valid data, but message is bigger than peer's
130 * buffer, so this will trigger fallback to copy.
131 * This test is for SOCK_STREAM only, because
132 * for SOCK_SEQPACKET, 'sendmsg()' returns EMSGSIZE.
133 */
134 {
135 .stream_only = true,
136 .zerocopied = false,
137 .so_zerocopy = true,
138 .sendmsg_errno = 0,
139 .vecs_cnt = 1,
140 {
141 { NULL, 100 * PAGE_SIZE }
142 }
143 },
144};
145
146#define POLL_TIMEOUT_MS 100
147
148static void test_client(const struct test_opts *opts,
149 const struct vsock_test_data *test_data,
150 bool sock_seqpacket)
151{
152 struct pollfd fds = { 0 };
153 struct msghdr msg = { 0 };
154 ssize_t sendmsg_res;
155 struct iovec *iovec;
156 int fd;
157
158 if (sock_seqpacket)
159 fd = vsock_seqpacket_connect(opts->peer_cid, opts->peer_port);
160 else
161 fd = vsock_stream_connect(opts->peer_cid, opts->peer_port);
162
163 if (fd < 0) {
164 perror("connect");
165 exit(EXIT_FAILURE);
166 }
167
168 if (test_data->so_zerocopy)
169 enable_so_zerocopy_check(fd);
170
171 iovec = alloc_test_iovec(test_data->vecs, test_data->vecs_cnt);
172
173 msg.msg_iov = iovec;
174 msg.msg_iovlen = test_data->vecs_cnt;
175
176 errno = 0;
177
178 sendmsg_res = sendmsg(fd, &msg, MSG_ZEROCOPY);
179 if (errno != test_data->sendmsg_errno) {
180 fprintf(stderr, "expected 'errno' == %i, got %i\n",
181 test_data->sendmsg_errno, errno);
182 exit(EXIT_FAILURE);
183 }
184
185 if (!errno) {
186 if (sendmsg_res != iovec_bytes(iovec, test_data->vecs_cnt)) {
187 fprintf(stderr, "expected 'sendmsg()' == %li, got %li\n",
188 iovec_bytes(iovec, test_data->vecs_cnt),
189 sendmsg_res);
190 exit(EXIT_FAILURE);
191 }
192 }
193
194 fds.fd = fd;
195 fds.events = 0;
196
197 if (poll(&fds, 1, POLL_TIMEOUT_MS) < 0) {
198 perror("poll");
199 exit(EXIT_FAILURE);
200 }
201
202 if (fds.revents & POLLERR) {
203 vsock_recv_completion(fd, &test_data->zerocopied);
204 } else if (test_data->so_zerocopy && !test_data->sendmsg_errno) {
205 /* If we don't have data in the error queue, but
206 * SO_ZEROCOPY was enabled and 'sendmsg()' was
207 * successful - this is an error.
208 */
209 fprintf(stderr, "POLLERR expected\n");
210 exit(EXIT_FAILURE);
211 }
212
213 if (!test_data->sendmsg_errno)
214 control_writeulong(iovec_hash_djb2(iovec, test_data->vecs_cnt));
215 else
216 control_writeulong(0);
217
218 control_writeln("DONE");
219 free_test_iovec(test_data->vecs, iovec, test_data->vecs_cnt);
220 close(fd);
221}
222
223void test_stream_msgzcopy_client(const struct test_opts *opts)
224{
225 int i;
226
227 for (i = 0; i < ARRAY_SIZE(test_data_array); i++)
228 test_client(opts, &test_data_array[i], false);
229}
230
231void test_seqpacket_msgzcopy_client(const struct test_opts *opts)
232{
233 int i;
234
235 for (i = 0; i < ARRAY_SIZE(test_data_array); i++) {
236 if (test_data_array[i].stream_only)
237 continue;
238
239 test_client(opts, &test_data_array[i], true);
240 }
241}
242
243static void test_server(const struct test_opts *opts,
244 const struct vsock_test_data *test_data,
245 bool sock_seqpacket)
246{
247 unsigned long remote_hash;
248 unsigned long local_hash;
249 ssize_t total_bytes_rec;
250 unsigned char *data;
251 size_t data_len;
252 int fd;
253
254 if (sock_seqpacket)
255 fd = vsock_seqpacket_accept(VMADDR_CID_ANY, opts->peer_port, NULL);
256 else
257 fd = vsock_stream_accept(VMADDR_CID_ANY, opts->peer_port, NULL);
258
259 if (fd < 0) {
260 perror("accept");
261 exit(EXIT_FAILURE);
262 }
263
264 data_len = iovec_bytes(test_data->vecs, test_data->vecs_cnt);
265
266 data = malloc(data_len);
267 if (!data) {
268 perror("malloc");
269 exit(EXIT_FAILURE);
270 }
271
272 total_bytes_rec = 0;
273
274 while (total_bytes_rec != data_len) {
275 ssize_t bytes_rec;
276
277 bytes_rec = read(fd, data + total_bytes_rec,
278 data_len - total_bytes_rec);
279 if (bytes_rec <= 0)
280 break;
281
282 total_bytes_rec += bytes_rec;
283 }
284
285 if (test_data->sendmsg_errno == 0)
286 local_hash = hash_djb2(data, data_len);
287 else
288 local_hash = 0;
289
290 free(data);
291
292 /* Waiting for some result. */
293 remote_hash = control_readulong();
294 if (remote_hash != local_hash) {
295 fprintf(stderr, "hash mismatch\n");
296 exit(EXIT_FAILURE);
297 }
298
299 control_expectln("DONE");
300 close(fd);
301}
302
303void test_stream_msgzcopy_server(const struct test_opts *opts)
304{
305 int i;
306
307 for (i = 0; i < ARRAY_SIZE(test_data_array); i++)
308 test_server(opts, &test_data_array[i], false);
309}
310
311void test_seqpacket_msgzcopy_server(const struct test_opts *opts)
312{
313 int i;
314
315 for (i = 0; i < ARRAY_SIZE(test_data_array); i++) {
316 if (test_data_array[i].stream_only)
317 continue;
318
319 test_server(opts, &test_data_array[i], true);
320 }
321}
322
323void test_stream_msgzcopy_empty_errq_client(const struct test_opts *opts)
324{
325 struct msghdr msg = { 0 };
326 char cmsg_data[128];
327 ssize_t res;
328 int fd;
329
330 fd = vsock_stream_connect(opts->peer_cid, opts->peer_port);
331 if (fd < 0) {
332 perror("connect");
333 exit(EXIT_FAILURE);
334 }
335
336 msg.msg_control = cmsg_data;
337 msg.msg_controllen = sizeof(cmsg_data);
338
339 res = recvmsg(fd, &msg, MSG_ERRQUEUE);
340 if (res != -1) {
341 fprintf(stderr, "expected 'recvmsg(2)' failure, got %zi\n",
342 res);
343 exit(EXIT_FAILURE);
344 }
345
346 control_writeln("DONE");
347 close(fd);
348}
349
350void test_stream_msgzcopy_empty_errq_server(const struct test_opts *opts)
351{
352 int fd;
353
354 fd = vsock_stream_accept(VMADDR_CID_ANY, opts->peer_port, NULL);
355 if (fd < 0) {
356 perror("accept");
357 exit(EXIT_FAILURE);
358 }
359
360 control_expectln("DONE");
361 close(fd);
362}
363
364#define GOOD_COPY_LEN 128 /* net/vmw_vsock/virtio_transport_common.c */
365
366void test_stream_msgzcopy_mangle_client(const struct test_opts *opts)
367{
368 char sbuf1[PAGE_SIZE + 1], sbuf2[GOOD_COPY_LEN];
369 unsigned long hash;
370 struct pollfd fds;
371 int fd, i;
372
373 fd = vsock_stream_connect(opts->peer_cid, opts->peer_port);
374 if (fd < 0) {
375 perror("connect");
376 exit(EXIT_FAILURE);
377 }
378
379 enable_so_zerocopy_check(fd);
380
381 memset(sbuf1, 'x', sizeof(sbuf1));
382 send_buf(fd, sbuf1, sizeof(sbuf1), 0, sizeof(sbuf1));
383
384 for (i = 0; i < sizeof(sbuf2); i++)
385 sbuf2[i] = rand() & 0xff;
386
387 send_buf(fd, sbuf2, sizeof(sbuf2), MSG_ZEROCOPY, sizeof(sbuf2));
388
389 hash = hash_djb2(sbuf2, sizeof(sbuf2));
390 control_writeulong(hash);
391
392 fds.fd = fd;
393 fds.events = 0;
394
395 if (poll(&fds, 1, TIMEOUT * MSEC_PER_SEC) != 1 ||
396 !(fds.revents & POLLERR)) {
397 perror("poll");
398 exit(EXIT_FAILURE);
399 }
400
401 close(fd);
402}
403
404void test_stream_msgzcopy_mangle_server(const struct test_opts *opts)
405{
406 unsigned long local_hash, remote_hash;
407 char rbuf[PAGE_SIZE + 1];
408 int fd;
409
410 fd = vsock_stream_accept(VMADDR_CID_ANY, opts->peer_port, NULL);
411 if (fd < 0) {
412 perror("accept");
413 exit(EXIT_FAILURE);
414 }
415
416 /* Wait, don't race the (buggy) skbs coalescence. */
417 vsock_ioctl_int(fd, SIOCINQ, PAGE_SIZE + 1 + GOOD_COPY_LEN);
418
419 /* Discard the first packet. */
420 recv_buf(fd, rbuf, PAGE_SIZE + 1, 0, PAGE_SIZE + 1);
421
422 recv_buf(fd, rbuf, GOOD_COPY_LEN, 0, GOOD_COPY_LEN);
423 remote_hash = control_readulong();
424 local_hash = hash_djb2(rbuf, GOOD_COPY_LEN);
425
426 if (local_hash != remote_hash) {
427 fprintf(stderr, "Data received corrupted\n");
428 exit(EXIT_FAILURE);
429 }
430
431 close(fd);
432}