projects/linux·blob master · refs

1903 lines · 53.6 KiB

1// SPDX-License-Identifier: GPL-2.0-only
2/*
3 * Copyright (c) 2009, Microsoft Corporation.
4 *
5 * Authors:
6 * Haiyang Zhang <[email protected]>
7 * Hank Janssen <[email protected]>
8 */
9#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
10
11#include <linux/kernel.h>
12#include <linux/sched.h>
13#include <linux/wait.h>
14#include <linux/mm.h>
15#include <linux/highmem.h>
16#include <linux/delay.h>
17#include <linux/io.h>
18#include <linux/slab.h>
19#include <linux/netdevice.h>
20#include <linux/if_ether.h>
21#include <linux/vmalloc.h>
22#include <linux/rtnetlink.h>
23#include <linux/prefetch.h>
24#include <linux/filter.h>
25
26#include <asm/sync_bitops.h>
27#include <asm/mshyperv.h>
28
29#include "hyperv_net.h"
30#include "netvsc_trace.h"
31
32static struct workqueue_struct *netvsc_wq;
33
34/*
35 * Switch the data path from the synthetic interface to the VF
36 * interface.
37 */
38int netvsc_switch_datapath(struct net_device *ndev, bool vf)
39{
40 struct net_device_context *net_device_ctx = netdev_priv(ndev);
41 struct hv_device *dev = net_device_ctx->device_ctx;
42 struct netvsc_device *nv_dev = rtnl_dereference(net_device_ctx->nvdev);
43 struct nvsp_message *init_pkt = &nv_dev->channel_init_pkt;
44 int ret, retry = 0;
45
46 /* Block sending traffic to VF if it's about to be gone */
47 if (!vf)
48 net_device_ctx->data_path_is_vf = vf;
49
50 memset(init_pkt, 0, sizeof(struct nvsp_message));
51 init_pkt->hdr.msg_type = NVSP_MSG4_TYPE_SWITCH_DATA_PATH;
52 if (vf)
53 init_pkt->msg.v4_msg.active_dp.active_datapath =
54 NVSP_DATAPATH_VF;
55 else
56 init_pkt->msg.v4_msg.active_dp.active_datapath =
57 NVSP_DATAPATH_SYNTHETIC;
58
59again:
60 trace_nvsp_send(ndev, init_pkt);
61
62 ret = vmbus_sendpacket(dev->channel, init_pkt,
63 sizeof(struct nvsp_message),
64 (unsigned long)init_pkt, VM_PKT_DATA_INBAND,
65 VMBUS_DATA_PACKET_FLAG_COMPLETION_REQUESTED);
66
67 /* If failed to switch to/from VF, let data_path_is_vf stay false,
68 * so we use synthetic path to send data.
69 */
70 if (ret) {
71 if (ret != -EAGAIN) {
72 netdev_err(ndev,
73 "Unable to send sw datapath msg, err: %d\n",
74 ret);
75 return ret;
76 }
77
78 if (retry++ < RETRY_MAX) {
79 usleep_range(RETRY_US_LO, RETRY_US_HI);
80 goto again;
81 } else {
82 netdev_err(
83 ndev,
84 "Retry failed to send sw datapath msg, err: %d\n",
85 ret);
86 return ret;
87 }
88 }
89
90 wait_for_completion(&nv_dev->channel_init_wait);
91 net_device_ctx->data_path_is_vf = vf;
92
93 return 0;
94}
95
96/* Worker to setup sub channels on initial setup
97 * Initial hotplug event occurs in softirq context
98 * and can't wait for channels.
99 */
100static void netvsc_subchan_work(struct work_struct *w)
101{
102 struct netvsc_device *nvdev =
103 container_of(w, struct netvsc_device, subchan_work);
104 struct rndis_device *rdev;
105 int i, ret;
106
107 /* Avoid deadlock with device removal already under RTNL */
108 if (!rtnl_trylock()) {
109 schedule_work(w);
110 return;
111 }
112
113 rdev = nvdev->extension;
114 if (rdev) {
115 ret = rndis_set_subchannel(rdev->ndev, nvdev, NULL);
116 if (ret == 0) {
117 netif_device_attach(rdev->ndev);
118 } else {
119 /* fallback to only primary channel */
120 for (i = 1; i < nvdev->num_chn; i++)
121 netif_napi_del(&nvdev->chan_table[i].napi);
122
123 nvdev->max_chn = 1;
124 nvdev->num_chn = 1;
125 }
126 }
127
128 rtnl_unlock();
129}
130
131static void __free_netvsc_device(struct netvsc_device *nvdev)
132{
133 int i;
134
135 kfree(nvdev->extension);
136
137 vmbus_free_buffer(nvdev->recv_buf, nvdev->recv_buf_chunks,
138 nvdev->recv_buf_chunk_cnt);
139 vmbus_free_buffer(nvdev->send_buf, nvdev->send_buf_chunks,
140 nvdev->send_buf_chunk_cnt);
141 bitmap_free(nvdev->send_section_map);
142
143 for (i = 0; i < VRSS_CHANNEL_MAX; i++) {
144 xdp_rxq_info_unreg(&nvdev->chan_table[i].xdp_rxq);
145 kfree(nvdev->chan_table[i].recv_buf);
146 vfree(nvdev->chan_table[i].mrc.slots);
147 }
148
149 kfree(nvdev);
150}
151
152static void free_netvsc_device(struct work_struct *w)
153{
154 struct rcu_work *rwork = to_rcu_work(w);
155
156 __free_netvsc_device(container_of(rwork, struct netvsc_device, rwork));
157}
158
159int netvsc_workqueue_init(void)
160{
161 netvsc_wq = alloc_workqueue("hv_netvsc", WQ_UNBOUND, 0);
162
163 return netvsc_wq ? 0 : -ENOMEM;
164}
165
166void netvsc_workqueue_destroy(void)
167{
168 rcu_barrier();
169 destroy_workqueue(netvsc_wq);
170}
171
172static struct netvsc_device *alloc_net_device(void)
173{
174 struct netvsc_device *net_device;
175
176 net_device = kzalloc_obj(struct netvsc_device);
177 if (!net_device)
178 return NULL;
179
180 init_waitqueue_head(&net_device->wait_drain);
181 net_device->destroy = false;
182 net_device->tx_disable = true;
183
184 net_device->max_pkt = RNDIS_MAX_PKT_DEFAULT;
185 net_device->pkt_align = RNDIS_PKT_ALIGN_DEFAULT;
186
187 init_completion(&net_device->channel_init_wait);
188 init_waitqueue_head(&net_device->subchan_open);
189 INIT_WORK(&net_device->subchan_work, netvsc_subchan_work);
190 INIT_RCU_WORK(&net_device->rwork, free_netvsc_device);
191
192 return net_device;
193}
194
195static void free_netvsc_device_rcu(struct netvsc_device *nvdev)
196{
197 /*
198 * Defer the actual free to process context: vunmap() and
199 * set_memory_encrypted() cannot run from RCU softirq context.
200 */
201 queue_rcu_work(netvsc_wq, &nvdev->rwork);
202}
203
204static void netvsc_revoke_recv_buf(struct hv_device *device,
205 struct netvsc_device *net_device,
206 struct net_device *ndev)
207{
208 struct nvsp_message *revoke_packet;
209 int ret;
210
211 /*
212 * If we got a section count, it means we received a
213 * SendReceiveBufferComplete msg (ie sent
214 * NvspMessage1TypeSendReceiveBuffer msg) therefore, we need
215 * to send a revoke msg here
216 */
217 if (net_device->recv_section_cnt) {
218 /* Send the revoke receive buffer */
219 revoke_packet = &net_device->revoke_packet;
220 memset(revoke_packet, 0, sizeof(struct nvsp_message));
221
222 revoke_packet->hdr.msg_type =
223 NVSP_MSG1_TYPE_REVOKE_RECV_BUF;
224 revoke_packet->msg.v1_msg.
225 revoke_recv_buf.id = NETVSC_RECEIVE_BUFFER_ID;
226
227 trace_nvsp_send(ndev, revoke_packet);
228
229 ret = vmbus_sendpacket(device->channel,
230 revoke_packet,
231 sizeof(struct nvsp_message),
232 VMBUS_RQST_ID_NO_RESPONSE,
233 VM_PKT_DATA_INBAND, 0);
234 /* If the failure is because the channel is rescinded;
235 * ignore the failure since we cannot send on a rescinded
236 * channel. This would allow us to properly cleanup
237 * even when the channel is rescinded.
238 */
239 if (device->channel->rescind)
240 ret = 0;
241 /*
242 * If we failed here, we might as well return and
243 * have a leak rather than continue and a bugchk
244 */
245 if (ret != 0) {
246 netdev_err(ndev, "unable to send "
247 "revoke receive buffer to netvsp\n");
248 return;
249 }
250 net_device->recv_section_cnt = 0;
251 }
252}
253
254static void netvsc_revoke_send_buf(struct hv_device *device,
255 struct netvsc_device *net_device,
256 struct net_device *ndev)
257{
258 struct nvsp_message *revoke_packet;
259 int ret;
260
261 /* Deal with the send buffer we may have setup.
262 * If we got a send section size, it means we received a
263 * NVSP_MSG1_TYPE_SEND_SEND_BUF_COMPLETE msg (ie sent
264 * NVSP_MSG1_TYPE_SEND_SEND_BUF msg) therefore, we need
265 * to send a revoke msg here
266 */
267 if (net_device->send_section_cnt) {
268 /* Send the revoke receive buffer */
269 revoke_packet = &net_device->revoke_packet;
270 memset(revoke_packet, 0, sizeof(struct nvsp_message));
271
272 revoke_packet->hdr.msg_type =
273 NVSP_MSG1_TYPE_REVOKE_SEND_BUF;
274 revoke_packet->msg.v1_msg.revoke_send_buf.id =
275 NETVSC_SEND_BUFFER_ID;
276
277 trace_nvsp_send(ndev, revoke_packet);
278
279 ret = vmbus_sendpacket(device->channel,
280 revoke_packet,
281 sizeof(struct nvsp_message),
282 VMBUS_RQST_ID_NO_RESPONSE,
283 VM_PKT_DATA_INBAND, 0);
284
285 /* If the failure is because the channel is rescinded;
286 * ignore the failure since we cannot send on a rescinded
287 * channel. This would allow us to properly cleanup
288 * even when the channel is rescinded.
289 */
290 if (device->channel->rescind)
291 ret = 0;
292
293 /* If we failed here, we might as well return and
294 * have a leak rather than continue and a bugchk
295 */
296 if (ret != 0) {
297 netdev_err(ndev, "unable to send "
298 "revoke send buffer to netvsp\n");
299 return;
300 }
301 net_device->send_section_cnt = 0;
302 }
303}
304
305static void netvsc_teardown_recv_gpadl(struct hv_device *device,
306 struct netvsc_device *net_device,
307 struct net_device *ndev)
308{
309 int ret;
310
311 if (net_device->recv_buf_gpadl_handle.gpadl_handle) {
312 ret = vmbus_teardown_gpadl(device->channel,
313 &net_device->recv_buf_gpadl_handle);
314
315 /* If we failed here, we might as well return and have a leak
316 * rather than continue and a bugchk
317 */
318 if (ret != 0) {
319 netdev_err(ndev,
320 "unable to teardown receive buffer's gpadl\n");
321 return;
322 }
323 }
324}
325
326static void netvsc_teardown_send_gpadl(struct hv_device *device,
327 struct netvsc_device *net_device,
328 struct net_device *ndev)
329{
330 int ret;
331
332 if (net_device->send_buf_gpadl_handle.gpadl_handle) {
333 ret = vmbus_teardown_gpadl(device->channel,
334 &net_device->send_buf_gpadl_handle);
335
336 /* If we failed here, we might as well return and have a leak
337 * rather than continue and a bugchk
338 */
339 if (ret != 0) {
340 netdev_err(ndev,
341 "unable to teardown send buffer's gpadl\n");
342 return;
343 }
344 }
345}
346
347int netvsc_alloc_recv_comp_ring(struct netvsc_device *net_device, u32 q_idx)
348{
349 struct netvsc_channel *nvchan = &net_device->chan_table[q_idx];
350 int node = cpu_to_node(nvchan->channel->target_cpu);
351 size_t size;
352
353 size = net_device->recv_completion_cnt * sizeof(struct recv_comp_data);
354 nvchan->mrc.slots = vzalloc_node(size, node);
355 if (!nvchan->mrc.slots)
356 nvchan->mrc.slots = vzalloc(size);
357
358 return nvchan->mrc.slots ? 0 : -ENOMEM;
359}
360
361static int netvsc_init_buf(struct hv_device *device,
362 struct netvsc_device *net_device,
363 const struct netvsc_device_info *device_info)
364{
365 struct nvsp_1_message_send_receive_buffer_complete *resp;
366 struct net_device *ndev = hv_get_drvdata(device);
367 struct nvsp_message *init_packet;
368 unsigned int buf_size;
369 int i, ret = 0;
370
371 /* Get receive buffer area. */
372 buf_size = device_info->recv_sections * device_info->recv_section_size;
373 buf_size = roundup(buf_size, PAGE_SIZE);
374
375 /* Legacy hosts only allow smaller receive buffer */
376 if (net_device->nvsp_version <= NVSP_PROTOCOL_VERSION_2)
377 buf_size = min_t(unsigned int, buf_size,
378 NETVSC_RECEIVE_BUFFER_SIZE_LEGACY);
379
380 net_device->recv_buf =
381 vmbus_alloc_buffer(device->channel, buf_size,
382 &net_device->recv_buf_chunks,
383 &net_device->recv_buf_chunk_cnt);
384 if (!net_device->recv_buf) {
385 netdev_err(ndev,
386 "unable to allocate receive buffer of size %u\n",
387 buf_size);
388 ret = -ENOMEM;
389 goto cleanup;
390 }
391
392 net_device->recv_buf_size = buf_size;
393
394 /*
395 * Establish the gpadl handle for this buffer on this
396 * channel. Note: This call uses the vmbus connection rather
397 * than the channel to establish the gpadl handle.
398 */
399 ret = vmbus_establish_gpadl_caller_decrypted(device->channel,
400 net_device->recv_buf,
401 buf_size,
402 &net_device->recv_buf_gpadl_handle);
403 if (ret != 0) {
404 netdev_err(ndev,
405 "unable to establish receive buffer's gpadl\n");
406 goto cleanup;
407 }
408
409 /* Notify the NetVsp of the gpadl handle */
410 init_packet = &net_device->channel_init_pkt;
411 memset(init_packet, 0, sizeof(struct nvsp_message));
412 init_packet->hdr.msg_type = NVSP_MSG1_TYPE_SEND_RECV_BUF;
413 init_packet->msg.v1_msg.send_recv_buf.
414 gpadl_handle = net_device->recv_buf_gpadl_handle.gpadl_handle;
415 init_packet->msg.v1_msg.
416 send_recv_buf.id = NETVSC_RECEIVE_BUFFER_ID;
417
418 trace_nvsp_send(ndev, init_packet);
419
420 /* Send the gpadl notification request */
421 ret = vmbus_sendpacket(device->channel, init_packet,
422 sizeof(struct nvsp_message),
423 (unsigned long)init_packet,
424 VM_PKT_DATA_INBAND,
425 VMBUS_DATA_PACKET_FLAG_COMPLETION_REQUESTED);
426 if (ret != 0) {
427 netdev_err(ndev,
428 "unable to send receive buffer's gpadl to netvsp\n");
429 goto cleanup;
430 }
431
432 wait_for_completion(&net_device->channel_init_wait);
433
434 /* Check the response */
435 resp = &init_packet->msg.v1_msg.send_recv_buf_complete;
436 if (resp->status != NVSP_STAT_SUCCESS) {
437 netdev_err(ndev,
438 "Unable to complete receive buffer initialization with NetVsp - status %d\n",
439 resp->status);
440 ret = -EINVAL;
441 goto cleanup;
442 }
443
444 /* Parse the response */
445 netdev_dbg(ndev, "Receive sections: %u sub_allocs: size %u count: %u\n",
446 resp->num_sections, resp->sections[0].sub_alloc_size,
447 resp->sections[0].num_sub_allocs);
448
449 /* There should only be one section for the entire receive buffer */
450 if (resp->num_sections != 1 || resp->sections[0].offset != 0) {
451 ret = -EINVAL;
452 goto cleanup;
453 }
454
455 net_device->recv_section_size = resp->sections[0].sub_alloc_size;
456 net_device->recv_section_cnt = resp->sections[0].num_sub_allocs;
457
458 /* Ensure buffer will not overflow */
459 if (net_device->recv_section_size < NETVSC_MTU_MIN || (u64)net_device->recv_section_size *
460 (u64)net_device->recv_section_cnt > (u64)buf_size) {
461 netdev_err(ndev, "invalid recv_section_size %u\n",
462 net_device->recv_section_size);
463 ret = -EINVAL;
464 goto cleanup;
465 }
466
467 for (i = 0; i < VRSS_CHANNEL_MAX; i++) {
468 struct netvsc_channel *nvchan = &net_device->chan_table[i];
469
470 nvchan->recv_buf = kzalloc(net_device->recv_section_size, GFP_KERNEL);
471 if (nvchan->recv_buf == NULL) {
472 ret = -ENOMEM;
473 goto cleanup;
474 }
475 }
476
477 /* Setup receive completion ring.
478 * Add 1 to the recv_section_cnt because at least one entry in a
479 * ring buffer has to be empty.
480 */
481 net_device->recv_completion_cnt = net_device->recv_section_cnt + 1;
482 ret = netvsc_alloc_recv_comp_ring(net_device, 0);
483 if (ret)
484 goto cleanup;
485
486 /* Now setup the send buffer. */
487 buf_size = device_info->send_sections * device_info->send_section_size;
488 buf_size = round_up(buf_size, PAGE_SIZE);
489
490 net_device->send_buf =
491 vmbus_alloc_buffer(device->channel, buf_size,
492 &net_device->send_buf_chunks,
493 &net_device->send_buf_chunk_cnt);
494 if (!net_device->send_buf) {
495 netdev_err(ndev, "unable to allocate send buffer of size %u\n",
496 buf_size);
497 ret = -ENOMEM;
498 goto cleanup;
499 }
500 net_device->send_buf_size = buf_size;
501
502 /* Establish the gpadl handle for this buffer on this
503 * channel. Note: This call uses the vmbus connection rather
504 * than the channel to establish the gpadl handle.
505 */
506 ret = vmbus_establish_gpadl_caller_decrypted(device->channel,
507 net_device->send_buf,
508 buf_size,
509 &net_device->send_buf_gpadl_handle);
510 if (ret != 0) {
511 netdev_err(ndev,
512 "unable to establish send buffer's gpadl\n");
513 goto cleanup;
514 }
515
516 /* Notify the NetVsp of the gpadl handle */
517 init_packet = &net_device->channel_init_pkt;
518 memset(init_packet, 0, sizeof(struct nvsp_message));
519 init_packet->hdr.msg_type = NVSP_MSG1_TYPE_SEND_SEND_BUF;
520 init_packet->msg.v1_msg.send_send_buf.gpadl_handle =
521 net_device->send_buf_gpadl_handle.gpadl_handle;
522 init_packet->msg.v1_msg.send_send_buf.id = NETVSC_SEND_BUFFER_ID;
523
524 trace_nvsp_send(ndev, init_packet);
525
526 /* Send the gpadl notification request */
527 ret = vmbus_sendpacket(device->channel, init_packet,
528 sizeof(struct nvsp_message),
529 (unsigned long)init_packet,
530 VM_PKT_DATA_INBAND,
531 VMBUS_DATA_PACKET_FLAG_COMPLETION_REQUESTED);
532 if (ret != 0) {
533 netdev_err(ndev,
534 "unable to send send buffer's gpadl to netvsp\n");
535 goto cleanup;
536 }
537
538 wait_for_completion(&net_device->channel_init_wait);
539
540 /* Check the response */
541 if (init_packet->msg.v1_msg.
542 send_send_buf_complete.status != NVSP_STAT_SUCCESS) {
543 netdev_err(ndev, "Unable to complete send buffer "
544 "initialization with NetVsp - status %d\n",
545 init_packet->msg.v1_msg.
546 send_send_buf_complete.status);
547 ret = -EINVAL;
548 goto cleanup;
549 }
550
551 /* Parse the response */
552 net_device->send_section_size = init_packet->msg.
553 v1_msg.send_send_buf_complete.section_size;
554 if (net_device->send_section_size < NETVSC_MTU_MIN) {
555 netdev_err(ndev, "invalid send_section_size %u\n",
556 net_device->send_section_size);
557 ret = -EINVAL;
558 goto cleanup;
559 }
560
561 /* Section count is simply the size divided by the section size. */
562 net_device->send_section_cnt = buf_size / net_device->send_section_size;
563
564 netdev_dbg(ndev, "Send section size: %d, Section count:%d\n",
565 net_device->send_section_size, net_device->send_section_cnt);
566
567 /* Setup state for managing the send buffer. */
568 net_device->send_section_map = bitmap_zalloc(net_device->send_section_cnt,
569 GFP_KERNEL);
570 if (!net_device->send_section_map) {
571 ret = -ENOMEM;
572 goto cleanup;
573 }
574
575 goto exit;
576
577cleanup:
578 netvsc_revoke_recv_buf(device, net_device, ndev);
579 netvsc_revoke_send_buf(device, net_device, ndev);
580 netvsc_teardown_recv_gpadl(device, net_device, ndev);
581 netvsc_teardown_send_gpadl(device, net_device, ndev);
582
583exit:
584 return ret;
585}
586
587/* Negotiate NVSP protocol version */
588static int negotiate_nvsp_ver(struct hv_device *device,
589 struct netvsc_device *net_device,
590 struct nvsp_message *init_packet,
591 u32 nvsp_ver)
592{
593 struct net_device *ndev = hv_get_drvdata(device);
594 int ret;
595
596 memset(init_packet, 0, sizeof(struct nvsp_message));
597 init_packet->hdr.msg_type = NVSP_MSG_TYPE_INIT;
598 init_packet->msg.init_msg.init.min_protocol_ver = nvsp_ver;
599 init_packet->msg.init_msg.init.max_protocol_ver = nvsp_ver;
600 trace_nvsp_send(ndev, init_packet);
601
602 /* Send the init request */
603 ret = vmbus_sendpacket(device->channel, init_packet,
604 sizeof(struct nvsp_message),
605 (unsigned long)init_packet,
606 VM_PKT_DATA_INBAND,
607 VMBUS_DATA_PACKET_FLAG_COMPLETION_REQUESTED);
608
609 if (ret != 0)
610 return ret;
611
612 wait_for_completion(&net_device->channel_init_wait);
613
614 if (init_packet->msg.init_msg.init_complete.status !=
615 NVSP_STAT_SUCCESS)
616 return -EINVAL;
617
618 if (nvsp_ver == NVSP_PROTOCOL_VERSION_1)
619 return 0;
620
621 /* NVSPv2 or later: Send NDIS config */
622 memset(init_packet, 0, sizeof(struct nvsp_message));
623 init_packet->hdr.msg_type = NVSP_MSG2_TYPE_SEND_NDIS_CONFIG;
624 init_packet->msg.v2_msg.send_ndis_config.mtu = ndev->mtu + ETH_HLEN;
625 init_packet->msg.v2_msg.send_ndis_config.capability.ieee8021q = 1;
626
627 if (nvsp_ver >= NVSP_PROTOCOL_VERSION_5) {
628 if (hv_is_isolation_supported())
629 netdev_info(ndev, "SR-IOV not advertised by guests on the host supporting isolation\n");
630 else
631 init_packet->msg.v2_msg.send_ndis_config.capability.sriov = 1;
632
633 /* Teaming bit is needed to receive link speed updates */
634 init_packet->msg.v2_msg.send_ndis_config.capability.teaming = 1;
635 }
636
637 if (nvsp_ver >= NVSP_PROTOCOL_VERSION_61)
638 init_packet->msg.v2_msg.send_ndis_config.capability.rsc = 1;
639
640 trace_nvsp_send(ndev, init_packet);
641
642 ret = vmbus_sendpacket(device->channel, init_packet,
643 sizeof(struct nvsp_message),
644 VMBUS_RQST_ID_NO_RESPONSE,
645 VM_PKT_DATA_INBAND, 0);
646
647 return ret;
648}
649
650static int netvsc_connect_vsp(struct hv_device *device,
651 struct netvsc_device *net_device,
652 const struct netvsc_device_info *device_info)
653{
654 struct net_device *ndev = hv_get_drvdata(device);
655 static const u32 ver_list[] = {
656 NVSP_PROTOCOL_VERSION_1, NVSP_PROTOCOL_VERSION_2,
657 NVSP_PROTOCOL_VERSION_4, NVSP_PROTOCOL_VERSION_5,
658 NVSP_PROTOCOL_VERSION_6, NVSP_PROTOCOL_VERSION_61
659 };
660 struct nvsp_message *init_packet;
661 int ndis_version, i, ret;
662
663 init_packet = &net_device->channel_init_pkt;
664
665 /* Negotiate the latest NVSP protocol supported */
666 for (i = ARRAY_SIZE(ver_list) - 1; i >= 0; i--)
667 if (negotiate_nvsp_ver(device, net_device, init_packet,
668 ver_list[i]) == 0) {
669 net_device->nvsp_version = ver_list[i];
670 break;
671 }
672
673 if (i < 0) {
674 ret = -EPROTO;
675 goto cleanup;
676 }
677
678 if (hv_is_isolation_supported() && net_device->nvsp_version < NVSP_PROTOCOL_VERSION_61) {
679 netdev_err(ndev, "Invalid NVSP version 0x%x (expected >= 0x%x) from the host supporting isolation\n",
680 net_device->nvsp_version, NVSP_PROTOCOL_VERSION_61);
681 ret = -EPROTO;
682 goto cleanup;
683 }
684
685 pr_debug("Negotiated NVSP version:%x\n", net_device->nvsp_version);
686
687 /* Send the ndis version */
688 memset(init_packet, 0, sizeof(struct nvsp_message));
689
690 if (net_device->nvsp_version <= NVSP_PROTOCOL_VERSION_4)
691 ndis_version = 0x00060001;
692 else
693 ndis_version = 0x0006001e;
694
695 init_packet->hdr.msg_type = NVSP_MSG1_TYPE_SEND_NDIS_VER;
696 init_packet->msg.v1_msg.
697 send_ndis_ver.ndis_major_ver =
698 (ndis_version & 0xFFFF0000) >> 16;
699 init_packet->msg.v1_msg.
700 send_ndis_ver.ndis_minor_ver =
701 ndis_version & 0xFFFF;
702
703 trace_nvsp_send(ndev, init_packet);
704
705 /* Send the init request */
706 ret = vmbus_sendpacket(device->channel, init_packet,
707 sizeof(struct nvsp_message),
708 VMBUS_RQST_ID_NO_RESPONSE,
709 VM_PKT_DATA_INBAND, 0);
710 if (ret != 0)
711 goto cleanup;
712
713
714 ret = netvsc_init_buf(device, net_device, device_info);
715
716cleanup:
717 return ret;
718}
719
720/*
721 * netvsc_device_remove - Callback when the root bus device is removed
722 */
723void netvsc_device_remove(struct hv_device *device)
724{
725 struct net_device *ndev = hv_get_drvdata(device);
726 struct net_device_context *net_device_ctx = netdev_priv(ndev);
727 struct netvsc_device *net_device
728 = rtnl_dereference(net_device_ctx->nvdev);
729 int i;
730
731 netvsc_revoke_recv_buf(device, net_device, ndev);
732 netvsc_revoke_send_buf(device, net_device, ndev);
733
734 RCU_INIT_POINTER(net_device_ctx->nvdev, NULL);
735
736 /* Disable NAPI and disassociate its context from the device. */
737 for (i = 0; i < net_device->num_chn; i++) {
738 /* See also vmbus_reset_channel_cb(). */
739 /* only disable enabled NAPI channel */
740 if (i < ndev->real_num_rx_queues) {
741 netif_queue_set_napi(ndev, i, NETDEV_QUEUE_TYPE_TX,
742 NULL);
743 netif_queue_set_napi(ndev, i, NETDEV_QUEUE_TYPE_RX,
744 NULL);
745 napi_disable(&net_device->chan_table[i].napi);
746 }
747
748 netif_napi_del(&net_device->chan_table[i].napi);
749 }
750
751 /*
752 * At this point, no one should be accessing net_device
753 * except in here
754 */
755 netdev_dbg(ndev, "net device safe to remove\n");
756
757 /* Now, we can close the channel safely */
758 vmbus_close(device->channel);
759
760 /* Must do the GPADL teardown after channel is closed */
761 netvsc_teardown_recv_gpadl(device, net_device, ndev);
762 netvsc_teardown_send_gpadl(device, net_device, ndev);
763
764 /* Release all resources */
765 free_netvsc_device_rcu(net_device);
766}
767
768#define RING_AVAIL_PERCENT_HIWATER 20
769#define RING_AVAIL_PERCENT_LOWATER 10
770
771static inline void netvsc_free_send_slot(struct netvsc_device *net_device,
772 u32 index)
773{
774 sync_change_bit(index, net_device->send_section_map);
775}
776
777static void netvsc_send_tx_complete(struct net_device *ndev,
778 struct netvsc_device *net_device,
779 struct vmbus_channel *channel,
780 const struct vmpacket_descriptor *desc,
781 int budget)
782{
783 struct net_device_context *ndev_ctx = netdev_priv(ndev);
784 struct sk_buff *skb;
785 u16 q_idx = 0;
786 int queue_sends;
787 u64 cmd_rqst;
788
789 cmd_rqst = channel->request_addr_callback(channel, desc->trans_id);
790 if (cmd_rqst == VMBUS_RQST_ERROR) {
791 netdev_err(ndev, "Invalid transaction ID %llx\n", desc->trans_id);
792 return;
793 }
794
795 skb = (struct sk_buff *)(unsigned long)cmd_rqst;
796
797 /* Notify the layer above us */
798 if (likely(skb)) {
799 struct hv_netvsc_packet *packet
800 = (struct hv_netvsc_packet *)skb->cb;
801 u32 send_index = packet->send_buf_index;
802 struct netvsc_stats_tx *tx_stats;
803
804 if (send_index != NETVSC_INVALID_INDEX)
805 netvsc_free_send_slot(net_device, send_index);
806 q_idx = packet->q_idx;
807
808 tx_stats = &net_device->chan_table[q_idx].tx_stats;
809
810 u64_stats_update_begin(&tx_stats->syncp);
811 tx_stats->packets += packet->total_packets;
812 tx_stats->bytes += packet->total_bytes;
813 u64_stats_update_end(&tx_stats->syncp);
814
815 netvsc_dma_unmap(ndev_ctx->device_ctx, packet);
816 napi_consume_skb(skb, budget);
817 }
818
819 queue_sends =
820 atomic_dec_return(&net_device->chan_table[q_idx].queue_sends);
821
822 if (unlikely(net_device->destroy)) {
823 if (queue_sends == 0)
824 wake_up(&net_device->wait_drain);
825 } else {
826 struct netdev_queue *txq = netdev_get_tx_queue(ndev, q_idx);
827
828 if (netif_tx_queue_stopped(txq) && !net_device->tx_disable &&
829 (hv_get_avail_to_write_percent(&channel->outbound) >
830 RING_AVAIL_PERCENT_HIWATER || queue_sends < 1)) {
831 netif_tx_wake_queue(txq);
832 ndev_ctx->eth_stats.wake_queue++;
833 }
834 }
835}
836
837static void netvsc_send_completion(struct net_device *ndev,
838 struct netvsc_device *net_device,
839 struct vmbus_channel *incoming_channel,
840 const struct vmpacket_descriptor *desc,
841 int budget)
842{
843 const struct nvsp_message *nvsp_packet;
844 u32 msglen = hv_pkt_datalen(desc);
845 struct nvsp_message *pkt_rqst;
846 u64 cmd_rqst;
847 u32 status;
848
849 /* First check if this is a VMBUS completion without data payload */
850 if (!msglen) {
851 cmd_rqst = incoming_channel->request_addr_callback(incoming_channel,
852 desc->trans_id);
853 if (cmd_rqst == VMBUS_RQST_ERROR) {
854 netdev_err(ndev, "Invalid transaction ID %llx\n", desc->trans_id);
855 return;
856 }
857
858 pkt_rqst = (struct nvsp_message *)(uintptr_t)cmd_rqst;
859 switch (pkt_rqst->hdr.msg_type) {
860 case NVSP_MSG4_TYPE_SWITCH_DATA_PATH:
861 complete(&net_device->channel_init_wait);
862 break;
863
864 default:
865 netdev_err(ndev, "Unexpected VMBUS completion!!\n");
866 }
867 return;
868 }
869
870 /* Ensure packet is big enough to read header fields */
871 if (msglen < sizeof(struct nvsp_message_header)) {
872 netdev_err(ndev, "nvsp_message length too small: %u\n", msglen);
873 return;
874 }
875
876 nvsp_packet = hv_pkt_data(desc);
877 switch (nvsp_packet->hdr.msg_type) {
878 case NVSP_MSG_TYPE_INIT_COMPLETE:
879 if (msglen < sizeof(struct nvsp_message_header) +
880 sizeof(struct nvsp_message_init_complete)) {
881 netdev_err(ndev, "nvsp_msg length too small: %u\n",
882 msglen);
883 return;
884 }
885 break;
886
887 case NVSP_MSG1_TYPE_SEND_RECV_BUF_COMPLETE:
888 if (msglen < sizeof(struct nvsp_message_header) +
889 struct_size_t(struct nvsp_1_message_send_receive_buffer_complete,
890 sections, 1)) {
891 netdev_err(ndev, "nvsp_msg1 length too small: %u\n",
892 msglen);
893 return;
894 }
895 break;
896
897 case NVSP_MSG1_TYPE_SEND_SEND_BUF_COMPLETE:
898 if (msglen < sizeof(struct nvsp_message_header) +
899 sizeof(struct nvsp_1_message_send_send_buffer_complete)) {
900 netdev_err(ndev, "nvsp_msg1 length too small: %u\n",
901 msglen);
902 return;
903 }
904 break;
905
906 case NVSP_MSG5_TYPE_SUBCHANNEL:
907 if (msglen < sizeof(struct nvsp_message_header) +
908 sizeof(struct nvsp_5_subchannel_complete)) {
909 netdev_err(ndev, "nvsp_msg5 length too small: %u\n",
910 msglen);
911 return;
912 }
913 break;
914
915 case NVSP_MSG1_TYPE_SEND_RNDIS_PKT_COMPLETE:
916 if (msglen < sizeof(struct nvsp_message_header) +
917 sizeof(struct nvsp_1_message_send_rndis_packet_complete)) {
918 if (net_ratelimit())
919 netdev_err(ndev, "nvsp_rndis_pkt_complete length too small: %u\n",
920 msglen);
921 return;
922 }
923
924 /* If status indicates an error, output a message so we know
925 * there's a problem. But process the completion anyway so the
926 * resources are released.
927 */
928 status = nvsp_packet->msg.v1_msg.send_rndis_pkt_complete.status;
929 if (status != NVSP_STAT_SUCCESS && net_ratelimit())
930 netdev_err(ndev, "nvsp_rndis_pkt_complete error status: %x\n",
931 status);
932
933 netvsc_send_tx_complete(ndev, net_device, incoming_channel,
934 desc, budget);
935 return;
936
937 default:
938 netdev_err(ndev,
939 "Unknown send completion type %d received!!\n",
940 nvsp_packet->hdr.msg_type);
941 return;
942 }
943
944 /* Copy the response back */
945 memcpy(&net_device->channel_init_pkt, nvsp_packet,
946 sizeof(struct nvsp_message));
947 complete(&net_device->channel_init_wait);
948}
949
950static u32 netvsc_get_next_send_section(struct netvsc_device *net_device)
951{
952 unsigned long *map_addr = net_device->send_section_map;
953 unsigned int i;
954
955 for_each_clear_bit(i, map_addr, net_device->send_section_cnt) {
956 if (sync_test_and_set_bit(i, map_addr) == 0)
957 return i;
958 }
959
960 return NETVSC_INVALID_INDEX;
961}
962
963static void netvsc_copy_to_send_buf(struct netvsc_device *net_device,
964 unsigned int section_index,
965 u32 pend_size,
966 struct hv_netvsc_packet *packet,
967 struct rndis_message *rndis_msg,
968 struct hv_page_buffer *pb,
969 bool xmit_more)
970{
971 char *start = net_device->send_buf;
972 char *dest = start + (section_index * net_device->send_section_size)
973 + pend_size;
974 int i;
975 u32 padding = 0;
976 u32 page_count = packet->cp_partial ? 1 : packet->page_buf_cnt;
977 u32 remain;
978
979 /* Add padding */
980 remain = packet->total_data_buflen & (net_device->pkt_align - 1);
981 if (xmit_more && remain) {
982 padding = net_device->pkt_align - remain;
983 rndis_msg->msg_len += padding;
984 packet->total_data_buflen += padding;
985 }
986
987 for (i = 0; i < page_count; i++) {
988 phys_addr_t paddr = (pb[i].pfn << HV_HYP_PAGE_SHIFT) +
989 pb[i].offset;
990 u32 len = pb[i].len;
991
992 while (len) {
993 struct page *page = phys_to_page(paddr);
994 u32 off = offset_in_page(paddr);
995 u32 chunk = min_t(u32, len, PAGE_SIZE - off);
996 char *src = kmap_local_page(page);
997
998 memcpy(dest, src + off, chunk);
999 kunmap_local(src);
1000 dest += chunk;
1001 paddr += chunk;
1002 len -= chunk;
1003 }
1004 }
1005
1006 if (padding)
1007 memset(dest, 0, padding);
1008}
1009
1010void netvsc_dma_unmap(struct hv_device *hv_dev,
1011 struct hv_netvsc_packet *packet)
1012{
1013 int i;
1014
1015 if (!hv_is_isolation_supported())
1016 return;
1017
1018 if (!packet->dma_range)
1019 return;
1020
1021 for (i = 0; i < packet->page_buf_cnt; i++)
1022 dma_unmap_single(&hv_dev->device, packet->dma_range[i].dma,
1023 packet->dma_range[i].mapping_size,
1024 DMA_TO_DEVICE);
1025
1026 kfree(packet->dma_range);
1027}
1028
1029/* netvsc_dma_map - Map swiotlb bounce buffer with data page of
1030 * packet sent by vmbus_sendpacket_pagebuffer() in the Isolation
1031 * VM.
1032 *
1033 * In isolation VM, netvsc send buffer has been marked visible to
1034 * host and so the data copied to send buffer doesn't need to use
1035 * bounce buffer. The data pages handled by vmbus_sendpacket_pagebuffer()
1036 * may not be copied to send buffer and so these pages need to be
1037 * mapped with swiotlb bounce buffer. netvsc_dma_map() is to do
1038 * that. The pfns in the struct hv_page_buffer need to be converted
1039 * to bounce buffer's pfn. The loop here is necessary because the
1040 * entries in the page buffer array are not necessarily full
1041 * pages of data. Each entry in the array has a separate offset and
1042 * len that may be non-zero, even for entries in the middle of the
1043 * array. And the entries are not physically contiguous. So each
1044 * entry must be individually mapped rather than as a contiguous unit.
1045 * So not use dma_map_sg() here.
1046 */
1047static int netvsc_dma_map(struct hv_device *hv_dev,
1048 struct hv_netvsc_packet *packet,
1049 struct hv_page_buffer *pb)
1050{
1051 u32 page_count = packet->page_buf_cnt;
1052 dma_addr_t dma;
1053 int i;
1054
1055 if (!hv_is_isolation_supported())
1056 return 0;
1057
1058 packet->dma_range = kzalloc_objs(*packet->dma_range, page_count,
1059 GFP_ATOMIC);
1060 if (!packet->dma_range)
1061 return -ENOMEM;
1062
1063 for (i = 0; i < page_count; i++) {
1064 char *src = phys_to_virt((pb[i].pfn << HV_HYP_PAGE_SHIFT)
1065 + pb[i].offset);
1066 u32 len = pb[i].len;
1067
1068 dma = dma_map_single(&hv_dev->device, src, len,
1069 DMA_TO_DEVICE);
1070 if (dma_mapping_error(&hv_dev->device, dma)) {
1071 kfree(packet->dma_range);
1072 return -ENOMEM;
1073 }
1074
1075 /* pb[].offset and pb[].len are not changed during dma mapping
1076 * and so not reassign.
1077 */
1078 packet->dma_range[i].dma = dma;
1079 packet->dma_range[i].mapping_size = len;
1080 pb[i].pfn = dma >> HV_HYP_PAGE_SHIFT;
1081 }
1082
1083 return 0;
1084}
1085
1086/* Build an "array" of mpb entries describing the data to be transferred
1087 * over VMBus. After the desc header fields, each "array" entry is variable
1088 * size, and each entry starts after the end of the previous entry. The
1089 * "offset" and "len" fields for each entry imply the size of the entry.
1090 *
1091 * The pfns are in HV_HYP_PAGE_SIZE, because all communication with Hyper-V
1092 * uses that granularity, even if the system page size of the guest is larger.
1093 * Each entry in the input "pb" array must describe a contiguous range of
1094 * guest physical memory so that the pfns are sequential if the range crosses
1095 * a page boundary. The offset field must be < HV_HYP_PAGE_SIZE.
1096 */
1097static inline void netvsc_build_mpb_array(struct hv_page_buffer *pb,
1098 u32 page_buffer_count,
1099 struct vmbus_packet_mpb_array *desc,
1100 u32 *desc_size)
1101{
1102 struct hv_mpb_array *mpb_entry = &desc->range;
1103 int i, j;
1104
1105 for (i = 0; i < page_buffer_count; i++) {
1106 u32 offset = pb[i].offset;
1107 u32 len = pb[i].len;
1108
1109 mpb_entry->offset = offset;
1110 mpb_entry->len = len;
1111
1112 for (j = 0; j < HVPFN_UP(offset + len); j++)
1113 mpb_entry->pfn_array[j] = pb[i].pfn + j;
1114
1115 mpb_entry = (struct hv_mpb_array *)&mpb_entry->pfn_array[j];
1116 }
1117
1118 desc->rangecount = page_buffer_count;
1119 *desc_size = (char *)mpb_entry - (char *)desc;
1120}
1121
1122static inline int netvsc_send_pkt(
1123 struct hv_device *device,
1124 struct hv_netvsc_packet *packet,
1125 struct netvsc_device *net_device,
1126 struct hv_page_buffer *pb,
1127 struct sk_buff *skb)
1128{
1129 struct nvsp_message nvmsg;
1130 struct nvsp_1_message_send_rndis_packet *rpkt =
1131 &nvmsg.msg.v1_msg.send_rndis_pkt;
1132 struct netvsc_channel * const nvchan =
1133 &net_device->chan_table[packet->q_idx];
1134 struct vmbus_channel *out_channel = nvchan->channel;
1135 struct net_device *ndev = hv_get_drvdata(device);
1136 struct net_device_context *ndev_ctx = netdev_priv(ndev);
1137 struct netdev_queue *txq = netdev_get_tx_queue(ndev, packet->q_idx);
1138 u64 req_id;
1139 int ret;
1140 u32 ring_avail = hv_get_avail_to_write_percent(&out_channel->outbound);
1141
1142 memset(&nvmsg, 0, sizeof(struct nvsp_message));
1143 nvmsg.hdr.msg_type = NVSP_MSG1_TYPE_SEND_RNDIS_PKT;
1144 if (skb)
1145 rpkt->channel_type = 0; /* 0 is RMC_DATA */
1146 else
1147 rpkt->channel_type = 1; /* 1 is RMC_CONTROL */
1148
1149 rpkt->send_buf_section_index = packet->send_buf_index;
1150 if (packet->send_buf_index == NETVSC_INVALID_INDEX)
1151 rpkt->send_buf_section_size = 0;
1152 else
1153 rpkt->send_buf_section_size = packet->total_data_buflen;
1154
1155 req_id = (ulong)skb;
1156
1157 if (out_channel->rescind)
1158 return -ENODEV;
1159
1160 trace_nvsp_send_pkt(ndev, out_channel, rpkt);
1161
1162 packet->dma_range = NULL;
1163 if (packet->page_buf_cnt) {
1164 struct vmbus_channel_packet_page_buffer desc;
1165 u32 desc_size;
1166
1167 if (packet->cp_partial)
1168 pb++;
1169
1170 ret = netvsc_dma_map(ndev_ctx->device_ctx, packet, pb);
1171 if (ret) {
1172 ret = -EAGAIN;
1173 goto exit;
1174 }
1175
1176 netvsc_build_mpb_array(pb, packet->page_buf_cnt,
1177 (struct vmbus_packet_mpb_array *)&desc,
1178 &desc_size);
1179 ret = vmbus_sendpacket_mpb_desc(out_channel,
1180 (struct vmbus_packet_mpb_array *)&desc,
1181 desc_size, &nvmsg, sizeof(nvmsg), req_id);
1182 if (ret)
1183 netvsc_dma_unmap(ndev_ctx->device_ctx, packet);
1184 } else {
1185 ret = vmbus_sendpacket(out_channel,
1186 &nvmsg, sizeof(nvmsg),
1187 req_id, VM_PKT_DATA_INBAND,
1188 VMBUS_DATA_PACKET_FLAG_COMPLETION_REQUESTED);
1189 }
1190
1191exit:
1192 if (ret == 0) {
1193 atomic_inc_return(&nvchan->queue_sends);
1194
1195 if (ring_avail < RING_AVAIL_PERCENT_LOWATER) {
1196 netif_tx_stop_queue(txq);
1197 ndev_ctx->eth_stats.stop_queue++;
1198 }
1199 } else if (ret == -EAGAIN) {
1200 netif_tx_stop_queue(txq);
1201 ndev_ctx->eth_stats.stop_queue++;
1202 } else {
1203 netdev_err(ndev,
1204 "Unable to send packet pages %u len %u, ret %d\n",
1205 packet->page_buf_cnt, packet->total_data_buflen,
1206 ret);
1207 }
1208
1209 if (netif_tx_queue_stopped(txq) &&
1210 atomic_read(&nvchan->queue_sends) < 1 &&
1211 !net_device->tx_disable) {
1212 netif_tx_wake_queue(txq);
1213 ndev_ctx->eth_stats.wake_queue++;
1214 if (ret == -EAGAIN)
1215 ret = -ENOSPC;
1216 }
1217
1218 return ret;
1219}
1220
1221/* Move packet out of multi send data (msd), and clear msd */
1222static inline void move_pkt_msd(struct hv_netvsc_packet **msd_send,
1223 struct sk_buff **msd_skb,
1224 struct multi_send_data *msdp)
1225{
1226 *msd_skb = msdp->skb;
1227 *msd_send = msdp->pkt;
1228 msdp->skb = NULL;
1229 msdp->pkt = NULL;
1230 msdp->count = 0;
1231}
1232
1233/* RCU already held by caller */
1234/* Batching/bouncing logic is designed to attempt to optimize
1235 * performance.
1236 *
1237 * For small, non-LSO packets we copy the packet to a send buffer
1238 * which is pre-registered with the Hyper-V side. This enables the
1239 * hypervisor to avoid remapping the aperture to access the packet
1240 * descriptor and data.
1241 *
1242 * If we already started using a buffer and the netdev is transmitting
1243 * a burst of packets, keep on copying into the buffer until it is
1244 * full or we are done collecting a burst. If there is an existing
1245 * buffer with space for the RNDIS descriptor but not the packet, copy
1246 * the RNDIS descriptor to the buffer, keeping the packet in place.
1247 *
1248 * If we do batching and send more than one packet using a single
1249 * NetVSC message, free the SKBs of the packets copied, except for the
1250 * last packet. This is done to streamline the handling of the case
1251 * where the last packet only had the RNDIS descriptor copied to the
1252 * send buffer, with the data pointers included in the NetVSC message.
1253 */
1254int netvsc_send(struct net_device *ndev,
1255 struct hv_netvsc_packet *packet,
1256 struct rndis_message *rndis_msg,
1257 struct hv_page_buffer *pb,
1258 struct sk_buff *skb,
1259 bool xdp_tx)
1260{
1261 struct net_device_context *ndev_ctx = netdev_priv(ndev);
1262 struct netvsc_device *net_device
1263 = rcu_dereference_bh(ndev_ctx->nvdev);
1264 struct hv_device *device = ndev_ctx->device_ctx;
1265 int ret = 0;
1266 struct netvsc_channel *nvchan;
1267 u32 pktlen = packet->total_data_buflen, msd_len = 0;
1268 unsigned int section_index = NETVSC_INVALID_INDEX;
1269 struct multi_send_data *msdp;
1270 struct hv_netvsc_packet *msd_send = NULL, *cur_send = NULL;
1271 struct sk_buff *msd_skb = NULL;
1272 bool try_batch, xmit_more;
1273
1274 /* If device is rescinded, return error and packet will get dropped. */
1275 if (unlikely(!net_device || net_device->destroy))
1276 return -ENODEV;
1277
1278 nvchan = &net_device->chan_table[packet->q_idx];
1279 packet->send_buf_index = NETVSC_INVALID_INDEX;
1280 packet->cp_partial = false;
1281
1282 /* Send a control message or XDP packet directly without accessing
1283 * msd (Multi-Send Data) field which may be changed during data packet
1284 * processing.
1285 */
1286 if (!skb || xdp_tx)
1287 return netvsc_send_pkt(device, packet, net_device, pb, skb);
1288
1289 /* batch packets in send buffer if possible */
1290 msdp = &nvchan->msd;
1291 if (msdp->pkt)
1292 msd_len = msdp->pkt->total_data_buflen;
1293
1294 try_batch = msd_len > 0 && msdp->count < net_device->max_pkt;
1295 if (try_batch && msd_len + pktlen + net_device->pkt_align <
1296 net_device->send_section_size) {
1297 section_index = msdp->pkt->send_buf_index;
1298
1299 } else if (try_batch && msd_len + packet->rmsg_size <
1300 net_device->send_section_size) {
1301 section_index = msdp->pkt->send_buf_index;
1302 packet->cp_partial = true;
1303
1304 } else if (pktlen + net_device->pkt_align <
1305 net_device->send_section_size) {
1306 section_index = netvsc_get_next_send_section(net_device);
1307 if (unlikely(section_index == NETVSC_INVALID_INDEX)) {
1308 ++ndev_ctx->eth_stats.tx_send_full;
1309 } else {
1310 move_pkt_msd(&msd_send, &msd_skb, msdp);
1311 msd_len = 0;
1312 }
1313 }
1314
1315 /* Keep aggregating only if stack says more data is coming
1316 * and not doing mixed modes send and not flow blocked
1317 */
1318 xmit_more = netdev_xmit_more() &&
1319 !packet->cp_partial &&
1320 !netif_xmit_stopped(netdev_get_tx_queue(ndev, packet->q_idx));
1321
1322 if (section_index != NETVSC_INVALID_INDEX) {
1323 netvsc_copy_to_send_buf(net_device,
1324 section_index, msd_len,
1325 packet, rndis_msg, pb, xmit_more);
1326
1327 packet->send_buf_index = section_index;
1328
1329 if (packet->cp_partial) {
1330 packet->page_buf_cnt--;
1331 packet->total_data_buflen = msd_len + packet->rmsg_size;
1332 } else {
1333 packet->page_buf_cnt = 0;
1334 packet->total_data_buflen += msd_len;
1335 }
1336
1337 if (msdp->pkt) {
1338 packet->total_packets += msdp->pkt->total_packets;
1339 packet->total_bytes += msdp->pkt->total_bytes;
1340 }
1341
1342 if (msdp->skb)
1343 dev_consume_skb_any(msdp->skb);
1344
1345 if (xmit_more) {
1346 msdp->skb = skb;
1347 msdp->pkt = packet;
1348 msdp->count++;
1349 } else {
1350 cur_send = packet;
1351 msdp->skb = NULL;
1352 msdp->pkt = NULL;
1353 msdp->count = 0;
1354 }
1355 } else {
1356 move_pkt_msd(&msd_send, &msd_skb, msdp);
1357 cur_send = packet;
1358 }
1359
1360 if (msd_send) {
1361 int m_ret = netvsc_send_pkt(device, msd_send, net_device,
1362 NULL, msd_skb);
1363
1364 if (m_ret != 0) {
1365 netvsc_free_send_slot(net_device,
1366 msd_send->send_buf_index);
1367 dev_kfree_skb_any(msd_skb);
1368 }
1369 }
1370
1371 if (cur_send)
1372 ret = netvsc_send_pkt(device, cur_send, net_device, pb, skb);
1373
1374 if (ret != 0 && section_index != NETVSC_INVALID_INDEX)
1375 netvsc_free_send_slot(net_device, section_index);
1376
1377 return ret;
1378}
1379
1380/* Send pending recv completions */
1381static int send_recv_completions(struct net_device *ndev,
1382 struct netvsc_device *nvdev,
1383 struct netvsc_channel *nvchan)
1384{
1385 struct multi_recv_comp *mrc = &nvchan->mrc;
1386 struct recv_comp_msg {
1387 struct nvsp_message_header hdr;
1388 u32 status;
1389 } __packed;
1390 struct recv_comp_msg msg = {
1391 .hdr.msg_type = NVSP_MSG1_TYPE_SEND_RNDIS_PKT_COMPLETE,
1392 };
1393 int ret;
1394
1395 while (mrc->first != mrc->next) {
1396 const struct recv_comp_data *rcd
1397 = mrc->slots + mrc->first;
1398
1399 msg.status = rcd->status;
1400 ret = vmbus_sendpacket(nvchan->channel, &msg, sizeof(msg),
1401 rcd->tid, VM_PKT_COMP, 0);
1402 if (unlikely(ret)) {
1403 struct net_device_context *ndev_ctx = netdev_priv(ndev);
1404
1405 ++ndev_ctx->eth_stats.rx_comp_busy;
1406 return ret;
1407 }
1408
1409 if (++mrc->first == nvdev->recv_completion_cnt)
1410 mrc->first = 0;
1411 }
1412
1413 /* receive completion ring has been emptied */
1414 if (unlikely(nvdev->destroy))
1415 wake_up(&nvdev->wait_drain);
1416
1417 return 0;
1418}
1419
1420/* Count how many receive completions are outstanding */
1421static void recv_comp_slot_avail(const struct netvsc_device *nvdev,
1422 const struct multi_recv_comp *mrc,
1423 u32 *filled, u32 *avail)
1424{
1425 u32 count = nvdev->recv_completion_cnt;
1426
1427 if (mrc->next >= mrc->first)
1428 *filled = mrc->next - mrc->first;
1429 else
1430 *filled = (count - mrc->first) + mrc->next;
1431
1432 *avail = count - *filled - 1;
1433}
1434
1435/* Add receive complete to ring to send to host. */
1436static void enq_receive_complete(struct net_device *ndev,
1437 struct netvsc_device *nvdev, u16 q_idx,
1438 u64 tid, u32 status)
1439{
1440 struct netvsc_channel *nvchan = &nvdev->chan_table[q_idx];
1441 struct multi_recv_comp *mrc = &nvchan->mrc;
1442 struct recv_comp_data *rcd;
1443 u32 filled, avail;
1444
1445 recv_comp_slot_avail(nvdev, mrc, &filled, &avail);
1446
1447 if (unlikely(filled > NAPI_POLL_WEIGHT)) {
1448 send_recv_completions(ndev, nvdev, nvchan);
1449 recv_comp_slot_avail(nvdev, mrc, &filled, &avail);
1450 }
1451
1452 if (unlikely(!avail)) {
1453 netdev_err(ndev, "Recv_comp full buf q:%hd, tid:%llx\n",
1454 q_idx, tid);
1455 return;
1456 }
1457
1458 rcd = mrc->slots + mrc->next;
1459 rcd->tid = tid;
1460 rcd->status = status;
1461
1462 if (++mrc->next == nvdev->recv_completion_cnt)
1463 mrc->next = 0;
1464}
1465
1466static int netvsc_receive(struct net_device *ndev,
1467 struct netvsc_device *net_device,
1468 struct netvsc_channel *nvchan,
1469 const struct vmpacket_descriptor *desc)
1470{
1471 struct net_device_context *net_device_ctx = netdev_priv(ndev);
1472 struct vmbus_channel *channel = nvchan->channel;
1473 const struct vmtransfer_page_packet_header *vmxferpage_packet
1474 = container_of(desc, const struct vmtransfer_page_packet_header, d);
1475 const struct nvsp_message *nvsp = hv_pkt_data(desc);
1476 u32 msglen = hv_pkt_datalen(desc);
1477 u16 q_idx = channel->offermsg.offer.sub_channel_index;
1478 char *recv_buf = net_device->recv_buf;
1479 u32 status = NVSP_STAT_SUCCESS;
1480 int i;
1481 int count = 0;
1482
1483 /* Ensure packet is big enough to read header fields */
1484 if (msglen < sizeof(struct nvsp_message_header)) {
1485 netif_err(net_device_ctx, rx_err, ndev,
1486 "invalid nvsp header, length too small: %u\n",
1487 msglen);
1488 return 0;
1489 }
1490
1491 /* Make sure this is a valid nvsp packet */
1492 if (unlikely(nvsp->hdr.msg_type != NVSP_MSG1_TYPE_SEND_RNDIS_PKT)) {
1493 netif_err(net_device_ctx, rx_err, ndev,
1494 "Unknown nvsp packet type received %u\n",
1495 nvsp->hdr.msg_type);
1496 return 0;
1497 }
1498
1499 /* Validate xfer page pkt header */
1500 if ((desc->offset8 << 3) < sizeof(struct vmtransfer_page_packet_header)) {
1501 netif_err(net_device_ctx, rx_err, ndev,
1502 "Invalid xfer page pkt, offset too small: %u\n",
1503 desc->offset8 << 3);
1504 return 0;
1505 }
1506
1507 if (unlikely(vmxferpage_packet->xfer_pageset_id != NETVSC_RECEIVE_BUFFER_ID)) {
1508 netif_err(net_device_ctx, rx_err, ndev,
1509 "Invalid xfer page set id - expecting %x got %x\n",
1510 NETVSC_RECEIVE_BUFFER_ID,
1511 vmxferpage_packet->xfer_pageset_id);
1512 return 0;
1513 }
1514
1515 count = vmxferpage_packet->range_cnt;
1516
1517 /* Check count for a valid value */
1518 if (NETVSC_XFER_HEADER_SIZE(count) > desc->offset8 << 3) {
1519 netif_err(net_device_ctx, rx_err, ndev,
1520 "Range count is not valid: %d\n",
1521 count);
1522 return 0;
1523 }
1524
1525 /* Each range represents 1 RNDIS pkt that contains 1 ethernet frame */
1526 for (i = 0; i < count; i++) {
1527 u32 offset = vmxferpage_packet->ranges[i].byte_offset;
1528 u32 buflen = vmxferpage_packet->ranges[i].byte_count;
1529 void *data;
1530 int ret;
1531
1532 if (unlikely(offset > net_device->recv_buf_size ||
1533 buflen > net_device->recv_buf_size - offset)) {
1534 nvchan->rsc.cnt = 0;
1535 status = NVSP_STAT_FAIL;
1536 netif_err(net_device_ctx, rx_err, ndev,
1537 "Packet offset:%u + len:%u too big\n",
1538 offset, buflen);
1539
1540 continue;
1541 }
1542
1543 /* We're going to copy (sections of) the packet into nvchan->recv_buf;
1544 * make sure that nvchan->recv_buf is large enough to hold the packet.
1545 */
1546 if (unlikely(buflen > net_device->recv_section_size)) {
1547 nvchan->rsc.cnt = 0;
1548 status = NVSP_STAT_FAIL;
1549 netif_err(net_device_ctx, rx_err, ndev,
1550 "Packet too big: buflen=%u recv_section_size=%u\n",
1551 buflen, net_device->recv_section_size);
1552
1553 continue;
1554 }
1555
1556 data = recv_buf + offset;
1557
1558 nvchan->rsc.is_last = (i == count - 1);
1559
1560 trace_rndis_recv(ndev, q_idx, data);
1561
1562 /* Pass it to the upper layer */
1563 ret = rndis_filter_receive(ndev, net_device,
1564 nvchan, data, buflen);
1565
1566 if (unlikely(ret != NVSP_STAT_SUCCESS)) {
1567 /* Drop incomplete packet */
1568 nvchan->rsc.cnt = 0;
1569 status = NVSP_STAT_FAIL;
1570 }
1571 }
1572
1573 enq_receive_complete(ndev, net_device, q_idx,
1574 vmxferpage_packet->d.trans_id, status);
1575
1576 return count;
1577}
1578
1579static void netvsc_send_table(struct net_device *ndev,
1580 struct netvsc_device *nvscdev,
1581 const struct nvsp_message *nvmsg,
1582 u32 msglen)
1583{
1584 struct net_device_context *net_device_ctx = netdev_priv(ndev);
1585 u32 count, offset, *tab;
1586 int i;
1587
1588 /* Ensure packet is big enough to read send_table fields */
1589 if (msglen < sizeof(struct nvsp_message_header) +
1590 sizeof(struct nvsp_5_send_indirect_table)) {
1591 netdev_err(ndev, "nvsp_v5_msg length too small: %u\n", msglen);
1592 return;
1593 }
1594
1595 count = nvmsg->msg.v5_msg.send_table.count;
1596 offset = nvmsg->msg.v5_msg.send_table.offset;
1597
1598 if (count != VRSS_SEND_TAB_SIZE) {
1599 netdev_err(ndev, "Received wrong send-table size:%u\n", count);
1600 return;
1601 }
1602
1603 /* If negotiated version <= NVSP_PROTOCOL_VERSION_6, the offset may be
1604 * wrong due to a host bug. So fix the offset here.
1605 */
1606 if (nvscdev->nvsp_version <= NVSP_PROTOCOL_VERSION_6 &&
1607 msglen >= sizeof(struct nvsp_message_header) +
1608 sizeof(union nvsp_6_message_uber) + count * sizeof(u32))
1609 offset = sizeof(struct nvsp_message_header) +
1610 sizeof(union nvsp_6_message_uber);
1611
1612 /* Boundary check for all versions */
1613 if (msglen < count * sizeof(u32) || offset > msglen - count * sizeof(u32)) {
1614 netdev_err(ndev, "Received send-table offset too big:%u\n",
1615 offset);
1616 return;
1617 }
1618
1619 tab = (void *)nvmsg + offset;
1620
1621 for (i = 0; i < count; i++)
1622 net_device_ctx->tx_table[i] = tab[i];
1623}
1624
1625static void netvsc_send_vf(struct net_device *ndev,
1626 const struct nvsp_message *nvmsg,
1627 u32 msglen)
1628{
1629 struct net_device_context *net_device_ctx = netdev_priv(ndev);
1630
1631 /* Ensure packet is big enough to read its fields */
1632 if (msglen < sizeof(struct nvsp_message_header) +
1633 sizeof(struct nvsp_4_send_vf_association)) {
1634 netdev_err(ndev, "nvsp_v4_msg length too small: %u\n", msglen);
1635 return;
1636 }
1637
1638 net_device_ctx->vf_alloc = nvmsg->msg.v4_msg.vf_assoc.allocated;
1639 net_device_ctx->vf_serial = nvmsg->msg.v4_msg.vf_assoc.serial;
1640
1641 if (net_device_ctx->vf_alloc)
1642 complete(&net_device_ctx->vf_add);
1643
1644 netdev_info(ndev, "VF slot %u %s\n",
1645 net_device_ctx->vf_serial,
1646 net_device_ctx->vf_alloc ? "added" : "removed");
1647}
1648
1649static void netvsc_receive_inband(struct net_device *ndev,
1650 struct netvsc_device *nvscdev,
1651 const struct vmpacket_descriptor *desc)
1652{
1653 const struct nvsp_message *nvmsg = hv_pkt_data(desc);
1654 u32 msglen = hv_pkt_datalen(desc);
1655
1656 /* Ensure packet is big enough to read header fields */
1657 if (msglen < sizeof(struct nvsp_message_header)) {
1658 netdev_err(ndev, "inband nvsp_message length too small: %u\n", msglen);
1659 return;
1660 }
1661
1662 switch (nvmsg->hdr.msg_type) {
1663 case NVSP_MSG5_TYPE_SEND_INDIRECTION_TABLE:
1664 netvsc_send_table(ndev, nvscdev, nvmsg, msglen);
1665 break;
1666
1667 case NVSP_MSG4_TYPE_SEND_VF_ASSOCIATION:
1668 if (hv_is_isolation_supported())
1669 netdev_err(ndev, "Ignore VF_ASSOCIATION msg from the host supporting isolation\n");
1670 else
1671 netvsc_send_vf(ndev, nvmsg, msglen);
1672 break;
1673 }
1674}
1675
1676static int netvsc_process_raw_pkt(struct hv_device *device,
1677 struct netvsc_channel *nvchan,
1678 struct netvsc_device *net_device,
1679 struct net_device *ndev,
1680 const struct vmpacket_descriptor *desc,
1681 int budget)
1682{
1683 struct vmbus_channel *channel = nvchan->channel;
1684 const struct nvsp_message *nvmsg = hv_pkt_data(desc);
1685
1686 trace_nvsp_recv(ndev, channel, nvmsg);
1687
1688 switch (desc->type) {
1689 case VM_PKT_COMP:
1690 netvsc_send_completion(ndev, net_device, channel, desc, budget);
1691 break;
1692
1693 case VM_PKT_DATA_USING_XFER_PAGES:
1694 return netvsc_receive(ndev, net_device, nvchan, desc);
1695
1696 case VM_PKT_DATA_INBAND:
1697 netvsc_receive_inband(ndev, net_device, desc);
1698 break;
1699
1700 default:
1701 netdev_err(ndev, "unhandled packet type %d, tid %llx\n",
1702 desc->type, desc->trans_id);
1703 break;
1704 }
1705
1706 return 0;
1707}
1708
1709static struct hv_device *netvsc_channel_to_device(struct vmbus_channel *channel)
1710{
1711 struct vmbus_channel *primary = channel->primary_channel;
1712
1713 return primary ? primary->device_obj : channel->device_obj;
1714}
1715
1716/* Network processing softirq
1717 * Process data in incoming ring buffer from host
1718 * Stops when ring is empty or budget is met or exceeded.
1719 */
1720int netvsc_poll(struct napi_struct *napi, int budget)
1721{
1722 struct netvsc_channel *nvchan
1723 = container_of(napi, struct netvsc_channel, napi);
1724 struct netvsc_device *net_device = nvchan->net_device;
1725 struct vmbus_channel *channel = nvchan->channel;
1726 struct hv_device *device = netvsc_channel_to_device(channel);
1727 struct net_device *ndev = hv_get_drvdata(device);
1728 int work_done = 0;
1729 int ret;
1730
1731 /* If starting a new interval */
1732 if (!nvchan->desc)
1733 nvchan->desc = hv_pkt_iter_first(channel);
1734
1735 nvchan->xdp_flush = false;
1736
1737 while (nvchan->desc && work_done < budget) {
1738 work_done += netvsc_process_raw_pkt(device, nvchan, net_device,
1739 ndev, nvchan->desc, budget);
1740 nvchan->desc = hv_pkt_iter_next(channel, nvchan->desc);
1741 }
1742
1743 if (nvchan->xdp_flush)
1744 xdp_do_flush();
1745
1746 /* Send any pending receive completions */
1747 ret = send_recv_completions(ndev, net_device, nvchan);
1748
1749 /* If it did not exhaust NAPI budget this time
1750 * and not doing busy poll
1751 * then re-enable host interrupts
1752 * and reschedule if ring is not empty
1753 * or sending receive completion failed.
1754 */
1755 if (work_done < budget &&
1756 napi_complete_done(napi, work_done) &&
1757 (ret || hv_end_read(&channel->inbound)) &&
1758 napi_schedule_prep(napi)) {
1759 hv_begin_read(&channel->inbound);
1760 __napi_schedule(napi);
1761 }
1762
1763 /* Driver may overshoot since multiple packets per descriptor */
1764 return min(work_done, budget);
1765}
1766
1767/* Call back when data is available in host ring buffer.
1768 * Processing is deferred until network softirq (NAPI)
1769 */
1770void netvsc_channel_cb(void *context)
1771{
1772 struct netvsc_channel *nvchan = context;
1773 struct vmbus_channel *channel = nvchan->channel;
1774 struct hv_ring_buffer_info *rbi = &channel->inbound;
1775
1776 /* preload first vmpacket descriptor */
1777 prefetch(hv_get_ring_buffer(rbi) + rbi->priv_read_index);
1778
1779 if (napi_schedule_prep(&nvchan->napi)) {
1780 /* disable interrupts from host */
1781 hv_begin_read(rbi);
1782
1783 __napi_schedule_irqoff(&nvchan->napi);
1784 }
1785}
1786
1787/*
1788 * netvsc_device_add - Callback when the device belonging to this
1789 * driver is added
1790 */
1791struct netvsc_device *netvsc_device_add(struct hv_device *device,
1792 const struct netvsc_device_info *device_info)
1793{
1794 int i, ret = 0;
1795 struct netvsc_device *net_device;
1796 struct net_device *ndev = hv_get_drvdata(device);
1797 struct net_device_context *net_device_ctx = netdev_priv(ndev);
1798
1799 net_device = alloc_net_device();
1800 if (!net_device)
1801 return ERR_PTR(-ENOMEM);
1802
1803 for (i = 0; i < VRSS_SEND_TAB_SIZE; i++)
1804 net_device_ctx->tx_table[i] = 0;
1805
1806 /* Because the device uses NAPI, all the interrupt batching and
1807 * control is done via Net softirq, not the channel handling
1808 */
1809 set_channel_read_mode(device->channel, HV_CALL_ISR);
1810
1811 /* If we're reopening the device we may have multiple queues, fill the
1812 * chn_table with the default channel to use it before subchannels are
1813 * opened.
1814 * Initialize the channel state before we open;
1815 * we can be interrupted as soon as we open the channel.
1816 */
1817
1818 for (i = 0; i < VRSS_CHANNEL_MAX; i++) {
1819 struct netvsc_channel *nvchan = &net_device->chan_table[i];
1820
1821 nvchan->channel = device->channel;
1822 nvchan->net_device = net_device;
1823 u64_stats_init(&nvchan->tx_stats.syncp);
1824 u64_stats_init(&nvchan->rx_stats.syncp);
1825
1826 ret = xdp_rxq_info_reg(&nvchan->xdp_rxq, ndev, i, 0);
1827
1828 if (ret) {
1829 netdev_err(ndev, "xdp_rxq_info_reg fail: %d\n", ret);
1830 goto cleanup2;
1831 }
1832
1833 ret = xdp_rxq_info_reg_mem_model(&nvchan->xdp_rxq,
1834 MEM_TYPE_PAGE_SHARED, NULL);
1835
1836 if (ret) {
1837 netdev_err(ndev, "xdp reg_mem_model fail: %d\n", ret);
1838 goto cleanup2;
1839 }
1840 }
1841
1842 /* Enable NAPI handler before init callbacks */
1843 netif_napi_add(ndev, &net_device->chan_table[0].napi, netvsc_poll);
1844 napi_enable(&net_device->chan_table[0].napi);
1845 netif_queue_set_napi(ndev, 0, NETDEV_QUEUE_TYPE_RX,
1846 &net_device->chan_table[0].napi);
1847 netif_queue_set_napi(ndev, 0, NETDEV_QUEUE_TYPE_TX,
1848 &net_device->chan_table[0].napi);
1849
1850 /* Open the channel */
1851 device->channel->next_request_id_callback = vmbus_next_request_id;
1852 device->channel->request_addr_callback = vmbus_request_addr;
1853 device->channel->rqstor_size = netvsc_rqstor_size(netvsc_ring_bytes);
1854 device->channel->max_pkt_size = NETVSC_MAX_PKT_SIZE;
1855
1856 ret = vmbus_open(device->channel, netvsc_ring_bytes,
1857 netvsc_ring_bytes, NULL, 0,
1858 netvsc_channel_cb, net_device->chan_table);
1859
1860 if (ret != 0) {
1861 netdev_err(ndev, "unable to open channel: %d\n", ret);
1862 goto cleanup;
1863 }
1864
1865 /* Channel is opened */
1866 netdev_dbg(ndev, "hv_netvsc channel opened successfully\n");
1867
1868 /* Connect with the NetVsp */
1869 ret = netvsc_connect_vsp(device, net_device, device_info);
1870 if (ret != 0) {
1871 netdev_err(ndev,
1872 "unable to connect to NetVSP - %d\n", ret);
1873 goto close;
1874 }
1875
1876 /* Writing nvdev pointer unlocks netvsc_send(), make sure chn_table is
1877 * populated.
1878 */
1879 rcu_assign_pointer(net_device_ctx->nvdev, net_device);
1880
1881 return net_device;
1882
1883close:
1884 RCU_INIT_POINTER(net_device_ctx->nvdev, NULL);
1885
1886 /* Now, we can close the channel safely */
1887 vmbus_close(device->channel);
1888
1889cleanup:
1890 netif_queue_set_napi(ndev, 0, NETDEV_QUEUE_TYPE_TX, NULL);
1891 netif_queue_set_napi(ndev, 0, NETDEV_QUEUE_TYPE_RX, NULL);
1892 napi_disable(&net_device->chan_table[0].napi);
1893 netif_napi_del(&net_device->chan_table[0].napi);
1894
1895cleanup2:
1896 /*
1897 * net_device was never published, so we don't need to wait for an
1898 * RCU grace period -- call the free routine synchronously.
1899 */
1900 __free_netvsc_device(net_device);
1901
1902 return ERR_PTR(ret);
1903}