projects/linux·blob master · refs

636 lines · 16.2 KiB

1/* SPDX-License-Identifier: GPL-2.0 */
2#include <linux/kernel.h>
3#include <linux/slab.h>
4#include <net/act_api.h>
5#include <net/flow_offload.h>
6#include <linux/rtnetlink.h>
7#include <linux/mutex.h>
8#include <linux/rhashtable.h>
9
10struct flow_rule *flow_rule_alloc(unsigned int num_actions)
11{
12 struct flow_rule *rule;
13 int i;
14
15 rule = kzalloc_flex(*rule, action.entries, num_actions);
16 if (!rule)
17 return NULL;
18
19 rule->action.num_entries = num_actions;
20 /* Pre-fill each action hw_stats with DONT_CARE.
21 * Caller can override this if it wants stats for a given action.
22 */
23 for (i = 0; i < num_actions; i++)
24 rule->action.entries[i].hw_stats = FLOW_ACTION_HW_STATS_DONT_CARE;
25
26 return rule;
27}
28EXPORT_SYMBOL(flow_rule_alloc);
29
30struct flow_offload_action *offload_action_alloc(unsigned int num_actions)
31{
32 struct flow_offload_action *fl_action;
33 int i;
34
35 fl_action = kzalloc_flex(*fl_action, action.entries, num_actions);
36 if (!fl_action)
37 return NULL;
38
39 fl_action->action.num_entries = num_actions;
40 /* Pre-fill each action hw_stats with DONT_CARE.
41 * Caller can override this if it wants stats for a given action.
42 */
43 for (i = 0; i < num_actions; i++)
44 fl_action->action.entries[i].hw_stats = FLOW_ACTION_HW_STATS_DONT_CARE;
45
46 return fl_action;
47}
48
49#define FLOW_DISSECTOR_MATCH(__rule, __type, __out) \
50 const struct flow_match *__m = &(__rule)->match; \
51 struct flow_dissector *__d = (__m)->dissector; \
52 \
53 (__out)->key = skb_flow_dissector_target(__d, __type, (__m)->key); \
54 (__out)->mask = skb_flow_dissector_target(__d, __type, (__m)->mask); \
55
56void flow_rule_match_meta(const struct flow_rule *rule,
57 struct flow_match_meta *out)
58{
59 FLOW_DISSECTOR_MATCH(rule, FLOW_DISSECTOR_KEY_META, out);
60}
61EXPORT_SYMBOL(flow_rule_match_meta);
62
63void flow_rule_match_basic(const struct flow_rule *rule,
64 struct flow_match_basic *out)
65{
66 FLOW_DISSECTOR_MATCH(rule, FLOW_DISSECTOR_KEY_BASIC, out);
67}
68EXPORT_SYMBOL(flow_rule_match_basic);
69
70void flow_rule_match_control(const struct flow_rule *rule,
71 struct flow_match_control *out)
72{
73 FLOW_DISSECTOR_MATCH(rule, FLOW_DISSECTOR_KEY_CONTROL, out);
74}
75EXPORT_SYMBOL(flow_rule_match_control);
76
77void flow_rule_match_eth_addrs(const struct flow_rule *rule,
78 struct flow_match_eth_addrs *out)
79{
80 FLOW_DISSECTOR_MATCH(rule, FLOW_DISSECTOR_KEY_ETH_ADDRS, out);
81}
82EXPORT_SYMBOL(flow_rule_match_eth_addrs);
83
84void flow_rule_match_vlan(const struct flow_rule *rule,
85 struct flow_match_vlan *out)
86{
87 FLOW_DISSECTOR_MATCH(rule, FLOW_DISSECTOR_KEY_VLAN, out);
88}
89EXPORT_SYMBOL(flow_rule_match_vlan);
90
91void flow_rule_match_cvlan(const struct flow_rule *rule,
92 struct flow_match_vlan *out)
93{
94 FLOW_DISSECTOR_MATCH(rule, FLOW_DISSECTOR_KEY_CVLAN, out);
95}
96EXPORT_SYMBOL(flow_rule_match_cvlan);
97
98void flow_rule_match_arp(const struct flow_rule *rule,
99 struct flow_match_arp *out)
100{
101 FLOW_DISSECTOR_MATCH(rule, FLOW_DISSECTOR_KEY_ARP, out);
102}
103EXPORT_SYMBOL(flow_rule_match_arp);
104
105void flow_rule_match_ipv4_addrs(const struct flow_rule *rule,
106 struct flow_match_ipv4_addrs *out)
107{
108 FLOW_DISSECTOR_MATCH(rule, FLOW_DISSECTOR_KEY_IPV4_ADDRS, out);
109}
110EXPORT_SYMBOL(flow_rule_match_ipv4_addrs);
111
112void flow_rule_match_ipv6_addrs(const struct flow_rule *rule,
113 struct flow_match_ipv6_addrs *out)
114{
115 FLOW_DISSECTOR_MATCH(rule, FLOW_DISSECTOR_KEY_IPV6_ADDRS, out);
116}
117EXPORT_SYMBOL(flow_rule_match_ipv6_addrs);
118
119void flow_rule_match_ip(const struct flow_rule *rule,
120 struct flow_match_ip *out)
121{
122 FLOW_DISSECTOR_MATCH(rule, FLOW_DISSECTOR_KEY_IP, out);
123}
124EXPORT_SYMBOL(flow_rule_match_ip);
125
126void flow_rule_match_ports(const struct flow_rule *rule,
127 struct flow_match_ports *out)
128{
129 FLOW_DISSECTOR_MATCH(rule, FLOW_DISSECTOR_KEY_PORTS, out);
130}
131EXPORT_SYMBOL(flow_rule_match_ports);
132
133void flow_rule_match_ports_range(const struct flow_rule *rule,
134 struct flow_match_ports_range *out)
135{
136 FLOW_DISSECTOR_MATCH(rule, FLOW_DISSECTOR_KEY_PORTS_RANGE, out);
137}
138EXPORT_SYMBOL(flow_rule_match_ports_range);
139
140void flow_rule_match_tcp(const struct flow_rule *rule,
141 struct flow_match_tcp *out)
142{
143 FLOW_DISSECTOR_MATCH(rule, FLOW_DISSECTOR_KEY_TCP, out);
144}
145EXPORT_SYMBOL(flow_rule_match_tcp);
146
147void flow_rule_match_ipsec(const struct flow_rule *rule,
148 struct flow_match_ipsec *out)
149{
150 FLOW_DISSECTOR_MATCH(rule, FLOW_DISSECTOR_KEY_IPSEC, out);
151}
152EXPORT_SYMBOL(flow_rule_match_ipsec);
153
154void flow_rule_match_icmp(const struct flow_rule *rule,
155 struct flow_match_icmp *out)
156{
157 FLOW_DISSECTOR_MATCH(rule, FLOW_DISSECTOR_KEY_ICMP, out);
158}
159EXPORT_SYMBOL(flow_rule_match_icmp);
160
161void flow_rule_match_mpls(const struct flow_rule *rule,
162 struct flow_match_mpls *out)
163{
164 FLOW_DISSECTOR_MATCH(rule, FLOW_DISSECTOR_KEY_MPLS, out);
165}
166EXPORT_SYMBOL(flow_rule_match_mpls);
167
168void flow_rule_match_enc_control(const struct flow_rule *rule,
169 struct flow_match_control *out)
170{
171 FLOW_DISSECTOR_MATCH(rule, FLOW_DISSECTOR_KEY_ENC_CONTROL, out);
172}
173EXPORT_SYMBOL(flow_rule_match_enc_control);
174
175void flow_rule_match_enc_ipv4_addrs(const struct flow_rule *rule,
176 struct flow_match_ipv4_addrs *out)
177{
178 FLOW_DISSECTOR_MATCH(rule, FLOW_DISSECTOR_KEY_ENC_IPV4_ADDRS, out);
179}
180EXPORT_SYMBOL(flow_rule_match_enc_ipv4_addrs);
181
182void flow_rule_match_enc_ipv6_addrs(const struct flow_rule *rule,
183 struct flow_match_ipv6_addrs *out)
184{
185 FLOW_DISSECTOR_MATCH(rule, FLOW_DISSECTOR_KEY_ENC_IPV6_ADDRS, out);
186}
187EXPORT_SYMBOL(flow_rule_match_enc_ipv6_addrs);
188
189void flow_rule_match_enc_ip(const struct flow_rule *rule,
190 struct flow_match_ip *out)
191{
192 FLOW_DISSECTOR_MATCH(rule, FLOW_DISSECTOR_KEY_ENC_IP, out);
193}
194EXPORT_SYMBOL(flow_rule_match_enc_ip);
195
196void flow_rule_match_enc_ports(const struct flow_rule *rule,
197 struct flow_match_ports *out)
198{
199 FLOW_DISSECTOR_MATCH(rule, FLOW_DISSECTOR_KEY_ENC_PORTS, out);
200}
201EXPORT_SYMBOL(flow_rule_match_enc_ports);
202
203void flow_rule_match_enc_keyid(const struct flow_rule *rule,
204 struct flow_match_enc_keyid *out)
205{
206 FLOW_DISSECTOR_MATCH(rule, FLOW_DISSECTOR_KEY_ENC_KEYID, out);
207}
208EXPORT_SYMBOL(flow_rule_match_enc_keyid);
209
210void flow_rule_match_enc_opts(const struct flow_rule *rule,
211 struct flow_match_enc_opts *out)
212{
213 FLOW_DISSECTOR_MATCH(rule, FLOW_DISSECTOR_KEY_ENC_OPTS, out);
214}
215EXPORT_SYMBOL(flow_rule_match_enc_opts);
216
217struct flow_action_cookie *flow_action_cookie_create(void *data,
218 unsigned int len,
219 gfp_t gfp)
220{
221 struct flow_action_cookie *cookie;
222
223 cookie = kmalloc(sizeof(*cookie) + len, gfp);
224 if (!cookie)
225 return NULL;
226 cookie->cookie_len = len;
227 memcpy(cookie->cookie, data, len);
228 return cookie;
229}
230EXPORT_SYMBOL(flow_action_cookie_create);
231
232void flow_action_cookie_destroy(struct flow_action_cookie *cookie)
233{
234 kfree(cookie);
235}
236EXPORT_SYMBOL(flow_action_cookie_destroy);
237
238void flow_rule_match_ct(const struct flow_rule *rule,
239 struct flow_match_ct *out)
240{
241 FLOW_DISSECTOR_MATCH(rule, FLOW_DISSECTOR_KEY_CT, out);
242}
243EXPORT_SYMBOL(flow_rule_match_ct);
244
245void flow_rule_match_pppoe(const struct flow_rule *rule,
246 struct flow_match_pppoe *out)
247{
248 FLOW_DISSECTOR_MATCH(rule, FLOW_DISSECTOR_KEY_PPPOE, out);
249}
250EXPORT_SYMBOL(flow_rule_match_pppoe);
251
252void flow_rule_match_l2tpv3(const struct flow_rule *rule,
253 struct flow_match_l2tpv3 *out)
254{
255 FLOW_DISSECTOR_MATCH(rule, FLOW_DISSECTOR_KEY_L2TPV3, out);
256}
257EXPORT_SYMBOL(flow_rule_match_l2tpv3);
258
259struct flow_block_cb *flow_block_cb_alloc(flow_setup_cb_t *cb,
260 void *cb_ident, void *cb_priv,
261 void (*release)(void *cb_priv))
262{
263 struct flow_block_cb *block_cb;
264
265 block_cb = kzalloc_obj(*block_cb);
266 if (!block_cb)
267 return ERR_PTR(-ENOMEM);
268
269 block_cb->cb = cb;
270 block_cb->cb_ident = cb_ident;
271 block_cb->cb_priv = cb_priv;
272 block_cb->release = release;
273
274 return block_cb;
275}
276EXPORT_SYMBOL(flow_block_cb_alloc);
277
278void flow_block_cb_free(struct flow_block_cb *block_cb)
279{
280 if (block_cb->release)
281 block_cb->release(block_cb->cb_priv);
282
283 kfree(block_cb);
284}
285EXPORT_SYMBOL(flow_block_cb_free);
286
287struct flow_block_cb *flow_block_cb_lookup(struct flow_block *block,
288 flow_setup_cb_t *cb, void *cb_ident)
289{
290 struct flow_block_cb *block_cb;
291
292 list_for_each_entry(block_cb, &block->cb_list, list) {
293 if (block_cb->cb == cb &&
294 block_cb->cb_ident == cb_ident)
295 return block_cb;
296 }
297
298 return NULL;
299}
300EXPORT_SYMBOL(flow_block_cb_lookup);
301
302void *flow_block_cb_priv(struct flow_block_cb *block_cb)
303{
304 return block_cb->cb_priv;
305}
306EXPORT_SYMBOL(flow_block_cb_priv);
307
308void flow_block_cb_incref(struct flow_block_cb *block_cb)
309{
310 block_cb->refcnt++;
311}
312EXPORT_SYMBOL(flow_block_cb_incref);
313
314unsigned int flow_block_cb_decref(struct flow_block_cb *block_cb)
315{
316 return --block_cb->refcnt;
317}
318EXPORT_SYMBOL(flow_block_cb_decref);
319
320bool flow_block_cb_is_busy(flow_setup_cb_t *cb, void *cb_ident,
321 struct list_head *driver_block_list)
322{
323 struct flow_block_cb *block_cb;
324
325 list_for_each_entry(block_cb, driver_block_list, driver_list) {
326 if (block_cb->cb == cb &&
327 block_cb->cb_ident == cb_ident)
328 return true;
329 }
330
331 return false;
332}
333EXPORT_SYMBOL(flow_block_cb_is_busy);
334
335int flow_block_cb_setup_simple(struct flow_block_offload *f,
336 struct list_head *driver_block_list,
337 flow_setup_cb_t *cb,
338 void *cb_ident, void *cb_priv,
339 bool ingress_only)
340{
341 struct flow_block_cb *block_cb;
342
343 if (ingress_only &&
344 f->binder_type != FLOW_BLOCK_BINDER_TYPE_CLSACT_INGRESS)
345 return -EOPNOTSUPP;
346
347 f->driver_block_list = driver_block_list;
348
349 switch (f->command) {
350 case FLOW_BLOCK_BIND:
351 if (flow_block_cb_is_busy(cb, cb_ident, driver_block_list))
352 return -EBUSY;
353
354 block_cb = flow_block_cb_alloc(cb, cb_ident, cb_priv, NULL);
355 if (IS_ERR(block_cb))
356 return PTR_ERR(block_cb);
357
358 flow_block_cb_add(block_cb, f);
359 list_add_tail(&block_cb->driver_list, driver_block_list);
360 return 0;
361 case FLOW_BLOCK_UNBIND:
362 block_cb = flow_block_cb_lookup(f->block, cb, cb_ident);
363 if (!block_cb)
364 return -ENOENT;
365
366 flow_block_cb_remove(block_cb, f);
367 list_del(&block_cb->driver_list);
368 return 0;
369 default:
370 return -EOPNOTSUPP;
371 }
372}
373EXPORT_SYMBOL(flow_block_cb_setup_simple);
374
375static DEFINE_MUTEX(flow_indr_block_lock);
376static LIST_HEAD(flow_block_indr_list);
377static LIST_HEAD(flow_block_indr_dev_list);
378static LIST_HEAD(flow_indir_dev_list);
379
380struct flow_indr_dev {
381 struct list_head list;
382 flow_indr_block_bind_cb_t *cb;
383 void *cb_priv;
384 refcount_t refcnt;
385};
386
387static struct flow_indr_dev *flow_indr_dev_alloc(flow_indr_block_bind_cb_t *cb,
388 void *cb_priv)
389{
390 struct flow_indr_dev *indr_dev;
391
392 indr_dev = kmalloc_obj(*indr_dev);
393 if (!indr_dev)
394 return NULL;
395
396 indr_dev->cb = cb;
397 indr_dev->cb_priv = cb_priv;
398 refcount_set(&indr_dev->refcnt, 1);
399
400 return indr_dev;
401}
402
403struct flow_indir_dev_info {
404 void *data;
405 struct net_device *dev;
406 struct Qdisc *sch;
407 enum tc_setup_type type;
408 void (*cleanup)(struct flow_block_cb *block_cb);
409 struct list_head list;
410 enum flow_block_command command;
411 enum flow_block_binder_type binder_type;
412 struct list_head *cb_list;
413};
414
415static void existing_qdiscs_register(flow_indr_block_bind_cb_t *cb, void *cb_priv)
416{
417 struct flow_block_offload bo;
418 struct flow_indir_dev_info *cur;
419
420 list_for_each_entry(cur, &flow_indir_dev_list, list) {
421 memset(&bo, 0, sizeof(bo));
422 bo.command = cur->command;
423 bo.binder_type = cur->binder_type;
424 INIT_LIST_HEAD(&bo.cb_list);
425 cb(cur->dev, cur->sch, cb_priv, cur->type, &bo, cur->data, cur->cleanup);
426 list_splice(&bo.cb_list, cur->cb_list);
427 }
428}
429
430int flow_indr_dev_register(flow_indr_block_bind_cb_t *cb, void *cb_priv)
431{
432 struct flow_indr_dev *indr_dev;
433
434 mutex_lock(&flow_indr_block_lock);
435 list_for_each_entry(indr_dev, &flow_block_indr_dev_list, list) {
436 if (indr_dev->cb == cb &&
437 indr_dev->cb_priv == cb_priv) {
438 refcount_inc(&indr_dev->refcnt);
439 mutex_unlock(&flow_indr_block_lock);
440 return 0;
441 }
442 }
443
444 indr_dev = flow_indr_dev_alloc(cb, cb_priv);
445 if (!indr_dev) {
446 mutex_unlock(&flow_indr_block_lock);
447 return -ENOMEM;
448 }
449
450 list_add(&indr_dev->list, &flow_block_indr_dev_list);
451 existing_qdiscs_register(cb, cb_priv);
452 mutex_unlock(&flow_indr_block_lock);
453
454 tcf_action_reoffload_cb(cb, cb_priv, true);
455
456 return 0;
457}
458EXPORT_SYMBOL(flow_indr_dev_register);
459
460static void __flow_block_indr_cleanup(void (*release)(void *cb_priv),
461 void *cb_priv,
462 struct list_head *cleanup_list)
463{
464 struct flow_block_cb *this, *next;
465
466 list_for_each_entry_safe(this, next, &flow_block_indr_list, indr.list) {
467 if (this->release == release &&
468 this->indr.cb_priv == cb_priv)
469 list_move(&this->indr.list, cleanup_list);
470 }
471}
472
473static void flow_block_indr_notify(struct list_head *cleanup_list)
474{
475 struct flow_block_cb *this, *next;
476
477 list_for_each_entry_safe(this, next, cleanup_list, indr.list) {
478 list_del(&this->indr.list);
479 this->indr.cleanup(this);
480 }
481}
482
483void flow_indr_dev_unregister(flow_indr_block_bind_cb_t *cb, void *cb_priv,
484 void (*release)(void *cb_priv))
485{
486 struct flow_indr_dev *this, *next, *indr_dev = NULL;
487 LIST_HEAD(cleanup_list);
488
489 mutex_lock(&flow_indr_block_lock);
490 list_for_each_entry_safe(this, next, &flow_block_indr_dev_list, list) {
491 if (this->cb == cb &&
492 this->cb_priv == cb_priv &&
493 refcount_dec_and_test(&this->refcnt)) {
494 indr_dev = this;
495 list_del(&indr_dev->list);
496 break;
497 }
498 }
499
500 if (!indr_dev) {
501 mutex_unlock(&flow_indr_block_lock);
502 return;
503 }
504
505 __flow_block_indr_cleanup(release, cb_priv, &cleanup_list);
506 mutex_unlock(&flow_indr_block_lock);
507
508 tcf_action_reoffload_cb(cb, cb_priv, false);
509 flow_block_indr_notify(&cleanup_list);
510 kfree(indr_dev);
511}
512EXPORT_SYMBOL(flow_indr_dev_unregister);
513
514static void flow_block_indr_init(struct flow_block_cb *flow_block,
515 struct flow_block_offload *bo,
516 struct net_device *dev, struct Qdisc *sch, void *data,
517 void *cb_priv,
518 void (*cleanup)(struct flow_block_cb *block_cb))
519{
520 flow_block->indr.binder_type = bo->binder_type;
521 flow_block->indr.data = data;
522 flow_block->indr.cb_priv = cb_priv;
523 flow_block->indr.dev = dev;
524 flow_block->indr.sch = sch;
525 flow_block->indr.cleanup = cleanup;
526}
527
528struct flow_block_cb *flow_indr_block_cb_alloc(flow_setup_cb_t *cb,
529 void *cb_ident, void *cb_priv,
530 void (*release)(void *cb_priv),
531 struct flow_block_offload *bo,
532 struct net_device *dev,
533 struct Qdisc *sch, void *data,
534 void *indr_cb_priv,
535 void (*cleanup)(struct flow_block_cb *block_cb))
536{
537 struct flow_block_cb *block_cb;
538
539 block_cb = flow_block_cb_alloc(cb, cb_ident, cb_priv, release);
540 if (IS_ERR(block_cb))
541 goto out;
542
543 flow_block_indr_init(block_cb, bo, dev, sch, data, indr_cb_priv, cleanup);
544 list_add(&block_cb->indr.list, &flow_block_indr_list);
545
546out:
547 return block_cb;
548}
549EXPORT_SYMBOL(flow_indr_block_cb_alloc);
550
551static struct flow_indir_dev_info *find_indir_dev(void *data)
552{
553 struct flow_indir_dev_info *cur;
554
555 list_for_each_entry(cur, &flow_indir_dev_list, list) {
556 if (cur->data == data)
557 return cur;
558 }
559 return NULL;
560}
561
562static int indir_dev_add(void *data, struct net_device *dev, struct Qdisc *sch,
563 enum tc_setup_type type, void (*cleanup)(struct flow_block_cb *block_cb),
564 struct flow_block_offload *bo)
565{
566 struct flow_indir_dev_info *info;
567
568 info = find_indir_dev(data);
569 if (info)
570 return -EEXIST;
571
572 info = kzalloc_obj(*info);
573 if (!info)
574 return -ENOMEM;
575
576 info->data = data;
577 info->dev = dev;
578 info->sch = sch;
579 info->type = type;
580 info->cleanup = cleanup;
581 info->command = bo->command;
582 info->binder_type = bo->binder_type;
583 info->cb_list = bo->cb_list_head;
584
585 list_add(&info->list, &flow_indir_dev_list);
586 return 0;
587}
588
589static int indir_dev_remove(void *data)
590{
591 struct flow_indir_dev_info *info;
592
593 info = find_indir_dev(data);
594 if (!info)
595 return -ENOENT;
596
597 list_del(&info->list);
598
599 kfree(info);
600 return 0;
601}
602
603int flow_indr_dev_setup_offload(struct net_device *dev, struct Qdisc *sch,
604 enum tc_setup_type type, void *data,
605 struct flow_block_offload *bo,
606 void (*cleanup)(struct flow_block_cb *block_cb))
607{
608 struct flow_indr_dev *this;
609 u32 count = 0;
610 int err;
611
612 mutex_lock(&flow_indr_block_lock);
613 if (bo) {
614 if (bo->command == FLOW_BLOCK_BIND)
615 indir_dev_add(data, dev, sch, type, cleanup, bo);
616 else if (bo->command == FLOW_BLOCK_UNBIND)
617 indir_dev_remove(data);
618 }
619
620 list_for_each_entry(this, &flow_block_indr_dev_list, list) {
621 err = this->cb(dev, sch, this->cb_priv, type, bo, data, cleanup);
622 if (!err)
623 count++;
624 }
625
626 mutex_unlock(&flow_indr_block_lock);
627
628 return (bo && list_empty(&bo->cb_list)) ? -EOPNOTSUPP : count;
629}
630EXPORT_SYMBOL(flow_indr_dev_setup_offload);
631
632bool flow_indr_dev_exists(void)
633{
634 return !list_empty(&flow_block_indr_dev_list);
635}
636EXPORT_SYMBOL(flow_indr_dev_exists);