projects/linux·blob master · refs

1098 lines · 26.5 KiB

1/* SPDX-License-Identifier: GPL-2.0 */
2#ifndef __NET_PKT_CLS_H
3#define __NET_PKT_CLS_H
4
5#include <linux/pkt_cls.h>
6#include <linux/workqueue.h>
7#include <net/sch_generic.h>
8#include <net/act_api.h>
9#include <net/net_namespace.h>
10
11/* TC action not accessible from user space */
12#define TC_ACT_CONSUMED (TC_ACT_VALUE_MAX + 1)
13
14/* Basic packet classifier frontend definitions. */
15
16struct tcf_walker {
17 int stop;
18 int skip;
19 int count;
20 bool nonempty;
21 unsigned long cookie;
22 int (*fn)(struct tcf_proto *, void *node, struct tcf_walker *);
23};
24
25int register_tcf_proto_ops(struct tcf_proto_ops *ops);
26void unregister_tcf_proto_ops(struct tcf_proto_ops *ops);
27#define NET_CLS_ALIAS_PREFIX "net-cls-"
28#define MODULE_ALIAS_NET_CLS(kind) MODULE_ALIAS(NET_CLS_ALIAS_PREFIX kind)
29
30struct tcf_block_ext_info {
31 enum flow_block_binder_type binder_type;
32 tcf_chain_head_change_t *chain_head_change;
33 void *chain_head_change_priv;
34 u32 block_index;
35};
36
37struct tcf_qevent {
38 struct tcf_block *block;
39 struct tcf_block_ext_info info;
40 struct tcf_proto __rcu *filter_chain;
41};
42
43struct tcf_block_cb;
44bool tcf_queue_work(struct rcu_work *rwork, work_func_t func);
45
46#ifdef CONFIG_NET_CLS
47struct tcf_chain *tcf_chain_get_by_act(struct tcf_block *block,
48 u32 chain_index);
49void tcf_chain_put_by_act(struct tcf_chain *chain);
50struct tcf_chain *tcf_get_next_chain(struct tcf_block *block,
51 struct tcf_chain *chain);
52struct tcf_proto *tcf_get_next_proto(struct tcf_chain *chain,
53 struct tcf_proto *tp);
54void tcf_block_netif_keep_dst(struct tcf_block *block);
55int tcf_block_get(struct tcf_block **p_block,
56 struct tcf_proto __rcu **p_filter_chain, struct Qdisc *q,
57 struct netlink_ext_ack *extack);
58int tcf_block_get_ext(struct tcf_block **p_block, struct Qdisc *q,
59 struct tcf_block_ext_info *ei,
60 struct netlink_ext_ack *extack);
61void tcf_block_put(struct tcf_block *block);
62void tcf_block_put_ext(struct tcf_block *block, struct Qdisc *q,
63 struct tcf_block_ext_info *ei);
64int tcf_exts_init_ex(struct tcf_exts *exts, struct net *net, int action,
65 int police, struct tcf_proto *tp, u32 handle, bool used_action_miss);
66
67static inline bool tcf_block_shared(struct tcf_block *block)
68{
69 return block->index;
70}
71
72static inline bool tcf_block_non_null_shared(struct tcf_block *block)
73{
74 return block && block->index;
75}
76
77#ifdef CONFIG_NET_CLS_ACT
78DECLARE_STATIC_KEY_FALSE(tcf_sw_enabled_key);
79
80static inline bool tcf_block_bypass_sw(struct tcf_block *block)
81{
82 return block && !atomic_read(&block->useswcnt);
83}
84#endif
85
86static inline struct Qdisc *tcf_block_q(struct tcf_block *block)
87{
88 WARN_ON(tcf_block_shared(block));
89 return block->q;
90}
91
92int tcf_classify(struct sk_buff *skb,
93 const struct tcf_block *block,
94 const struct tcf_proto *tp, struct tcf_result *res,
95 bool compat_mode);
96
97static inline bool tc_cls_stats_dump(struct tcf_proto *tp,
98 struct tcf_walker *arg,
99 void *filter)
100{
101 if (arg->count >= arg->skip && arg->fn(tp, filter, arg) < 0) {
102 arg->stop = 1;
103 return false;
104 }
105
106 arg->count++;
107 return true;
108}
109
110#else
111static inline bool tcf_block_shared(struct tcf_block *block)
112{
113 return false;
114}
115
116static inline bool tcf_block_non_null_shared(struct tcf_block *block)
117{
118 return false;
119}
120
121static inline
122int tcf_block_get(struct tcf_block **p_block,
123 struct tcf_proto __rcu **p_filter_chain, struct Qdisc *q,
124 struct netlink_ext_ack *extack)
125{
126 return 0;
127}
128
129static inline
130int tcf_block_get_ext(struct tcf_block **p_block, struct Qdisc *q,
131 struct tcf_block_ext_info *ei,
132 struct netlink_ext_ack *extack)
133{
134 return 0;
135}
136
137static inline void tcf_block_put(struct tcf_block *block)
138{
139}
140
141static inline
142void tcf_block_put_ext(struct tcf_block *block, struct Qdisc *q,
143 struct tcf_block_ext_info *ei)
144{
145}
146
147static inline struct Qdisc *tcf_block_q(struct tcf_block *block)
148{
149 return NULL;
150}
151
152static inline int tcf_classify(struct sk_buff *skb,
153 const struct tcf_block *block,
154 const struct tcf_proto *tp,
155 struct tcf_result *res, bool compat_mode)
156{
157 return TC_ACT_UNSPEC;
158}
159#endif
160static inline int tcf_classify_qdisc(struct sk_buff *skb,
161 const struct tcf_proto *tp,
162 struct tcf_result *res, bool compat_mode)
163{
164 int ret = tcf_classify(skb, NULL, tp, res, compat_mode);
165
166 /* TC_ACT_REDIRECT from qdisc filter chains is not supported.
167 * Use BPF via tcx or mirred redirect instead.
168 */
169 if (unlikely(ret == TC_ACT_REDIRECT))
170 ret = TC_ACT_SHOT;
171 return ret;
172}
173
174static inline unsigned long
175__cls_set_class(unsigned long *clp, unsigned long cl)
176{
177 return xchg(clp, cl);
178}
179
180static inline void
181__tcf_bind_filter(struct Qdisc *q, struct tcf_result *r, unsigned long base)
182{
183 unsigned long cl;
184
185 cl = q->ops->cl_ops->bind_tcf(q, base, r->classid);
186 cl = __cls_set_class(&r->class, cl);
187 if (cl)
188 q->ops->cl_ops->unbind_tcf(q, cl);
189}
190
191static inline void
192tcf_bind_filter(struct tcf_proto *tp, struct tcf_result *r, unsigned long base)
193{
194 struct Qdisc *q = tp->chain->block->q;
195
196 /* Check q as it is not set for shared blocks. In that case,
197 * setting class is not supported.
198 */
199 if (!q)
200 return;
201 sch_tree_lock(q);
202 __tcf_bind_filter(q, r, base);
203 sch_tree_unlock(q);
204}
205
206static inline void
207__tcf_unbind_filter(struct Qdisc *q, struct tcf_result *r)
208{
209 unsigned long cl;
210
211 if ((cl = __cls_set_class(&r->class, 0)) != 0)
212 q->ops->cl_ops->unbind_tcf(q, cl);
213}
214
215static inline void
216tcf_unbind_filter(struct tcf_proto *tp, struct tcf_result *r)
217{
218 struct Qdisc *q = tp->chain->block->q;
219
220 if (!q)
221 return;
222 __tcf_unbind_filter(q, r);
223}
224
225static inline void tc_cls_bind_class(u32 classid, unsigned long cl,
226 void *q, struct tcf_result *res,
227 unsigned long base)
228{
229 if (res->classid == classid) {
230 if (cl)
231 __tcf_bind_filter(q, res, base);
232 else
233 __tcf_unbind_filter(q, res);
234 }
235}
236
237struct tcf_exts {
238#ifdef CONFIG_NET_CLS_ACT
239 __u32 type; /* for backward compat(TCA_OLD_COMPAT) */
240 int nr_actions;
241 struct tc_action **actions;
242 struct net *net;
243 netns_tracker ns_tracker;
244 struct tcf_exts_miss_cookie_node *miss_cookie_node;
245#endif
246 /* Map to export classifier specific extension TLV types to the
247 * generic extensions API. Unsupported extensions must be set to 0.
248 */
249 int action;
250 int police;
251};
252
253static inline int tcf_exts_init(struct tcf_exts *exts, struct net *net,
254 int action, int police)
255{
256#ifdef CONFIG_NET_CLS
257 return tcf_exts_init_ex(exts, net, action, police, NULL, 0, false);
258#else
259 return -EOPNOTSUPP;
260#endif
261}
262
263/* Return false if the netns is being destroyed in cleanup_net(). Callers
264 * need to do cleanup synchronously in this case, otherwise may race with
265 * tc_action_net_exit(). Return true for other cases.
266 */
267static inline bool tcf_exts_get_net(struct tcf_exts *exts)
268{
269#ifdef CONFIG_NET_CLS_ACT
270 exts->net = maybe_get_net(exts->net);
271 if (exts->net)
272 netns_tracker_alloc(exts->net, &exts->ns_tracker, GFP_KERNEL);
273 return exts->net != NULL;
274#else
275 return true;
276#endif
277}
278
279static inline void tcf_exts_put_net(struct tcf_exts *exts)
280{
281#ifdef CONFIG_NET_CLS_ACT
282 if (exts->net)
283 put_net_track(exts->net, &exts->ns_tracker);
284#endif
285}
286
287#ifdef CONFIG_NET_CLS_ACT
288#define tcf_exts_for_each_action(i, a, exts) \
289 for (i = 0; i < TCA_ACT_MAX_PRIO && ((a) = (exts)->actions[i]); i++)
290#else
291#define tcf_exts_for_each_action(i, a, exts) \
292 for (; 0; (void)(i), (void)(a), (void)(exts))
293#endif
294
295#define tcf_act_for_each_action(i, a, actions) \
296 for (i = 0; i < TCA_ACT_MAX_PRIO && ((a) = actions[i]); i++)
297
298static inline bool tc_act_in_hw(struct tc_action *act)
299{
300 return !!act->in_hw_count;
301}
302
303static inline void
304tcf_exts_hw_stats_update(const struct tcf_exts *exts,
305 struct flow_stats *stats,
306 bool use_act_stats)
307{
308#ifdef CONFIG_NET_CLS_ACT
309 int i;
310
311 for (i = 0; i < exts->nr_actions; i++) {
312 struct tc_action *a = exts->actions[i];
313
314 if (use_act_stats || tc_act_in_hw(a)) {
315 if (!tcf_action_update_hw_stats(a))
316 continue;
317 }
318
319 preempt_disable();
320 tcf_action_stats_update(a, stats->bytes, stats->pkts, stats->drops,
321 stats->lastused, true);
322 preempt_enable();
323
324 a->used_hw_stats = stats->used_hw_stats;
325 a->used_hw_stats_valid = stats->used_hw_stats_valid;
326 }
327#endif
328}
329
330/**
331 * tcf_exts_has_actions - check if at least one action is present
332 * @exts: tc filter extensions handle
333 *
334 * Returns: true if at least one action is present.
335 */
336static inline bool tcf_exts_has_actions(struct tcf_exts *exts)
337{
338#ifdef CONFIG_NET_CLS_ACT
339 return exts->nr_actions;
340#else
341 return false;
342#endif
343}
344
345/**
346 * tcf_exts_exec - execute tc filter extensions
347 * @skb: socket buffer
348 * @exts: tc filter extensions handle
349 * @res: desired result
350 *
351 * Executes all configured extensions. Returns TC_ACT_OK on a normal execution,
352 * a negative number if the filter must be considered unmatched or
353 * a positive action code (TC_ACT_*) which must be returned to the
354 * underlying layer.
355 */
356static inline int
357tcf_exts_exec(struct sk_buff *skb, struct tcf_exts *exts,
358 struct tcf_result *res)
359{
360#ifdef CONFIG_NET_CLS_ACT
361 return tcf_action_exec(skb, exts->actions, exts->nr_actions, res);
362#endif
363 return TC_ACT_OK;
364}
365
366static inline int
367tcf_exts_exec_ex(struct sk_buff *skb, struct tcf_exts *exts, int act_index,
368 struct tcf_result *res)
369{
370#ifdef CONFIG_NET_CLS_ACT
371 return tcf_action_exec(skb, exts->actions + act_index,
372 exts->nr_actions - act_index, res);
373#else
374 return TC_ACT_OK;
375#endif
376}
377
378int tcf_exts_validate(struct net *net, struct tcf_proto *tp,
379 struct nlattr **tb, struct nlattr *rate_tlv,
380 struct tcf_exts *exts, u32 flags,
381 struct netlink_ext_ack *extack);
382int tcf_exts_validate_ex(struct net *net, struct tcf_proto *tp, struct nlattr **tb,
383 struct nlattr *rate_tlv, struct tcf_exts *exts,
384 u32 flags, u32 fl_flags, struct netlink_ext_ack *extack);
385void tcf_exts_destroy(struct tcf_exts *exts);
386void tcf_exts_change(struct tcf_exts *dst, struct tcf_exts *src);
387int tcf_exts_dump(struct sk_buff *skb, struct tcf_exts *exts);
388int tcf_exts_terse_dump(struct sk_buff *skb, struct tcf_exts *exts);
389int tcf_exts_dump_stats(struct sk_buff *skb, struct tcf_exts *exts);
390
391/**
392 * struct tcf_pkt_info - packet information
393 *
394 * @ptr: start of the pkt data
395 * @nexthdr: offset of the next header
396 */
397struct tcf_pkt_info {
398 unsigned char * ptr;
399 int nexthdr;
400};
401
402#ifdef CONFIG_NET_EMATCH
403
404struct tcf_ematch_ops;
405
406/**
407 * struct tcf_ematch - extended match (ematch)
408 *
409 * @matchid: identifier to allow userspace to reidentify a match
410 * @flags: flags specifying attributes and the relation to other matches
411 * @ops: the operations lookup table of the corresponding ematch module
412 * @datalen: length of the ematch specific configuration data
413 * @data: ematch specific data
414 * @net: the network namespace
415 */
416struct tcf_ematch {
417 struct tcf_ematch_ops * ops;
418 unsigned long data;
419 unsigned int datalen;
420 u16 matchid;
421 u16 flags;
422 struct net *net;
423};
424
425static inline int tcf_em_is_container(struct tcf_ematch *em)
426{
427 return !em->ops;
428}
429
430static inline int tcf_em_is_simple(struct tcf_ematch *em)
431{
432 return em->flags & TCF_EM_SIMPLE;
433}
434
435static inline int tcf_em_is_inverted(struct tcf_ematch *em)
436{
437 return em->flags & TCF_EM_INVERT;
438}
439
440static inline int tcf_em_last_match(struct tcf_ematch *em)
441{
442 return (em->flags & TCF_EM_REL_MASK) == TCF_EM_REL_END;
443}
444
445static inline int tcf_em_early_end(struct tcf_ematch *em, int result)
446{
447 if (tcf_em_last_match(em))
448 return 1;
449
450 if (result == 0 && em->flags & TCF_EM_REL_AND)
451 return 1;
452
453 if (result != 0 && em->flags & TCF_EM_REL_OR)
454 return 1;
455
456 return 0;
457}
458
459/**
460 * struct tcf_ematch_tree - ematch tree handle
461 *
462 * @hdr: ematch tree header supplied by userspace
463 * @matches: array of ematches
464 */
465struct tcf_ematch_tree {
466 struct tcf_ematch_tree_hdr hdr;
467 struct tcf_ematch * matches;
468
469};
470
471/**
472 * struct tcf_ematch_ops - ematch module operations
473 *
474 * @kind: identifier (kind) of this ematch module
475 * @datalen: length of expected configuration data (optional)
476 * @change: called during validation (optional)
477 * @match: called during ematch tree evaluation, must return 1/0
478 * @destroy: called during destroyage (optional)
479 * @dump: called during dumping process (optional)
480 * @owner: owner, must be set to THIS_MODULE
481 * @link: link to previous/next ematch module (internal use)
482 */
483struct tcf_ematch_ops {
484 int kind;
485 int datalen;
486 int (*change)(struct net *net, void *,
487 int, struct tcf_ematch *);
488 int (*match)(struct sk_buff *, struct tcf_ematch *,
489 struct tcf_pkt_info *);
490 void (*destroy)(struct tcf_ematch *);
491 int (*dump)(struct sk_buff *, struct tcf_ematch *);
492 struct module *owner;
493 struct list_head link;
494};
495
496int tcf_em_register(struct tcf_ematch_ops *);
497void tcf_em_unregister(struct tcf_ematch_ops *);
498int tcf_em_tree_validate(struct tcf_proto *, struct nlattr *,
499 struct tcf_ematch_tree *);
500void tcf_em_tree_destroy(struct tcf_ematch_tree *);
501int tcf_em_tree_dump(struct sk_buff *, struct tcf_ematch_tree *, int);
502int __tcf_em_tree_match(struct sk_buff *, struct tcf_ematch_tree *,
503 struct tcf_pkt_info *);
504
505/**
506 * tcf_em_tree_match - evaluate an ematch tree
507 *
508 * @skb: socket buffer of the packet in question
509 * @tree: ematch tree to be used for evaluation
510 * @info: packet information examined by classifier
511 *
512 * This function matches @skb against the ematch tree in @tree by going
513 * through all ematches respecting their logic relations returning
514 * as soon as the result is obvious.
515 *
516 * Returns: 1 if the ematch tree as-one matches, no ematches are configured
517 * or ematch is not enabled in the kernel, otherwise 0 is returned.
518 */
519static inline int tcf_em_tree_match(struct sk_buff *skb,
520 struct tcf_ematch_tree *tree,
521 struct tcf_pkt_info *info)
522{
523 if (tree->hdr.nmatches)
524 return __tcf_em_tree_match(skb, tree, info);
525 else
526 return 1;
527}
528
529#define MODULE_ALIAS_TCF_EMATCH(kind) MODULE_ALIAS("ematch-kind-" __stringify(kind))
530
531#else /* CONFIG_NET_EMATCH */
532
533struct tcf_ematch_tree {
534};
535
536#define tcf_em_tree_validate(tp, tb, t) ((void)(t), 0)
537#define tcf_em_tree_destroy(t) do { (void)(t); } while(0)
538#define tcf_em_tree_dump(skb, t, tlv) (0)
539#define tcf_em_tree_match(skb, t, info) ((void)(info), 1)
540
541#endif /* CONFIG_NET_EMATCH */
542
543static inline unsigned char * tcf_get_base_ptr(struct sk_buff *skb, int layer)
544{
545 switch (layer) {
546 case TCF_LAYER_LINK:
547 return skb_mac_header(skb);
548 case TCF_LAYER_NETWORK:
549 return skb_network_header(skb);
550 case TCF_LAYER_TRANSPORT:
551 if (!skb_transport_header_was_set(skb))
552 break;
553 return skb_transport_header(skb);
554 }
555
556 return NULL;
557}
558
559static inline int tcf_valid_offset(const struct sk_buff *skb,
560 const unsigned char *ptr, const int len)
561{
562 return likely((ptr + len) <= skb_tail_pointer(skb) &&
563 ptr >= skb->head &&
564 (ptr <= (ptr + len)));
565}
566
567static inline int
568tcf_change_indev(struct net *net, struct nlattr *indev_tlv,
569 struct netlink_ext_ack *extack)
570{
571 char indev[IFNAMSIZ];
572 struct net_device *dev;
573
574 if (nla_strscpy(indev, indev_tlv, IFNAMSIZ) < 0) {
575 NL_SET_ERR_MSG_ATTR(extack, indev_tlv,
576 "Interface name too long");
577 return -EINVAL;
578 }
579 dev = __dev_get_by_name(net, indev);
580 if (!dev) {
581 NL_SET_ERR_MSG_ATTR(extack, indev_tlv,
582 "Network device not found");
583 return -ENODEV;
584 }
585 return dev->ifindex;
586}
587
588static inline bool
589tcf_match_indev(struct sk_buff *skb, int ifindex)
590{
591 if (!ifindex)
592 return true;
593 if (!skb->skb_iif)
594 return false;
595 return ifindex == skb->skb_iif;
596}
597
598int tc_setup_offload_action(struct flow_action *flow_action,
599 const struct tcf_exts *exts,
600 struct netlink_ext_ack *extack);
601void tc_cleanup_offload_action(struct flow_action *flow_action);
602int tc_setup_action(struct flow_action *flow_action,
603 struct tc_action *actions[],
604 u32 miss_cookie_base,
605 struct netlink_ext_ack *extack);
606
607int tc_setup_cb_call(struct tcf_block *block, enum tc_setup_type type,
608 void *type_data, bool err_stop, bool rtnl_held);
609int tc_setup_cb_add(struct tcf_block *block, struct tcf_proto *tp,
610 enum tc_setup_type type, void *type_data, bool err_stop,
611 u32 *flags, unsigned int *in_hw_count, bool rtnl_held);
612int tc_setup_cb_replace(struct tcf_block *block, struct tcf_proto *tp,
613 enum tc_setup_type type, void *type_data, bool err_stop,
614 u32 *old_flags, unsigned int *old_in_hw_count,
615 u32 *new_flags, unsigned int *new_in_hw_count,
616 bool rtnl_held);
617int tc_setup_cb_destroy(struct tcf_block *block, struct tcf_proto *tp,
618 enum tc_setup_type type, void *type_data, bool err_stop,
619 u32 *flags, unsigned int *in_hw_count, bool rtnl_held);
620int tc_setup_cb_reoffload(struct tcf_block *block, struct tcf_proto *tp,
621 bool add, flow_setup_cb_t *cb,
622 enum tc_setup_type type, void *type_data,
623 void *cb_priv, u32 *flags, unsigned int *in_hw_count);
624unsigned int tcf_exts_num_actions(struct tcf_exts *exts);
625
626#ifdef CONFIG_NET_CLS_ACT
627int tcf_qevent_init(struct tcf_qevent *qe, struct Qdisc *sch,
628 enum flow_block_binder_type binder_type,
629 struct nlattr *block_index_attr,
630 struct netlink_ext_ack *extack);
631void tcf_qevent_destroy(struct tcf_qevent *qe, struct Qdisc *sch);
632int tcf_qevent_validate_change(struct tcf_qevent *qe, struct nlattr *block_index_attr,
633 struct netlink_ext_ack *extack);
634struct sk_buff *tcf_qevent_handle(struct tcf_qevent *qe, struct Qdisc *sch, struct sk_buff *skb,
635 struct sk_buff **to_free, int *ret);
636int tcf_qevent_dump(struct sk_buff *skb, int attr_name, struct tcf_qevent *qe);
637#else
638static inline int tcf_qevent_init(struct tcf_qevent *qe, struct Qdisc *sch,
639 enum flow_block_binder_type binder_type,
640 struct nlattr *block_index_attr,
641 struct netlink_ext_ack *extack)
642{
643 return 0;
644}
645
646static inline void tcf_qevent_destroy(struct tcf_qevent *qe, struct Qdisc *sch)
647{
648}
649
650static inline int tcf_qevent_validate_change(struct tcf_qevent *qe, struct nlattr *block_index_attr,
651 struct netlink_ext_ack *extack)
652{
653 return 0;
654}
655
656static inline struct sk_buff *
657tcf_qevent_handle(struct tcf_qevent *qe, struct Qdisc *sch, struct sk_buff *skb,
658 struct sk_buff **to_free, int *ret)
659{
660 return skb;
661}
662
663static inline int tcf_qevent_dump(struct sk_buff *skb, int attr_name, struct tcf_qevent *qe)
664{
665 return 0;
666}
667#endif
668
669struct tc_cls_u32_knode {
670 struct tcf_exts *exts;
671 struct tcf_result *res;
672 struct tc_u32_sel *sel;
673 u32 handle;
674 u32 val;
675 u32 mask;
676 u32 link_handle;
677 u8 fshift;
678};
679
680struct tc_cls_u32_hnode {
681 u32 handle;
682 u32 prio;
683 unsigned int divisor;
684};
685
686enum tc_clsu32_command {
687 TC_CLSU32_NEW_KNODE,
688 TC_CLSU32_REPLACE_KNODE,
689 TC_CLSU32_DELETE_KNODE,
690 TC_CLSU32_NEW_HNODE,
691 TC_CLSU32_REPLACE_HNODE,
692 TC_CLSU32_DELETE_HNODE,
693};
694
695struct tc_cls_u32_offload {
696 struct flow_cls_common_offload common;
697 /* knode values */
698 enum tc_clsu32_command command;
699 union {
700 struct tc_cls_u32_knode knode;
701 struct tc_cls_u32_hnode hnode;
702 };
703};
704
705static inline bool tc_can_offload(const struct net_device *dev)
706{
707 return dev->features & NETIF_F_HW_TC;
708}
709
710static inline bool tc_can_offload_extack(const struct net_device *dev,
711 struct netlink_ext_ack *extack)
712{
713 bool can = tc_can_offload(dev);
714
715 if (!can)
716 NL_SET_ERR_MSG(extack, "TC offload is disabled on net device");
717
718 return can;
719}
720
721static inline bool
722tc_cls_can_offload_and_chain0(const struct net_device *dev,
723 struct flow_cls_common_offload *common)
724{
725 if (!tc_can_offload_extack(dev, common->extack))
726 return false;
727 if (common->chain_index) {
728 NL_SET_ERR_MSG(common->extack,
729 "Driver supports only offload of chain 0");
730 return false;
731 }
732 return true;
733}
734
735static inline bool tc_skip_hw(u32 flags)
736{
737 return (flags & TCA_CLS_FLAGS_SKIP_HW) ? true : false;
738}
739
740static inline bool tc_skip_sw(u32 flags)
741{
742 return (flags & TCA_CLS_FLAGS_SKIP_SW) ? true : false;
743}
744
745/* SKIP_HW and SKIP_SW are mutually exclusive flags. */
746static inline bool tc_flags_valid(u32 flags)
747{
748 if (flags & ~(TCA_CLS_FLAGS_SKIP_HW | TCA_CLS_FLAGS_SKIP_SW |
749 TCA_CLS_FLAGS_VERBOSE))
750 return false;
751
752 flags &= TCA_CLS_FLAGS_SKIP_HW | TCA_CLS_FLAGS_SKIP_SW;
753 if (!(flags ^ (TCA_CLS_FLAGS_SKIP_HW | TCA_CLS_FLAGS_SKIP_SW)))
754 return false;
755
756 return true;
757}
758
759static inline bool tc_in_hw(u32 flags)
760{
761 return (flags & TCA_CLS_FLAGS_IN_HW) ? true : false;
762}
763
764static inline void
765tc_cls_common_offload_init(struct flow_cls_common_offload *cls_common,
766 const struct tcf_proto *tp, u32 flags,
767 struct netlink_ext_ack *extack)
768{
769 cls_common->chain_index = tp->chain->index;
770 cls_common->protocol = tp->protocol;
771 cls_common->prio = tp->prio >> 16;
772 cls_common->skip_sw = tc_skip_sw(flags);
773 if (tc_skip_sw(flags) || flags & TCA_CLS_FLAGS_VERBOSE)
774 cls_common->extack = extack;
775}
776
777static inline void tcf_proto_update_usesw(struct tcf_proto *tp, u32 flags)
778{
779 if (tp->usesw)
780 return;
781 if (tc_skip_sw(flags) && tc_in_hw(flags))
782 return;
783 tp->usesw = true;
784}
785
786#if IS_ENABLED(CONFIG_NET_TC_SKB_EXT)
787static inline struct tc_skb_ext *tc_skb_ext_alloc(struct sk_buff *skb)
788{
789 struct tc_skb_ext *tc_skb_ext = skb_ext_add(skb, TC_SKB_EXT);
790
791 if (tc_skb_ext)
792 memset(tc_skb_ext, 0, sizeof(*tc_skb_ext));
793 return tc_skb_ext;
794}
795#endif
796
797enum tc_matchall_command {
798 TC_CLSMATCHALL_REPLACE,
799 TC_CLSMATCHALL_DESTROY,
800 TC_CLSMATCHALL_STATS,
801};
802
803struct tc_cls_matchall_offload {
804 struct flow_cls_common_offload common;
805 enum tc_matchall_command command;
806 struct flow_rule *rule;
807 struct flow_stats stats;
808 bool use_act_stats;
809 unsigned long cookie;
810};
811
812enum tc_clsbpf_command {
813 TC_CLSBPF_OFFLOAD,
814 TC_CLSBPF_STATS,
815};
816
817struct tc_cls_bpf_offload {
818 struct flow_cls_common_offload common;
819 enum tc_clsbpf_command command;
820 struct tcf_exts *exts;
821 struct bpf_prog *prog;
822 struct bpf_prog *oldprog;
823 const char *name;
824 bool exts_integrated;
825};
826
827/* This structure holds cookie structure that is passed from user
828 * to the kernel for actions and classifiers
829 */
830struct tc_cookie {
831 u8 *data;
832 u32 len;
833 struct rcu_head rcu;
834};
835
836struct tc_qopt_offload_stats {
837 struct gnet_stats_basic_sync *bstats;
838 struct gnet_stats_queue *qstats;
839};
840
841enum tc_mq_command {
842 TC_MQ_CREATE,
843 TC_MQ_DESTROY,
844 TC_MQ_STATS,
845 TC_MQ_GRAFT,
846};
847
848struct tc_mq_opt_offload_graft_params {
849 unsigned long queue;
850 u32 child_handle;
851};
852
853struct tc_mq_qopt_offload {
854 enum tc_mq_command command;
855 u32 handle;
856 union {
857 struct tc_qopt_offload_stats stats;
858 struct tc_mq_opt_offload_graft_params graft_params;
859 };
860};
861
862enum tc_htb_command {
863 /* Root */
864 TC_HTB_CREATE, /* Initialize HTB offload. */
865 TC_HTB_DESTROY, /* Destroy HTB offload. */
866
867 /* Classes */
868 /* Allocate qid and create leaf. */
869 TC_HTB_LEAF_ALLOC_QUEUE,
870 /* Convert leaf to inner, preserve and return qid, create new leaf. */
871 TC_HTB_LEAF_TO_INNER,
872 /* Delete leaf, while siblings remain. */
873 TC_HTB_LEAF_DEL,
874 /* Delete leaf, convert parent to leaf, preserving qid. */
875 TC_HTB_LEAF_DEL_LAST,
876 /* TC_HTB_LEAF_DEL_LAST, but delete driver data on hardware errors. */
877 TC_HTB_LEAF_DEL_LAST_FORCE,
878 /* Modify parameters of a node. */
879 TC_HTB_NODE_MODIFY,
880
881 /* Class qdisc */
882 TC_HTB_LEAF_QUERY_QUEUE, /* Query qid by classid. */
883};
884
885struct tc_htb_qopt_offload {
886 struct netlink_ext_ack *extack;
887 enum tc_htb_command command;
888 u32 parent_classid;
889 u16 classid;
890 u16 qid;
891 u32 quantum;
892 u64 rate;
893 u64 ceil;
894 u8 prio;
895};
896
897#define TC_HTB_CLASSID_ROOT U32_MAX
898
899enum tc_red_command {
900 TC_RED_REPLACE,
901 TC_RED_DESTROY,
902 TC_RED_STATS,
903 TC_RED_XSTATS,
904 TC_RED_GRAFT,
905};
906
907struct tc_red_qopt_offload_params {
908 u32 min;
909 u32 max;
910 u32 probability;
911 u32 limit;
912 bool is_ecn;
913 bool is_harddrop;
914 bool is_nodrop;
915 struct gnet_stats_queue *qstats;
916};
917
918struct tc_red_qopt_offload {
919 enum tc_red_command command;
920 u32 handle;
921 u32 parent;
922 union {
923 struct tc_red_qopt_offload_params set;
924 struct tc_qopt_offload_stats stats;
925 struct red_stats *xstats;
926 u32 child_handle;
927 };
928};
929
930enum tc_gred_command {
931 TC_GRED_REPLACE,
932 TC_GRED_DESTROY,
933 TC_GRED_STATS,
934};
935
936struct tc_gred_vq_qopt_offload_params {
937 bool present;
938 u32 limit;
939 u32 prio;
940 u32 min;
941 u32 max;
942 bool is_ecn;
943 bool is_harddrop;
944 u32 probability;
945 /* Only need backlog, see struct tc_prio_qopt_offload_params */
946 u32 *backlog;
947};
948
949struct tc_gred_qopt_offload_params {
950 bool grio_on;
951 bool wred_on;
952 unsigned int dp_cnt;
953 unsigned int dp_def;
954 struct gnet_stats_queue *qstats;
955 struct tc_gred_vq_qopt_offload_params tab[MAX_DPs];
956};
957
958struct tc_gred_qopt_offload_stats {
959 struct gnet_stats_basic_sync bstats[MAX_DPs];
960 struct gnet_stats_queue qstats[MAX_DPs];
961 struct red_stats *xstats[MAX_DPs];
962};
963
964struct tc_gred_qopt_offload {
965 enum tc_gred_command command;
966 u32 handle;
967 u32 parent;
968 union {
969 struct tc_gred_qopt_offload_params set;
970 struct tc_gred_qopt_offload_stats stats;
971 };
972};
973
974enum tc_prio_command {
975 TC_PRIO_REPLACE,
976 TC_PRIO_DESTROY,
977 TC_PRIO_STATS,
978 TC_PRIO_GRAFT,
979};
980
981struct tc_prio_qopt_offload_params {
982 int bands;
983 u8 priomap[TC_PRIO_MAX + 1];
984 /* At the point of un-offloading the Qdisc, the reported backlog and
985 * qlen need to be reduced by the portion that is in HW.
986 */
987 struct gnet_stats_queue *qstats;
988};
989
990struct tc_prio_qopt_offload_graft_params {
991 u8 band;
992 u32 child_handle;
993};
994
995struct tc_prio_qopt_offload {
996 enum tc_prio_command command;
997 u32 handle;
998 u32 parent;
999 union {
1000 struct tc_prio_qopt_offload_params replace_params;
1001 struct tc_qopt_offload_stats stats;
1002 struct tc_prio_qopt_offload_graft_params graft_params;
1003 };
1004};
1005
1006enum tc_root_command {
1007 TC_ROOT_GRAFT,
1008};
1009
1010struct tc_root_qopt_offload {
1011 enum tc_root_command command;
1012 u32 handle;
1013 bool ingress;
1014};
1015
1016enum tc_ets_command {
1017 TC_ETS_REPLACE,
1018 TC_ETS_DESTROY,
1019 TC_ETS_STATS,
1020 TC_ETS_GRAFT,
1021};
1022
1023struct tc_ets_qopt_offload_replace_params {
1024 unsigned int bands;
1025 u8 priomap[TC_PRIO_MAX + 1];
1026 unsigned int quanta[TCQ_ETS_MAX_BANDS]; /* 0 for strict bands. */
1027 unsigned int weights[TCQ_ETS_MAX_BANDS];
1028 struct gnet_stats_queue *qstats;
1029};
1030
1031struct tc_ets_qopt_offload_graft_params {
1032 u8 band;
1033 u32 child_handle;
1034};
1035
1036struct tc_ets_qopt_offload {
1037 enum tc_ets_command command;
1038 u32 handle;
1039 u32 parent;
1040 union {
1041 struct tc_ets_qopt_offload_replace_params replace_params;
1042 struct tc_qopt_offload_stats stats;
1043 struct tc_ets_qopt_offload_graft_params graft_params;
1044 };
1045};
1046
1047enum tc_tbf_command {
1048 TC_TBF_REPLACE,
1049 TC_TBF_DESTROY,
1050 TC_TBF_STATS,
1051 TC_TBF_GRAFT,
1052};
1053
1054struct tc_tbf_qopt_offload_replace_params {
1055 struct psched_ratecfg rate;
1056 u32 max_size;
1057 struct gnet_stats_queue *qstats;
1058};
1059
1060struct tc_tbf_qopt_offload {
1061 struct netlink_ext_ack *extack;
1062 enum tc_tbf_command command;
1063 u32 handle;
1064 u32 parent;
1065 union {
1066 struct tc_tbf_qopt_offload_replace_params replace_params;
1067 struct tc_qopt_offload_stats stats;
1068 u32 child_handle;
1069 };
1070};
1071
1072enum tc_fifo_command {
1073 TC_FIFO_REPLACE,
1074 TC_FIFO_DESTROY,
1075 TC_FIFO_STATS,
1076};
1077
1078struct tc_fifo_qopt_offload {
1079 enum tc_fifo_command command;
1080 u32 handle;
1081 u32 parent;
1082 union {
1083 struct tc_qopt_offload_stats stats;
1084 };
1085};
1086
1087#ifdef CONFIG_NET_CLS_ACT
1088DECLARE_STATIC_KEY_FALSE(tc_skb_ext_tc);
1089void tc_skb_ext_tc_enable(void);
1090void tc_skb_ext_tc_disable(void);
1091#define tc_skb_ext_tc_enabled() static_branch_unlikely(&tc_skb_ext_tc)
1092#else /* CONFIG_NET_CLS_ACT */
1093static inline void tc_skb_ext_tc_enable(void) { }
1094static inline void tc_skb_ext_tc_disable(void) { }
1095#define tc_skb_ext_tc_enabled() false
1096#endif
1097
1098#endif