projects/linux·blob master · refs

359 lines · 9.4 KiB

1// SPDX-License-Identifier: GPL-2.0
2#include <linux/export.h>
3#include <linux/if_vlan.h>
4#include <net/ip.h>
5#include <net/tso.h>
6#include <linux/dma-mapping.h>
7#include <linux/unaligned.h>
8
9void tso_build_hdr(const struct sk_buff *skb, char *hdr, struct tso_t *tso,
10 int size, bool is_last)
11{
12 int hdr_len = skb_transport_offset(skb) + tso->tlen;
13 int mac_hdr_len = skb_network_offset(skb);
14
15 memcpy(hdr, skb->data, hdr_len);
16 if (!tso->ipv6) {
17 struct iphdr *iph = (void *)(hdr + mac_hdr_len);
18
19 iph->id = htons(tso->ip_id);
20 iph->tot_len = htons(size + hdr_len - mac_hdr_len);
21 tso->ip_id++;
22 } else {
23 struct ipv6hdr *iph = (void *)(hdr + mac_hdr_len);
24
25 iph->payload_len = htons(size + tso->tlen);
26 }
27 hdr += skb_transport_offset(skb);
28 if (tso->tlen != sizeof(struct udphdr)) {
29 struct tcphdr *tcph = (struct tcphdr *)hdr;
30
31 put_unaligned_be32(tso->tcp_seq, &tcph->seq);
32
33 if (!is_last) {
34 /* Clear all special flags for not last packet */
35 tcph->psh = 0;
36 tcph->fin = 0;
37 tcph->rst = 0;
38 }
39 } else {
40 struct udphdr *uh = (struct udphdr *)hdr;
41
42 /* size is after segmentation. */
43 udp_set_len_short(uh, sizeof(*uh) + size);
44 }
45}
46EXPORT_SYMBOL(tso_build_hdr);
47
48void tso_build_data(const struct sk_buff *skb, struct tso_t *tso, int size)
49{
50 tso->tcp_seq += size; /* not worth avoiding this operation for UDP */
51 tso->size -= size;
52 tso->data += size;
53
54 if ((tso->size == 0) &&
55 (tso->next_frag_idx < skb_shinfo(skb)->nr_frags)) {
56 skb_frag_t *frag = &skb_shinfo(skb)->frags[tso->next_frag_idx];
57
58 /* Move to next segment */
59 tso->size = skb_frag_size(frag);
60 tso->data = skb_frag_address(frag);
61 tso->next_frag_idx++;
62 }
63}
64EXPORT_SYMBOL(tso_build_data);
65
66int tso_start(struct sk_buff *skb, struct tso_t *tso)
67{
68 int tlen = skb_is_gso_tcp(skb) ? tcp_hdrlen(skb) : sizeof(struct udphdr);
69 int hdr_len = skb_transport_offset(skb) + tlen;
70
71 tso->tlen = tlen;
72 tso->ip_id = ntohs(ip_hdr(skb)->id);
73 tso->tcp_seq = (tlen != sizeof(struct udphdr)) ? ntohl(tcp_hdr(skb)->seq) : 0;
74 tso->next_frag_idx = 0;
75 tso->ipv6 = vlan_get_protocol(skb) == htons(ETH_P_IPV6);
76
77 /* Build first data */
78 tso->size = skb_headlen(skb) - hdr_len;
79 tso->data = skb->data + hdr_len;
80 if ((tso->size == 0) &&
81 (tso->next_frag_idx < skb_shinfo(skb)->nr_frags)) {
82 skb_frag_t *frag = &skb_shinfo(skb)->frags[tso->next_frag_idx];
83
84 /* Move to next segment */
85 tso->size = skb_frag_size(frag);
86 tso->data = skb_frag_address(frag);
87 tso->next_frag_idx++;
88 }
89 return hdr_len;
90}
91EXPORT_SYMBOL(tso_start);
92
93static int tso_dma_iova_try(struct device *dev, struct tso_dma_map *map,
94 phys_addr_t phys, size_t linear_len,
95 size_t total_len, size_t *offset)
96{
97 const struct sk_buff *skb;
98 unsigned int nr_frags;
99 int i;
100
101 if (!dma_iova_try_alloc(dev, &map->iova_state, phys, total_len))
102 return 1;
103
104 skb = map->skb;
105 nr_frags = skb_shinfo(skb)->nr_frags;
106
107 if (linear_len) {
108 if (dma_iova_link(dev, &map->iova_state,
109 phys, *offset, linear_len,
110 DMA_TO_DEVICE, 0))
111 goto iova_fail;
112 map->linear_len = linear_len;
113 *offset += linear_len;
114 }
115
116 for (i = 0; i < nr_frags; i++) {
117 skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
118 unsigned int frag_len = skb_frag_size(frag);
119
120 if (dma_iova_link(dev, &map->iova_state,
121 skb_frag_phys(frag), *offset,
122 frag_len, DMA_TO_DEVICE, 0)) {
123 map->nr_frags = i;
124 goto iova_fail;
125 }
126 map->frags[i].len = frag_len;
127 *offset += frag_len;
128 map->nr_frags = i + 1;
129 }
130
131 if (dma_iova_sync(dev, &map->iova_state, 0, total_len))
132 goto iova_fail;
133
134 return 0;
135
136iova_fail:
137 dma_iova_destroy(dev, &map->iova_state, *offset,
138 DMA_TO_DEVICE, 0);
139 memset(&map->iova_state, 0, sizeof(map->iova_state));
140
141 /* reset map state */
142 map->frag_idx = -1;
143 map->offset = 0;
144 map->linear_len = 0;
145 map->nr_frags = 0;
146
147 return 1;
148}
149
150/**
151 * tso_dma_map_init - DMA-map GSO payload regions
152 * @map: map struct to initialize
153 * @dev: device for DMA mapping
154 * @skb: the GSO skb
155 * @hdr_len: per-segment header length in bytes
156 *
157 * DMA-maps the linear payload (after headers) and all frags.
158 * Prefers the DMA IOVA API (one contiguous mapping, one IOTLB sync);
159 * falls back to per-region dma_map_phys() when IOVA is not available.
160 * Positions the iterator at byte 0 of the payload.
161 *
162 * Return: 0 on success, -ENOMEM on DMA mapping failure (partial mappings
163 * are cleaned up internally).
164 */
165int tso_dma_map_init(struct tso_dma_map *map, struct device *dev,
166 const struct sk_buff *skb, unsigned int hdr_len)
167{
168 unsigned int linear_len = skb_headlen(skb) - hdr_len;
169 unsigned int nr_frags = skb_shinfo(skb)->nr_frags;
170 size_t total_len = skb->len - hdr_len;
171 size_t offset = 0;
172 phys_addr_t phys;
173 int i;
174
175 map->dev = dev;
176 map->skb = skb;
177 map->hdr_len = hdr_len;
178 map->frag_idx = -1;
179 map->offset = 0;
180 map->iova_offset = 0;
181 map->total_len = total_len;
182 map->linear_len = 0;
183 map->nr_frags = 0;
184 memset(&map->iova_state, 0, sizeof(map->iova_state));
185
186 if (!total_len)
187 return 0;
188
189 if (linear_len)
190 phys = virt_to_phys(skb->data + hdr_len);
191 else
192 phys = skb_frag_phys(&skb_shinfo(skb)->frags[0]);
193
194 if (tso_dma_iova_try(dev, map, phys, linear_len, total_len, &offset)) {
195 /* IOVA path failed, map state was reset. Fallback to
196 * per-region dma_map_phys()
197 */
198 if (linear_len) {
199 map->linear_dma = dma_map_phys(dev, phys, linear_len,
200 DMA_TO_DEVICE, 0);
201 if (dma_mapping_error(dev, map->linear_dma))
202 return -ENOMEM;
203 map->linear_len = linear_len;
204 }
205
206 for (i = 0; i < nr_frags; i++) {
207 skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
208 unsigned int frag_len = skb_frag_size(frag);
209
210 map->frags[i].len = frag_len;
211 map->frags[i].dma = dma_map_phys(dev, skb_frag_phys(frag),
212 frag_len, DMA_TO_DEVICE, 0);
213 if (dma_mapping_error(dev, map->frags[i].dma)) {
214 tso_dma_map_cleanup(map);
215 return -ENOMEM;
216 }
217 map->nr_frags = i + 1;
218 }
219 }
220
221 if (linear_len == 0 && nr_frags > 0)
222 map->frag_idx = 0;
223
224 return 0;
225}
226EXPORT_SYMBOL(tso_dma_map_init);
227
228/**
229 * tso_dma_map_cleanup - unmap all DMA regions in a tso_dma_map
230 * @map: the map to clean up
231 *
232 * Handles both IOVA and fallback paths. For IOVA, calls
233 * dma_iova_destroy(). For fallback, unmaps each region individually.
234 */
235void tso_dma_map_cleanup(struct tso_dma_map *map)
236{
237 int i;
238
239 if (dma_use_iova(&map->iova_state)) {
240 dma_iova_destroy(map->dev, &map->iova_state, map->total_len,
241 DMA_TO_DEVICE, 0);
242 memset(&map->iova_state, 0, sizeof(map->iova_state));
243 } else {
244 if (map->linear_len)
245 dma_unmap_phys(map->dev, map->linear_dma,
246 map->linear_len, DMA_TO_DEVICE, 0);
247
248 for (i = 0; i < map->nr_frags; i++)
249 dma_unmap_phys(map->dev, map->frags[i].dma,
250 map->frags[i].len, DMA_TO_DEVICE, 0);
251 }
252
253 map->linear_len = 0;
254 map->nr_frags = 0;
255}
256EXPORT_SYMBOL(tso_dma_map_cleanup);
257
258/**
259 * tso_dma_map_count - count descriptors for a payload range
260 * @map: the payload map
261 * @len: number of payload bytes in this segment
262 *
263 * Counts how many contiguous DMA region chunks the next @len bytes
264 * will span, without advancing the iterator. On the IOVA path this
265 * is always 1 (contiguous). On the fallback path, uses region sizes
266 * from the current position.
267 *
268 * Return: the number of descriptors needed for @len bytes of payload.
269 */
270unsigned int tso_dma_map_count(struct tso_dma_map *map, unsigned int len)
271{
272 unsigned int offset = map->offset;
273 int idx = map->frag_idx;
274 unsigned int count = 0;
275
276 if (!len)
277 return 0;
278
279 if (dma_use_iova(&map->iova_state))
280 return 1;
281
282 while (len > 0) {
283 unsigned int region_len, chunk;
284
285 if (idx == -1)
286 region_len = map->linear_len;
287 else
288 region_len = map->frags[idx].len;
289
290 chunk = min(len, region_len - offset);
291 len -= chunk;
292 count++;
293 offset = 0;
294 idx++;
295 }
296
297 return count;
298}
299EXPORT_SYMBOL(tso_dma_map_count);
300
301/**
302 * tso_dma_map_next - yield the next DMA address range
303 * @map: the payload map
304 * @addr: output DMA address
305 * @chunk_len: output chunk length
306 * @mapping_len: full DMA mapping length when this chunk starts a new
307 * mapping region, or 0 when continuing a previous one.
308 * On the IOVA path this is always 0 (driver must not
309 * do per-region unmaps; use tso_dma_map_cleanup instead).
310 * @seg_remaining: bytes left in current segment
311 *
312 * Yields the next (dma_addr, chunk_len) pair and advances the iterator.
313 * On the IOVA path, the entire payload is contiguous so each segment
314 * is always a single chunk.
315 *
316 * Return: true if a chunk was yielded, false when @seg_remaining is 0.
317 */
318bool tso_dma_map_next(struct tso_dma_map *map, dma_addr_t *addr,
319 unsigned int *chunk_len, unsigned int *mapping_len,
320 unsigned int seg_remaining)
321{
322 unsigned int region_len, chunk;
323
324 if (!seg_remaining)
325 return false;
326
327 /* IOVA path: contiguous DMA range, no region boundaries */
328 if (dma_use_iova(&map->iova_state)) {
329 *addr = map->iova_state.addr + map->iova_offset;
330 *chunk_len = seg_remaining;
331 *mapping_len = 0;
332 map->iova_offset += seg_remaining;
333 return true;
334 }
335
336 /* Fallback path: per-region iteration */
337
338 if (map->frag_idx == -1) {
339 region_len = map->linear_len;
340 chunk = min(seg_remaining, region_len - map->offset);
341 *addr = map->linear_dma + map->offset;
342 } else {
343 region_len = map->frags[map->frag_idx].len;
344 chunk = min(seg_remaining, region_len - map->offset);
345 *addr = map->frags[map->frag_idx].dma + map->offset;
346 }
347
348 *mapping_len = (map->offset == 0) ? region_len : 0;
349 *chunk_len = chunk;
350 map->offset += chunk;
351
352 if (map->offset >= region_len) {
353 map->frag_idx++;
354 map->offset = 0;
355 }
356
357 return true;
358}
359EXPORT_SYMBOL(tso_dma_map_next);