projects/linux·blob master · refs

204 lines · 5.5 KiB

1// SPDX-License-Identifier: GPL-2.0-or-later
2/*
3 * Cryptographic API.
4 *
5 * Cipher operations.
6 *
7 * Copyright (c) 2002 James Morris <[email protected]>
8 * 2002 Adam J. Richter <[email protected]>
9 * 2004 Jean-Luc Cooke <[email protected]>
10 */
11
12#include <crypto/scatterwalk.h>
13#include <linux/kernel.h>
14#include <linux/mm.h>
15#include <linux/module.h>
16#include <linux/scatterlist.h>
17
18void scatterwalk_skip(struct scatter_walk *walk, unsigned int nbytes)
19{
20 struct scatterlist *sg = walk->sg;
21
22 nbytes += walk->offset - sg->offset;
23
24 while (nbytes > sg->length) {
25 nbytes -= sg->length;
26 sg = sg_next(sg);
27 }
28 walk->sg = sg;
29 walk->offset = sg->offset + nbytes;
30}
31EXPORT_SYMBOL_GPL(scatterwalk_skip);
32
33inline void memcpy_from_scatterwalk(void *buf, struct scatter_walk *walk,
34 unsigned int nbytes)
35{
36 do {
37 unsigned int to_copy;
38
39 to_copy = scatterwalk_next(walk, nbytes);
40 memcpy(buf, walk->addr, to_copy);
41 scatterwalk_done_src(walk, to_copy);
42 buf += to_copy;
43 nbytes -= to_copy;
44 } while (nbytes);
45}
46EXPORT_SYMBOL_GPL(memcpy_from_scatterwalk);
47
48inline void memcpy_to_scatterwalk(struct scatter_walk *walk, const void *buf,
49 unsigned int nbytes)
50{
51 do {
52 unsigned int to_copy;
53
54 to_copy = scatterwalk_next(walk, nbytes);
55 memcpy(walk->addr, buf, to_copy);
56 scatterwalk_done_dst(walk, to_copy);
57 buf += to_copy;
58 nbytes -= to_copy;
59 } while (nbytes);
60}
61EXPORT_SYMBOL_GPL(memcpy_to_scatterwalk);
62
63void memcpy_from_sglist(void *buf, struct scatterlist *sg,
64 unsigned int start, unsigned int nbytes)
65{
66 struct scatter_walk walk;
67
68 if (unlikely(nbytes == 0)) /* in case sg == NULL */
69 return;
70
71 scatterwalk_start_at_pos(&walk, sg, start);
72 memcpy_from_scatterwalk(buf, &walk, nbytes);
73}
74EXPORT_SYMBOL_GPL(memcpy_from_sglist);
75
76void memcpy_to_sglist(struct scatterlist *sg, unsigned int start,
77 const void *buf, unsigned int nbytes)
78{
79 struct scatter_walk walk;
80
81 if (unlikely(nbytes == 0)) /* in case sg == NULL */
82 return;
83
84 scatterwalk_start_at_pos(&walk, sg, start);
85 memcpy_to_scatterwalk(&walk, buf, nbytes);
86}
87EXPORT_SYMBOL_GPL(memcpy_to_sglist);
88
89/**
90 * memcpy_sglist() - Copy data from one scatterlist to another
91 * @dst: The destination scatterlist. Can be NULL if @nbytes == 0.
92 * @src: The source scatterlist. Can be NULL if @nbytes == 0.
93 * @nbytes: Number of bytes to copy
94 *
95 * The scatterlists can describe exactly the same memory, in which case this
96 * function is a no-op. No other overlaps are supported.
97 *
98 * Context: Any context
99 */
100void memcpy_sglist(struct scatterlist *dst, struct scatterlist *src,
101 unsigned int nbytes)
102{
103 unsigned int src_offset, dst_offset;
104
105 if (unlikely(nbytes == 0)) /* in case src and/or dst is NULL */
106 return;
107
108 src_offset = src->offset;
109 dst_offset = dst->offset;
110 for (;;) {
111 /* Compute the length to copy this step. */
112 unsigned int len = min3(src->offset + src->length - src_offset,
113 dst->offset + dst->length - dst_offset,
114 nbytes);
115 struct page *src_page = sg_page(src);
116 struct page *dst_page = sg_page(dst);
117 const void *src_virt;
118 void *dst_virt;
119
120 if (IS_ENABLED(CONFIG_HIGHMEM)) {
121 /* HIGHMEM: we may have to actually map the pages. */
122 const unsigned int src_oip = offset_in_page(src_offset);
123 const unsigned int dst_oip = offset_in_page(dst_offset);
124 const unsigned int limit = PAGE_SIZE;
125
126 /* Further limit len to not cross a page boundary. */
127 len = min3(len, limit - src_oip, limit - dst_oip);
128
129 /* Compute the source and destination pages. */
130 src_page += src_offset / PAGE_SIZE;
131 dst_page += dst_offset / PAGE_SIZE;
132
133 if (src_page != dst_page) {
134 /* Copy between different pages. */
135 memcpy_page(dst_page, dst_oip,
136 src_page, src_oip, len);
137 flush_dcache_page(dst_page);
138 } else if (src_oip != dst_oip) {
139 /* Copy between different parts of same page. */
140 dst_virt = kmap_local_page(dst_page);
141 memcpy(dst_virt + dst_oip, dst_virt + src_oip,
142 len);
143 kunmap_local(dst_virt);
144 flush_dcache_page(dst_page);
145 } /* Else, it's the same memory. No action needed. */
146 } else {
147 /*
148 * !HIGHMEM: no mapping needed. Just work in the linear
149 * buffer of each sg entry. Note that we can cross page
150 * boundaries, as they are not significant in this case.
151 */
152 src_virt = page_address(src_page) + src_offset;
153 dst_virt = page_address(dst_page) + dst_offset;
154 if (src_virt != dst_virt) {
155 memcpy(dst_virt, src_virt, len);
156 if (ARCH_IMPLEMENTS_FLUSH_DCACHE_PAGE)
157 __scatterwalk_flush_dcache_pages(
158 dst_page, dst_offset, len);
159 } /* Else, it's the same memory. No action needed. */
160 }
161 nbytes -= len;
162 if (nbytes == 0) /* No more to copy? */
163 break;
164
165 /*
166 * There's more to copy. Advance the offsets by the length
167 * copied this step, and advance the sg entries as needed.
168 */
169 src_offset += len;
170 if (src_offset >= src->offset + src->length) {
171 src = sg_next(src);
172 src_offset = src->offset;
173 }
174 dst_offset += len;
175 if (dst_offset >= dst->offset + dst->length) {
176 dst = sg_next(dst);
177 dst_offset = dst->offset;
178 }
179 }
180}
181EXPORT_SYMBOL_GPL(memcpy_sglist);
182
183struct scatterlist *scatterwalk_ffwd(struct scatterlist dst[2],
184 struct scatterlist *src,
185 unsigned int len)
186{
187 for (;;) {
188 if (!len)
189 return src;
190
191 if (src->length > len)
192 break;
193
194 len -= src->length;
195 src = sg_next(src);
196 }
197
198 sg_init_table(dst, 2);
199 sg_set_page(dst, sg_page(src), src->length - len, src->offset + len);
200 scatterwalk_crypto_chain(dst, sg_next(src), 2);
201
202 return dst;
203}
204EXPORT_SYMBOL_GPL(scatterwalk_ffwd);