projects/linux·blob master · refs

2443 lines · 62.4 KiB

1// SPDX-License-Identifier: GPL-2.0-only
2/*
3 * linux/net/sunrpc/xdr.c
4 *
5 * Generic XDR support.
6 *
7 * Copyright (C) 1995, 1996 Olaf Kirch <[email protected]>
8 */
9
10#include <linux/module.h>
11#include <linux/slab.h>
12#include <linux/types.h>
13#include <linux/string.h>
14#include <linux/kernel.h>
15#include <linux/pagemap.h>
16#include <linux/errno.h>
17#include <linux/sunrpc/xdr.h>
18#include <linux/sunrpc/msg_prot.h>
19#include <linux/bvec.h>
20#include <trace/events/sunrpc.h>
21
22static void _copy_to_pages(struct page **, size_t, const char *, size_t);
23
24
25/*
26 * XDR functions for basic NFS types
27 */
28__be32 *
29xdr_encode_netobj(__be32 *p, const struct xdr_netobj *obj)
30{
31 unsigned int quadlen = XDR_QUADLEN(obj->len);
32
33 p[quadlen] = 0; /* zero trailing bytes */
34 *p++ = cpu_to_be32(obj->len);
35 memcpy(p, obj->data, obj->len);
36 return p + XDR_QUADLEN(obj->len);
37}
38EXPORT_SYMBOL_GPL(xdr_encode_netobj);
39
40/**
41 * xdr_encode_opaque_fixed - Encode fixed length opaque data
42 * @p: pointer to current position in XDR buffer.
43 * @ptr: pointer to data to encode (or NULL)
44 * @nbytes: size of data.
45 *
46 * Copy the array of data of length nbytes at ptr to the XDR buffer
47 * at position p, then align to the next 32-bit boundary by padding
48 * with zero bytes (see RFC1832).
49 * Note: if ptr is NULL, only the padding is performed.
50 *
51 * Returns the updated current XDR buffer position
52 *
53 */
54__be32 *xdr_encode_opaque_fixed(__be32 *p, const void *ptr, unsigned int nbytes)
55{
56 if (likely(nbytes != 0)) {
57 unsigned int quadlen = XDR_QUADLEN(nbytes);
58 unsigned int padding = (quadlen << 2) - nbytes;
59
60 if (ptr != NULL)
61 memcpy(p, ptr, nbytes);
62 if (padding != 0)
63 memset((char *)p + nbytes, 0, padding);
64 p += quadlen;
65 }
66 return p;
67}
68EXPORT_SYMBOL_GPL(xdr_encode_opaque_fixed);
69
70/**
71 * xdr_encode_opaque - Encode variable length opaque data
72 * @p: pointer to current position in XDR buffer.
73 * @ptr: pointer to data to encode (or NULL)
74 * @nbytes: size of data.
75 *
76 * Returns the updated current XDR buffer position
77 */
78__be32 *xdr_encode_opaque(__be32 *p, const void *ptr, unsigned int nbytes)
79{
80 *p++ = cpu_to_be32(nbytes);
81 return xdr_encode_opaque_fixed(p, ptr, nbytes);
82}
83EXPORT_SYMBOL_GPL(xdr_encode_opaque);
84
85__be32 *
86xdr_encode_string(__be32 *p, const char *string)
87{
88 return xdr_encode_array(p, string, strlen(string));
89}
90EXPORT_SYMBOL_GPL(xdr_encode_string);
91
92/**
93 * xdr_terminate_string - '\0'-terminate a string residing in an xdr_buf
94 * @buf: XDR buffer where string resides
95 * @len: length of string, in bytes
96 *
97 */
98void xdr_terminate_string(const struct xdr_buf *buf, const u32 len)
99{
100 char *kaddr;
101
102 kaddr = kmap_atomic(buf->pages[0]);
103 kaddr[buf->page_base + len] = '\0';
104 kunmap_atomic(kaddr);
105}
106EXPORT_SYMBOL_GPL(xdr_terminate_string);
107
108size_t xdr_buf_pagecount(const struct xdr_buf *buf)
109{
110 if (!buf->page_len)
111 return 0;
112 return (buf->page_base + buf->page_len + PAGE_SIZE - 1) >> PAGE_SHIFT;
113}
114
115int
116xdr_alloc_bvec(struct xdr_buf *buf, gfp_t gfp)
117{
118 size_t i, n = xdr_buf_pagecount(buf);
119
120 if (n != 0 && buf->bvec == NULL) {
121 buf->bvec = kmalloc_objs(buf->bvec[0], n, gfp);
122 if (!buf->bvec)
123 return -ENOMEM;
124 for (i = 0; i < n; i++) {
125 bvec_set_page(&buf->bvec[i], buf->pages[i], PAGE_SIZE,
126 0);
127 }
128 }
129 return 0;
130}
131
132void
133xdr_free_bvec(struct xdr_buf *buf)
134{
135 kfree(buf->bvec);
136 buf->bvec = NULL;
137}
138
139/**
140 * xdr_buf_to_bvec - Copy components of an xdr_buf into a bio_vec array
141 * @bvec: bio_vec array to populate
142 * @bvec_size: element count of @bvec
143 * @xdr: xdr_buf to be copied
144 *
145 * Returns the number of entries consumed in @bvec on success, or
146 * -ESERVERFAULT when @xdr does not fit within @bvec_size entries.
147 */
148int xdr_buf_to_bvec(struct bio_vec *bvec, unsigned int bvec_size,
149 const struct xdr_buf *xdr)
150{
151 const struct kvec *head = xdr->head;
152 const struct kvec *tail = xdr->tail;
153 unsigned int count = 0;
154
155 if (head->iov_len) {
156 if (unlikely(count >= bvec_size))
157 goto bvec_overflow;
158 bvec_set_virt(bvec++, head->iov_base, head->iov_len);
159 ++count;
160 }
161
162 if (xdr->page_len) {
163 unsigned int offset, len, remaining;
164 struct page **pages = xdr->pages;
165
166 offset = offset_in_page(xdr->page_base);
167 remaining = xdr->page_len;
168 while (remaining > 0) {
169 len = min_t(unsigned int, remaining,
170 PAGE_SIZE - offset);
171 if (unlikely(count >= bvec_size))
172 goto bvec_overflow;
173 bvec_set_page(bvec++, *pages++, len, offset);
174 remaining -= len;
175 offset = 0;
176 ++count;
177 }
178 }
179
180 if (tail->iov_len) {
181 if (unlikely(count >= bvec_size))
182 goto bvec_overflow;
183 bvec_set_virt(bvec, tail->iov_base, tail->iov_len);
184 ++count;
185 }
186
187 return count;
188
189bvec_overflow:
190 pr_warn_once("%s: bio_vec array overflow\n", __func__);
191 return -ESERVERFAULT;
192}
193EXPORT_SYMBOL_GPL(xdr_buf_to_bvec);
194
195/**
196 * xdr_buf_to_sg - Populate a scatterlist from an xdr_buf range
197 * @buf: xdr_buf to map
198 * @offset: starting byte offset within @buf
199 * @len: number of bytes to cover
200 * @sg: scatterlist array initialized with sg_init_table()
201 * @nsg: number of entries available in @sg
202 *
203 * @sg is traversed with sg_next(), so callers may pass a list
204 * assembled with sg_chain().
205 *
206 * Return: on success, the number of scatterlist entries used; the
207 * last used entry is marked with sg_mark_end(). On failure, a
208 * negative errno.
209 */
210int xdr_buf_to_sg(const struct xdr_buf *buf, unsigned int offset,
211 unsigned int len, struct scatterlist *sg, unsigned int nsg)
212{
213 unsigned int page_len, thislen, page_offset;
214 struct scatterlist *cur = sg, *prev = NULL;
215 int nents = 0;
216 int i;
217
218 if (len == 0)
219 return 0;
220
221 if (offset >= buf->head[0].iov_len) {
222 offset -= buf->head[0].iov_len;
223 } else {
224 thislen = min_t(unsigned int,
225 buf->head[0].iov_len - offset, len);
226 if (nents >= nsg)
227 return -ENOSPC;
228 sg_set_buf(cur, buf->head[0].iov_base + offset,
229 thislen);
230 prev = cur;
231 cur = sg_next(cur);
232 nents++;
233 len -= thislen;
234 offset = 0;
235 }
236 if (len == 0)
237 goto done;
238
239 if (offset >= buf->page_len) {
240 offset -= buf->page_len;
241 } else {
242 page_len = min(buf->page_len - offset, len);
243 len -= page_len;
244 page_offset = (offset + buf->page_base) & (PAGE_SIZE - 1);
245 i = (offset + buf->page_base) >> PAGE_SHIFT;
246 thislen = PAGE_SIZE - page_offset;
247 do {
248 if (thislen > page_len)
249 thislen = page_len;
250 if (nents >= nsg)
251 return -ENOSPC;
252 sg_set_page(cur, buf->pages[i],
253 thislen, page_offset);
254 prev = cur;
255 cur = sg_next(cur);
256 nents++;
257 page_len -= thislen;
258 i++;
259 page_offset = 0;
260 thislen = PAGE_SIZE;
261 } while (page_len != 0);
262 offset = 0;
263 }
264 if (len == 0)
265 goto done;
266
267 if (offset < buf->tail[0].iov_len) {
268 thislen = min_t(unsigned int,
269 buf->tail[0].iov_len - offset, len);
270 if (nents >= nsg)
271 return -ENOSPC;
272 sg_set_buf(cur, buf->tail[0].iov_base + offset,
273 thislen);
274 prev = cur;
275 nents++;
276 len -= thislen;
277 }
278 if (len != 0)
279 return -EINVAL;
280
281done:
282 if (prev)
283 sg_mark_end(prev);
284 return nents;
285}
286EXPORT_SYMBOL_GPL(xdr_buf_to_sg);
287
288/*
289 * Count the scatterlist entries needed to cover [offset, offset + len)
290 * within @buf. Mirrors the walk in xdr_buf_to_sg() so the caller can
291 * size an allocation that matches the requested sub-range rather than
292 * the full xdr_buf.
293 */
294static unsigned int xdr_buf_sg_nents(const struct xdr_buf *buf,
295 unsigned int offset, unsigned int len)
296{
297 unsigned int nsg = 0, thislen, page_offset;
298
299 if (len == 0)
300 return 0;
301
302 if (offset < buf->head[0].iov_len) {
303 thislen = min_t(unsigned int,
304 buf->head[0].iov_len - offset, len);
305 nsg++;
306 len -= thislen;
307 offset = 0;
308 } else {
309 offset -= buf->head[0].iov_len;
310 }
311 if (len == 0)
312 return nsg;
313
314 if (offset < buf->page_len) {
315 thislen = min(buf->page_len - offset, len);
316 page_offset = (offset + buf->page_base) & (PAGE_SIZE - 1);
317 nsg += DIV_ROUND_UP(page_offset + thislen, PAGE_SIZE);
318 len -= thislen;
319 offset = 0;
320 } else {
321 offset -= buf->page_len;
322 }
323 if (len == 0)
324 return nsg;
325
326 if (offset < buf->tail[0].iov_len)
327 nsg++;
328 return nsg;
329}
330
331/**
332 * xdr_buf_to_sg_alloc - Populate a scatterlist for an xdr_buf range
333 * @buf: xdr_buf to map
334 * @offset: starting byte offset within @buf
335 * @len: number of bytes to cover
336 * @sg_head: caller-provided scatterlist array (typically stack-allocated)
337 * @sg_head_nents: number of entries in @sg_head
338 * @sg_overflow: OUT: chained extension, or NULL when @sg_head sufficed
339 * @gfp: memory allocation flags for overflow
340 *
341 * Populates @sg_head directly when the xdr_buf fits. When more
342 * entries are needed, an overflow scatterlist is allocated and
343 * chained from @sg_head so that the result is traversable with
344 * sg_next().
345 *
346 * Return: on success, the number of populated scatterlist entries
347 * (counting only data entries, not chain entries). @sg_head is
348 * the head of the resulting list. Caller must kfree @sg_overflow
349 * when done. On failure, a negative errno.
350 */
351int xdr_buf_to_sg_alloc(const struct xdr_buf *buf, unsigned int offset,
352 unsigned int len, struct scatterlist *sg_head,
353 unsigned int sg_head_nents,
354 struct scatterlist **sg_overflow, gfp_t gfp)
355{
356 unsigned int nsg;
357 int ret;
358
359 *sg_overflow = NULL;
360 if (len == 0)
361 return 0;
362
363 nsg = xdr_buf_sg_nents(buf, offset, len);
364 if (nsg == 0)
365 return -EINVAL;
366
367 if (nsg <= sg_head_nents) {
368 sg_init_table(sg_head, nsg);
369 } else {
370 /* +1 replaces the slot sg_chain() consumes as the link. */
371 unsigned int overflow_nents = nsg - sg_head_nents + 1;
372 struct scatterlist *overflow;
373
374 overflow = kmalloc_objs(*overflow, overflow_nents, gfp);
375 if (!overflow)
376 return -ENOMEM;
377
378 sg_init_table(sg_head, sg_head_nents);
379 sg_init_table(overflow, overflow_nents);
380 sg_chain(sg_head, sg_head_nents, overflow);
381 *sg_overflow = overflow;
382 }
383
384 ret = xdr_buf_to_sg(buf, offset, len, sg_head, nsg);
385 if (ret < 0) {
386 kfree(*sg_overflow);
387 *sg_overflow = NULL;
388 }
389 return ret;
390}
391EXPORT_SYMBOL_GPL(xdr_buf_to_sg_alloc);
392
393/**
394 * xdr_inline_pages - Prepare receive buffer for a large reply
395 * @xdr: xdr_buf into which reply will be placed
396 * @offset: expected offset where data payload will start, in bytes
397 * @pages: vector of struct page pointers
398 * @base: offset in first page where receive should start, in bytes
399 * @len: expected size of the upper layer data payload, in bytes
400 *
401 */
402void
403xdr_inline_pages(struct xdr_buf *xdr, unsigned int offset,
404 struct page **pages, unsigned int base, unsigned int len)
405{
406 struct kvec *head = xdr->head;
407 struct kvec *tail = xdr->tail;
408 char *buf = (char *)head->iov_base;
409 unsigned int buflen = head->iov_len;
410
411 head->iov_len = offset;
412
413 xdr->pages = pages;
414 xdr->page_base = base;
415 xdr->page_len = len;
416
417 tail->iov_base = buf + offset;
418 tail->iov_len = buflen - offset;
419 xdr->buflen += len;
420}
421EXPORT_SYMBOL_GPL(xdr_inline_pages);
422
423/*
424 * Helper routines for doing 'memmove' like operations on a struct xdr_buf
425 */
426
427/**
428 * _shift_data_left_pages
429 * @pages: vector of pages containing both the source and dest memory area.
430 * @pgto_base: page vector address of destination
431 * @pgfrom_base: page vector address of source
432 * @len: number of bytes to copy
433 *
434 * Note: the addresses pgto_base and pgfrom_base are both calculated in
435 * the same way:
436 * if a memory area starts at byte 'base' in page 'pages[i]',
437 * then its address is given as (i << PAGE_CACHE_SHIFT) + base
438 * Alse note: pgto_base must be < pgfrom_base, but the memory areas
439 * they point to may overlap.
440 */
441static void
442_shift_data_left_pages(struct page **pages, size_t pgto_base,
443 size_t pgfrom_base, size_t len)
444{
445 struct page **pgfrom, **pgto;
446 char *vfrom, *vto;
447 size_t copy;
448
449 BUG_ON(pgfrom_base <= pgto_base);
450
451 if (!len)
452 return;
453
454 pgto = pages + (pgto_base >> PAGE_SHIFT);
455 pgfrom = pages + (pgfrom_base >> PAGE_SHIFT);
456
457 pgto_base &= ~PAGE_MASK;
458 pgfrom_base &= ~PAGE_MASK;
459
460 do {
461 if (pgto_base >= PAGE_SIZE) {
462 pgto_base = 0;
463 pgto++;
464 }
465 if (pgfrom_base >= PAGE_SIZE){
466 pgfrom_base = 0;
467 pgfrom++;
468 }
469
470 copy = len;
471 if (copy > (PAGE_SIZE - pgto_base))
472 copy = PAGE_SIZE - pgto_base;
473 if (copy > (PAGE_SIZE - pgfrom_base))
474 copy = PAGE_SIZE - pgfrom_base;
475
476 vto = kmap_atomic(*pgto);
477 if (*pgto != *pgfrom) {
478 vfrom = kmap_atomic(*pgfrom);
479 memcpy(vto + pgto_base, vfrom + pgfrom_base, copy);
480 kunmap_atomic(vfrom);
481 } else
482 memmove(vto + pgto_base, vto + pgfrom_base, copy);
483 flush_dcache_page(*pgto);
484 kunmap_atomic(vto);
485
486 pgto_base += copy;
487 pgfrom_base += copy;
488
489 } while ((len -= copy) != 0);
490}
491
492/**
493 * _shift_data_right_pages
494 * @pages: vector of pages containing both the source and dest memory area.
495 * @pgto_base: page vector address of destination
496 * @pgfrom_base: page vector address of source
497 * @len: number of bytes to copy
498 *
499 * Note: the addresses pgto_base and pgfrom_base are both calculated in
500 * the same way:
501 * if a memory area starts at byte 'base' in page 'pages[i]',
502 * then its address is given as (i << PAGE_SHIFT) + base
503 * Also note: pgfrom_base must be < pgto_base, but the memory areas
504 * they point to may overlap.
505 */
506static void
507_shift_data_right_pages(struct page **pages, size_t pgto_base,
508 size_t pgfrom_base, size_t len)
509{
510 struct page **pgfrom, **pgto;
511 char *vfrom, *vto;
512 size_t copy;
513
514 BUG_ON(pgto_base <= pgfrom_base);
515
516 if (!len)
517 return;
518
519 pgto_base += len;
520 pgfrom_base += len;
521
522 pgto = pages + (pgto_base >> PAGE_SHIFT);
523 pgfrom = pages + (pgfrom_base >> PAGE_SHIFT);
524
525 pgto_base &= ~PAGE_MASK;
526 pgfrom_base &= ~PAGE_MASK;
527
528 do {
529 /* Are any pointers crossing a page boundary? */
530 if (pgto_base == 0) {
531 pgto_base = PAGE_SIZE;
532 pgto--;
533 }
534 if (pgfrom_base == 0) {
535 pgfrom_base = PAGE_SIZE;
536 pgfrom--;
537 }
538
539 copy = len;
540 if (copy > pgto_base)
541 copy = pgto_base;
542 if (copy > pgfrom_base)
543 copy = pgfrom_base;
544 pgto_base -= copy;
545 pgfrom_base -= copy;
546
547 vto = kmap_atomic(*pgto);
548 if (*pgto != *pgfrom) {
549 vfrom = kmap_atomic(*pgfrom);
550 memcpy(vto + pgto_base, vfrom + pgfrom_base, copy);
551 kunmap_atomic(vfrom);
552 } else
553 memmove(vto + pgto_base, vto + pgfrom_base, copy);
554 flush_dcache_page(*pgto);
555 kunmap_atomic(vto);
556
557 } while ((len -= copy) != 0);
558}
559
560/**
561 * _copy_to_pages
562 * @pages: array of pages
563 * @pgbase: page vector address of destination
564 * @p: pointer to source data
565 * @len: length
566 *
567 * Copies data from an arbitrary memory location into an array of pages
568 * The copy is assumed to be non-overlapping.
569 */
570static void
571_copy_to_pages(struct page **pages, size_t pgbase, const char *p, size_t len)
572{
573 struct page **pgto;
574 char *vto;
575 size_t copy;
576
577 if (!len)
578 return;
579
580 pgto = pages + (pgbase >> PAGE_SHIFT);
581 pgbase &= ~PAGE_MASK;
582
583 for (;;) {
584 copy = PAGE_SIZE - pgbase;
585 if (copy > len)
586 copy = len;
587
588 vto = kmap_atomic(*pgto);
589 memcpy(vto + pgbase, p, copy);
590 kunmap_atomic(vto);
591
592 len -= copy;
593 if (len == 0)
594 break;
595
596 pgbase += copy;
597 if (pgbase == PAGE_SIZE) {
598 flush_dcache_page(*pgto);
599 pgbase = 0;
600 pgto++;
601 }
602 p += copy;
603 }
604 flush_dcache_page(*pgto);
605}
606
607/**
608 * _copy_from_pages
609 * @p: pointer to destination
610 * @pages: array of pages
611 * @pgbase: offset of source data
612 * @len: length
613 *
614 * Copies data into an arbitrary memory location from an array of pages
615 * The copy is assumed to be non-overlapping.
616 */
617void
618_copy_from_pages(char *p, struct page **pages, size_t pgbase, size_t len)
619{
620 struct page **pgfrom;
621 char *vfrom;
622 size_t copy;
623
624 if (!len)
625 return;
626
627 pgfrom = pages + (pgbase >> PAGE_SHIFT);
628 pgbase &= ~PAGE_MASK;
629
630 do {
631 copy = PAGE_SIZE - pgbase;
632 if (copy > len)
633 copy = len;
634
635 vfrom = kmap_atomic(*pgfrom);
636 memcpy(p, vfrom + pgbase, copy);
637 kunmap_atomic(vfrom);
638
639 pgbase += copy;
640 if (pgbase == PAGE_SIZE) {
641 pgbase = 0;
642 pgfrom++;
643 }
644 p += copy;
645
646 } while ((len -= copy) != 0);
647}
648EXPORT_SYMBOL_GPL(_copy_from_pages);
649
650static void xdr_buf_iov_zero(const struct kvec *iov, unsigned int base,
651 unsigned int len)
652{
653 if (base >= iov->iov_len)
654 return;
655 if (len > iov->iov_len - base)
656 len = iov->iov_len - base;
657 memset(iov->iov_base + base, 0, len);
658}
659
660/**
661 * xdr_buf_pages_zero
662 * @buf: xdr_buf
663 * @pgbase: beginning offset
664 * @len: length
665 */
666static void xdr_buf_pages_zero(const struct xdr_buf *buf, unsigned int pgbase,
667 unsigned int len)
668{
669 struct page **pages = buf->pages;
670 struct page **page;
671 char *vpage;
672 unsigned int zero;
673
674 if (!len)
675 return;
676 if (pgbase >= buf->page_len) {
677 xdr_buf_iov_zero(buf->tail, pgbase - buf->page_len, len);
678 return;
679 }
680 if (pgbase + len > buf->page_len) {
681 xdr_buf_iov_zero(buf->tail, 0, pgbase + len - buf->page_len);
682 len = buf->page_len - pgbase;
683 }
684
685 pgbase += buf->page_base;
686
687 page = pages + (pgbase >> PAGE_SHIFT);
688 pgbase &= ~PAGE_MASK;
689
690 do {
691 zero = PAGE_SIZE - pgbase;
692 if (zero > len)
693 zero = len;
694
695 vpage = kmap_atomic(*page);
696 memset(vpage + pgbase, 0, zero);
697 kunmap_atomic(vpage);
698
699 flush_dcache_page(*page);
700 pgbase = 0;
701 page++;
702
703 } while ((len -= zero) != 0);
704}
705
706static unsigned int xdr_buf_pages_fill_sparse(const struct xdr_buf *buf,
707 unsigned int buflen, gfp_t gfp)
708{
709 unsigned int i, npages, pagelen;
710
711 if (!(buf->flags & XDRBUF_SPARSE_PAGES))
712 return buflen;
713 if (buflen <= buf->head->iov_len)
714 return buflen;
715 pagelen = buflen - buf->head->iov_len;
716 if (pagelen > buf->page_len)
717 pagelen = buf->page_len;
718 npages = (pagelen + buf->page_base + PAGE_SIZE - 1) >> PAGE_SHIFT;
719 for (i = 0; i < npages; i++) {
720 if (!buf->pages[i])
721 continue;
722 buf->pages[i] = alloc_page(gfp);
723 if (likely(buf->pages[i]))
724 continue;
725 buflen -= pagelen;
726 pagelen = i << PAGE_SHIFT;
727 if (pagelen > buf->page_base)
728 buflen += pagelen - buf->page_base;
729 break;
730 }
731 return buflen;
732}
733
734static void xdr_buf_try_expand(struct xdr_buf *buf, unsigned int len)
735{
736 struct kvec *head = buf->head;
737 struct kvec *tail = buf->tail;
738 unsigned int sum = head->iov_len + buf->page_len + tail->iov_len;
739 unsigned int free_space, newlen;
740
741 if (sum > buf->len) {
742 free_space = min_t(unsigned int, sum - buf->len, len);
743 newlen = xdr_buf_pages_fill_sparse(buf, buf->len + free_space,
744 GFP_KERNEL);
745 free_space = newlen - buf->len;
746 buf->len = newlen;
747 len -= free_space;
748 if (!len)
749 return;
750 }
751
752 if (buf->buflen > sum) {
753 /* Expand the tail buffer */
754 free_space = min_t(unsigned int, buf->buflen - sum, len);
755 tail->iov_len += free_space;
756 buf->len += free_space;
757 }
758}
759
760static void xdr_buf_tail_copy_right(const struct xdr_buf *buf,
761 unsigned int base, unsigned int len,
762 unsigned int shift)
763{
764 const struct kvec *tail = buf->tail;
765 unsigned int to = base + shift;
766
767 if (to >= tail->iov_len)
768 return;
769 if (len + to > tail->iov_len)
770 len = tail->iov_len - to;
771 memmove(tail->iov_base + to, tail->iov_base + base, len);
772}
773
774static void xdr_buf_pages_copy_right(const struct xdr_buf *buf,
775 unsigned int base, unsigned int len,
776 unsigned int shift)
777{
778 const struct kvec *tail = buf->tail;
779 unsigned int to = base + shift;
780 unsigned int pglen = 0;
781 unsigned int talen = 0, tato = 0;
782
783 if (base >= buf->page_len)
784 return;
785 if (len > buf->page_len - base)
786 len = buf->page_len - base;
787 if (to >= buf->page_len) {
788 tato = to - buf->page_len;
789 if (tail->iov_len >= len + tato)
790 talen = len;
791 else if (tail->iov_len > tato)
792 talen = tail->iov_len - tato;
793 } else if (len + to >= buf->page_len) {
794 pglen = buf->page_len - to;
795 talen = len - pglen;
796 if (talen > tail->iov_len)
797 talen = tail->iov_len;
798 } else
799 pglen = len;
800
801 _copy_from_pages(tail->iov_base + tato, buf->pages,
802 buf->page_base + base + pglen, talen);
803 _shift_data_right_pages(buf->pages, buf->page_base + to,
804 buf->page_base + base, pglen);
805}
806
807static void xdr_buf_head_copy_right(const struct xdr_buf *buf,
808 unsigned int base, unsigned int len,
809 unsigned int shift)
810{
811 const struct kvec *head = buf->head;
812 const struct kvec *tail = buf->tail;
813 unsigned int to = base + shift;
814 unsigned int pglen = 0, pgto = 0;
815 unsigned int talen = 0, tato = 0;
816
817 if (base >= head->iov_len)
818 return;
819 if (len > head->iov_len - base)
820 len = head->iov_len - base;
821 if (to >= buf->page_len + head->iov_len) {
822 tato = to - buf->page_len - head->iov_len;
823 talen = len;
824 } else if (to >= head->iov_len) {
825 pgto = to - head->iov_len;
826 pglen = len;
827 if (pgto + pglen > buf->page_len) {
828 talen = pgto + pglen - buf->page_len;
829 pglen -= talen;
830 }
831 } else {
832 pglen = len - to;
833 if (pglen > buf->page_len) {
834 talen = pglen - buf->page_len;
835 pglen = buf->page_len;
836 }
837 }
838
839 len -= talen;
840 base += len;
841 if (talen + tato > tail->iov_len)
842 talen = tail->iov_len > tato ? tail->iov_len - tato : 0;
843 memcpy(tail->iov_base + tato, head->iov_base + base, talen);
844
845 len -= pglen;
846 base -= pglen;
847 _copy_to_pages(buf->pages, buf->page_base + pgto, head->iov_base + base,
848 pglen);
849
850 base -= len;
851 memmove(head->iov_base + to, head->iov_base + base, len);
852}
853
854static void xdr_buf_tail_shift_right(const struct xdr_buf *buf,
855 unsigned int base, unsigned int len,
856 unsigned int shift)
857{
858 const struct kvec *tail = buf->tail;
859
860 if (base >= tail->iov_len || !shift || !len)
861 return;
862 xdr_buf_tail_copy_right(buf, base, len, shift);
863}
864
865static void xdr_buf_pages_shift_right(const struct xdr_buf *buf,
866 unsigned int base, unsigned int len,
867 unsigned int shift)
868{
869 if (!shift || !len)
870 return;
871 if (base >= buf->page_len) {
872 xdr_buf_tail_shift_right(buf, base - buf->page_len, len, shift);
873 return;
874 }
875 if (base + len > buf->page_len)
876 xdr_buf_tail_shift_right(buf, 0, base + len - buf->page_len,
877 shift);
878 xdr_buf_pages_copy_right(buf, base, len, shift);
879}
880
881static void xdr_buf_head_shift_right(const struct xdr_buf *buf,
882 unsigned int base, unsigned int len,
883 unsigned int shift)
884{
885 const struct kvec *head = buf->head;
886
887 if (!shift)
888 return;
889 if (base >= head->iov_len) {
890 xdr_buf_pages_shift_right(buf, head->iov_len - base, len,
891 shift);
892 return;
893 }
894 if (base + len > head->iov_len)
895 xdr_buf_pages_shift_right(buf, 0, base + len - head->iov_len,
896 shift);
897 xdr_buf_head_copy_right(buf, base, len, shift);
898}
899
900static void xdr_buf_tail_copy_left(const struct xdr_buf *buf, unsigned int base,
901 unsigned int len, unsigned int shift)
902{
903 const struct kvec *tail = buf->tail;
904
905 if (base >= tail->iov_len)
906 return;
907 if (len > tail->iov_len - base)
908 len = tail->iov_len - base;
909 /* Shift data into head */
910 if (shift > buf->page_len + base) {
911 const struct kvec *head = buf->head;
912 unsigned int hdto =
913 head->iov_len + buf->page_len + base - shift;
914 unsigned int hdlen = len;
915
916 if (WARN_ONCE(shift > head->iov_len + buf->page_len + base,
917 "SUNRPC: Misaligned data.\n"))
918 return;
919 if (hdto + hdlen > head->iov_len)
920 hdlen = head->iov_len - hdto;
921 memcpy(head->iov_base + hdto, tail->iov_base + base, hdlen);
922 base += hdlen;
923 len -= hdlen;
924 if (!len)
925 return;
926 }
927 /* Shift data into pages */
928 if (shift > base) {
929 unsigned int pgto = buf->page_len + base - shift;
930 unsigned int pglen = len;
931
932 if (pgto + pglen > buf->page_len)
933 pglen = buf->page_len - pgto;
934 _copy_to_pages(buf->pages, buf->page_base + pgto,
935 tail->iov_base + base, pglen);
936 base += pglen;
937 len -= pglen;
938 if (!len)
939 return;
940 }
941 memmove(tail->iov_base + base - shift, tail->iov_base + base, len);
942}
943
944static void xdr_buf_pages_copy_left(const struct xdr_buf *buf,
945 unsigned int base, unsigned int len,
946 unsigned int shift)
947{
948 unsigned int pgto;
949
950 if (base >= buf->page_len)
951 return;
952 if (len > buf->page_len - base)
953 len = buf->page_len - base;
954 /* Shift data into head */
955 if (shift > base) {
956 const struct kvec *head = buf->head;
957 unsigned int hdto = head->iov_len + base - shift;
958 unsigned int hdlen = len;
959
960 if (WARN_ONCE(shift > head->iov_len + base,
961 "SUNRPC: Misaligned data.\n"))
962 return;
963 if (hdto + hdlen > head->iov_len)
964 hdlen = head->iov_len - hdto;
965 _copy_from_pages(head->iov_base + hdto, buf->pages,
966 buf->page_base + base, hdlen);
967 base += hdlen;
968 len -= hdlen;
969 if (!len)
970 return;
971 }
972 pgto = base - shift;
973 _shift_data_left_pages(buf->pages, buf->page_base + pgto,
974 buf->page_base + base, len);
975}
976
977static void xdr_buf_tail_shift_left(const struct xdr_buf *buf,
978 unsigned int base, unsigned int len,
979 unsigned int shift)
980{
981 if (!shift || !len)
982 return;
983 xdr_buf_tail_copy_left(buf, base, len, shift);
984}
985
986static void xdr_buf_pages_shift_left(const struct xdr_buf *buf,
987 unsigned int base, unsigned int len,
988 unsigned int shift)
989{
990 if (!shift || !len)
991 return;
992 if (base >= buf->page_len) {
993 xdr_buf_tail_shift_left(buf, base - buf->page_len, len, shift);
994 return;
995 }
996 xdr_buf_pages_copy_left(buf, base, len, shift);
997 len += base;
998 if (len <= buf->page_len)
999 return;
1000 xdr_buf_tail_copy_left(buf, 0, len - buf->page_len, shift);
1001}
1002
1003static void xdr_buf_head_shift_left(const struct xdr_buf *buf,
1004 unsigned int base, unsigned int len,
1005 unsigned int shift)
1006{
1007 const struct kvec *head = buf->head;
1008 unsigned int bytes;
1009
1010 if (!shift || !len)
1011 return;
1012
1013 if (shift > base) {
1014 bytes = (shift - base);
1015 if (bytes >= len)
1016 return;
1017 base += bytes;
1018 len -= bytes;
1019 }
1020
1021 if (base < head->iov_len) {
1022 bytes = min_t(unsigned int, len, head->iov_len - base);
1023 memmove(head->iov_base + (base - shift),
1024 head->iov_base + base, bytes);
1025 base += bytes;
1026 len -= bytes;
1027 }
1028 xdr_buf_pages_shift_left(buf, base - head->iov_len, len, shift);
1029}
1030
1031/**
1032 * xdr_shrink_bufhead
1033 * @buf: xdr_buf
1034 * @len: new length of buf->head[0]
1035 *
1036 * Shrinks XDR buffer's header kvec buf->head[0], setting it to
1037 * 'len' bytes. The extra data is not lost, but is instead
1038 * moved into the inlined pages and/or the tail.
1039 */
1040static unsigned int xdr_shrink_bufhead(struct xdr_buf *buf, unsigned int len)
1041{
1042 struct kvec *head = buf->head;
1043 unsigned int shift, buflen = max(buf->len, len);
1044
1045 WARN_ON_ONCE(len > head->iov_len);
1046 if (head->iov_len > buflen) {
1047 buf->buflen -= head->iov_len - buflen;
1048 head->iov_len = buflen;
1049 }
1050 if (len >= head->iov_len)
1051 return 0;
1052 shift = head->iov_len - len;
1053 xdr_buf_try_expand(buf, shift);
1054 xdr_buf_head_shift_right(buf, len, buflen - len, shift);
1055 head->iov_len = len;
1056 buf->buflen -= shift;
1057 buf->len -= shift;
1058 return shift;
1059}
1060
1061/**
1062 * xdr_shrink_pagelen - shrinks buf->pages to @len bytes
1063 * @buf: xdr_buf
1064 * @len: new page buffer length
1065 *
1066 * The extra data is not lost, but is instead moved into buf->tail.
1067 * Returns the actual number of bytes moved.
1068 */
1069static unsigned int xdr_shrink_pagelen(struct xdr_buf *buf, unsigned int len)
1070{
1071 unsigned int shift, buflen = buf->len - buf->head->iov_len;
1072
1073 WARN_ON_ONCE(len > buf->page_len);
1074 if (buf->head->iov_len >= buf->len || len > buflen)
1075 buflen = len;
1076 if (buf->page_len > buflen) {
1077 buf->buflen -= buf->page_len - buflen;
1078 buf->page_len = buflen;
1079 }
1080 if (len >= buf->page_len)
1081 return 0;
1082 shift = buf->page_len - len;
1083 xdr_buf_try_expand(buf, shift);
1084 xdr_buf_pages_shift_right(buf, len, buflen - len, shift);
1085 buf->page_len = len;
1086 buf->len -= shift;
1087 buf->buflen -= shift;
1088 return shift;
1089}
1090
1091/**
1092 * xdr_stream_pos - Return the current offset from the start of the xdr_stream
1093 * @xdr: pointer to struct xdr_stream
1094 */
1095unsigned int xdr_stream_pos(const struct xdr_stream *xdr)
1096{
1097 return (unsigned int)(XDR_QUADLEN(xdr->buf->len) - xdr->nwords) << 2;
1098}
1099EXPORT_SYMBOL_GPL(xdr_stream_pos);
1100
1101static void xdr_stream_set_pos(struct xdr_stream *xdr, unsigned int pos)
1102{
1103 unsigned int blen = xdr->buf->len;
1104
1105 xdr->nwords = blen > pos ? XDR_QUADLEN(blen) - XDR_QUADLEN(pos) : 0;
1106}
1107
1108static void xdr_stream_page_set_pos(struct xdr_stream *xdr, unsigned int pos)
1109{
1110 xdr_stream_set_pos(xdr, pos + xdr->buf->head[0].iov_len);
1111}
1112
1113/**
1114 * xdr_page_pos - Return the current offset from the start of the xdr pages
1115 * @xdr: pointer to struct xdr_stream
1116 */
1117unsigned int xdr_page_pos(const struct xdr_stream *xdr)
1118{
1119 unsigned int pos = xdr_stream_pos(xdr);
1120
1121 WARN_ON(pos < xdr->buf->head[0].iov_len);
1122 return pos - xdr->buf->head[0].iov_len;
1123}
1124EXPORT_SYMBOL_GPL(xdr_page_pos);
1125
1126/**
1127 * xdr_init_encode - Initialize a struct xdr_stream for sending data.
1128 * @xdr: pointer to xdr_stream struct
1129 * @buf: pointer to XDR buffer in which to encode data
1130 * @p: current pointer inside XDR buffer
1131 * @rqst: pointer to controlling rpc_rqst, for debugging
1132 *
1133 * Note: at the moment the RPC client only passes the length of our
1134 * scratch buffer in the xdr_buf's header kvec. Previously this
1135 * meant we needed to call xdr_adjust_iovec() after encoding the
1136 * data. With the new scheme, the xdr_stream manages the details
1137 * of the buffer length, and takes care of adjusting the kvec
1138 * length for us.
1139 */
1140void xdr_init_encode(struct xdr_stream *xdr, struct xdr_buf *buf, __be32 *p,
1141 struct rpc_rqst *rqst)
1142{
1143 struct kvec *iov = buf->head;
1144 int scratch_len = buf->buflen - buf->page_len - buf->tail[0].iov_len;
1145
1146 xdr_reset_scratch_buffer(xdr);
1147 BUG_ON(scratch_len < 0);
1148 xdr->buf = buf;
1149 xdr->iov = iov;
1150 xdr->p = (__be32 *)((char *)iov->iov_base + iov->iov_len);
1151 xdr->end = (__be32 *)((char *)iov->iov_base + scratch_len);
1152 BUG_ON(iov->iov_len > scratch_len);
1153
1154 if (p != xdr->p && p != NULL) {
1155 size_t len;
1156
1157 BUG_ON(p < xdr->p || p > xdr->end);
1158 len = (char *)p - (char *)xdr->p;
1159 xdr->p = p;
1160 buf->len += len;
1161 iov->iov_len += len;
1162 }
1163 xdr->rqst = rqst;
1164}
1165EXPORT_SYMBOL_GPL(xdr_init_encode);
1166
1167/**
1168 * xdr_init_encode_pages - Initialize an xdr_stream for encoding into pages
1169 * @xdr: pointer to xdr_stream struct
1170 * @buf: pointer to XDR buffer into which to encode data
1171 *
1172 */
1173void xdr_init_encode_pages(struct xdr_stream *xdr, struct xdr_buf *buf)
1174{
1175 xdr_reset_scratch_buffer(xdr);
1176
1177 xdr->buf = buf;
1178 xdr->page_ptr = buf->pages;
1179 xdr->iov = NULL;
1180 xdr->p = page_address(*xdr->page_ptr);
1181 xdr->end = (void *)xdr->p + min_t(u32, buf->buflen, PAGE_SIZE);
1182 xdr->rqst = NULL;
1183}
1184EXPORT_SYMBOL_GPL(xdr_init_encode_pages);
1185
1186/**
1187 * __xdr_commit_encode - Ensure all data is written to buffer
1188 * @xdr: pointer to xdr_stream
1189 *
1190 * We handle encoding across page boundaries by giving the caller a
1191 * temporary location to write to, then later copying the data into
1192 * place; xdr_commit_encode does that copying.
1193 *
1194 * Normally the caller doesn't need to call this directly, as the
1195 * following xdr_reserve_space will do it. But an explicit call may be
1196 * required at the end of encoding, or any other time when the xdr_buf
1197 * data might be read.
1198 */
1199void __xdr_commit_encode(struct xdr_stream *xdr)
1200{
1201 size_t shift = xdr->scratch.iov_len;
1202 void *page;
1203
1204 page = page_address(*xdr->page_ptr);
1205 memcpy(xdr->scratch.iov_base, page, shift);
1206 memmove(page, page + shift, (void *)xdr->p - page);
1207 xdr_reset_scratch_buffer(xdr);
1208}
1209EXPORT_SYMBOL_GPL(__xdr_commit_encode);
1210
1211/*
1212 * The buffer space to be reserved crosses the boundary between
1213 * xdr->buf->head and xdr->buf->pages, or between two pages
1214 * in xdr->buf->pages.
1215 */
1216static noinline __be32 *xdr_get_next_encode_buffer(struct xdr_stream *xdr,
1217 size_t nbytes)
1218{
1219 int space_left;
1220 int frag1bytes, frag2bytes;
1221 void *p;
1222
1223 if (nbytes > PAGE_SIZE)
1224 goto out_overflow; /* Bigger buffers require special handling */
1225 if (xdr->buf->len + nbytes > xdr->buf->buflen)
1226 goto out_overflow; /* Sorry, we're totally out of space */
1227 frag1bytes = (xdr->end - xdr->p) << 2;
1228 frag2bytes = nbytes - frag1bytes;
1229 if (xdr->iov)
1230 xdr->iov->iov_len += frag1bytes;
1231 else
1232 xdr->buf->page_len += frag1bytes;
1233 xdr->page_ptr++;
1234 xdr->iov = NULL;
1235
1236 /*
1237 * If the last encode didn't end exactly on a page boundary, the
1238 * next one will straddle boundaries. Encode into the next
1239 * page, then copy it back later in xdr_commit_encode. We use
1240 * the "scratch" iov to track any temporarily unused fragment of
1241 * space at the end of the previous buffer:
1242 */
1243 xdr_set_scratch_buffer(xdr, xdr->p, frag1bytes);
1244
1245 /*
1246 * xdr->p is where the next encode will start after
1247 * xdr_commit_encode() has shifted this one back:
1248 */
1249 p = page_address(*xdr->page_ptr);
1250 xdr->p = p + frag2bytes;
1251 space_left = xdr->buf->buflen - xdr->buf->len;
1252 if (space_left - frag1bytes >= PAGE_SIZE)
1253 xdr->end = p + PAGE_SIZE;
1254 else
1255 xdr->end = p + space_left - frag1bytes;
1256
1257 xdr->buf->page_len += frag2bytes;
1258 xdr->buf->len += nbytes;
1259 return p;
1260out_overflow:
1261 trace_rpc_xdr_overflow(xdr, nbytes);
1262 return NULL;
1263}
1264
1265/**
1266 * xdr_reserve_space - Reserve buffer space for sending
1267 * @xdr: pointer to xdr_stream
1268 * @nbytes: number of bytes to reserve
1269 *
1270 * Checks that we have enough buffer space to encode 'nbytes' more
1271 * bytes of data. If so, update the total xdr_buf length, and
1272 * adjust the length of the current kvec.
1273 *
1274 * The returned pointer is valid only until the next call to
1275 * xdr_reserve_space() or xdr_commit_encode() on @xdr. The current
1276 * implementation of this API guarantees that space reserved for a
1277 * four-byte data item remains valid until @xdr is destroyed, but
1278 * that might not always be true in the future.
1279 */
1280__be32 * xdr_reserve_space(struct xdr_stream *xdr, size_t nbytes)
1281{
1282 __be32 *p = xdr->p;
1283 __be32 *q;
1284
1285 xdr_commit_encode(xdr);
1286 /* align nbytes on the next 32-bit boundary */
1287 nbytes += 3;
1288 nbytes &= ~3;
1289 q = p + (nbytes >> 2);
1290 if (unlikely(q > xdr->end || q < p))
1291 return xdr_get_next_encode_buffer(xdr, nbytes);
1292 xdr->p = q;
1293 if (xdr->iov)
1294 xdr->iov->iov_len += nbytes;
1295 else
1296 xdr->buf->page_len += nbytes;
1297 xdr->buf->len += nbytes;
1298 return p;
1299}
1300EXPORT_SYMBOL_GPL(xdr_reserve_space);
1301
1302/**
1303 * xdr_reserve_space_vec - Reserves a large amount of buffer space for sending
1304 * @xdr: pointer to xdr_stream
1305 * @nbytes: number of bytes to reserve
1306 *
1307 * The size argument passed to xdr_reserve_space() is determined based
1308 * on the number of bytes remaining in the current page to avoid
1309 * invalidating iov_base pointers when xdr_commit_encode() is called.
1310 *
1311 * Return values:
1312 * %0: success
1313 * %-EMSGSIZE: not enough space is available in @xdr
1314 */
1315int xdr_reserve_space_vec(struct xdr_stream *xdr, size_t nbytes)
1316{
1317 size_t thislen;
1318 __be32 *p;
1319
1320 /*
1321 * svcrdma requires every READ payload to start somewhere
1322 * in xdr->pages.
1323 */
1324 if (xdr->iov == xdr->buf->head) {
1325 xdr->iov = NULL;
1326 xdr->end = xdr->p;
1327 }
1328
1329 /* XXX: Let's find a way to make this more efficient */
1330 while (nbytes) {
1331 thislen = xdr->buf->page_len % PAGE_SIZE;
1332 thislen = min_t(size_t, nbytes, PAGE_SIZE - thislen);
1333
1334 p = xdr_reserve_space(xdr, thislen);
1335 if (!p)
1336 return -EMSGSIZE;
1337
1338 nbytes -= thislen;
1339 }
1340
1341 return 0;
1342}
1343EXPORT_SYMBOL_GPL(xdr_reserve_space_vec);
1344
1345/**
1346 * xdr_truncate_encode - truncate an encode buffer
1347 * @xdr: pointer to xdr_stream
1348 * @len: new length of buffer
1349 *
1350 * Truncates the xdr stream, so that xdr->buf->len == len,
1351 * and xdr->p points at offset len from the start of the buffer, and
1352 * head, tail, and page lengths are adjusted to correspond.
1353 *
1354 * If this means moving xdr->p to a different buffer, we assume that
1355 * the end pointer should be set to the end of the current page,
1356 * except in the case of the head buffer when we assume the head
1357 * buffer's current length represents the end of the available buffer.
1358 *
1359 * This is *not* safe to use on a buffer that already has inlined page
1360 * cache pages (as in a zero-copy server read reply), except for the
1361 * simple case of truncating from one position in the tail to another.
1362 *
1363 */
1364void xdr_truncate_encode(struct xdr_stream *xdr, size_t len)
1365{
1366 struct xdr_buf *buf = xdr->buf;
1367 struct kvec *head = buf->head;
1368 struct kvec *tail = buf->tail;
1369 int fraglen;
1370 int new;
1371
1372 if (len > buf->len) {
1373 WARN_ON_ONCE(1);
1374 return;
1375 }
1376 xdr_commit_encode(xdr);
1377
1378 fraglen = min_t(int, buf->len - len, tail->iov_len);
1379 tail->iov_len -= fraglen;
1380 buf->len -= fraglen;
1381 if (tail->iov_len) {
1382 xdr->p = tail->iov_base + tail->iov_len;
1383 WARN_ON_ONCE(!xdr->end);
1384 WARN_ON_ONCE(!xdr->iov);
1385 return;
1386 }
1387 WARN_ON_ONCE(fraglen);
1388 fraglen = min_t(int, buf->len - len, buf->page_len);
1389 buf->page_len -= fraglen;
1390 buf->len -= fraglen;
1391
1392 new = buf->page_base + buf->page_len;
1393
1394 xdr->page_ptr = buf->pages + (new >> PAGE_SHIFT);
1395
1396 if (buf->page_len) {
1397 xdr->p = page_address(*xdr->page_ptr);
1398 xdr->end = (void *)xdr->p + PAGE_SIZE;
1399 xdr->p = (void *)xdr->p + (new % PAGE_SIZE);
1400 WARN_ON_ONCE(xdr->iov);
1401 return;
1402 }
1403 if (fraglen)
1404 xdr->end = head->iov_base + head->iov_len;
1405 /* (otherwise assume xdr->end is already set) */
1406 xdr->page_ptr--;
1407 head->iov_len = len;
1408 buf->len = len;
1409 xdr->p = head->iov_base + head->iov_len;
1410 xdr->iov = buf->head;
1411}
1412EXPORT_SYMBOL(xdr_truncate_encode);
1413
1414/**
1415 * xdr_truncate_decode - Truncate a decoding stream
1416 * @xdr: pointer to struct xdr_stream
1417 * @len: Number of bytes to remove
1418 *
1419 */
1420void xdr_truncate_decode(struct xdr_stream *xdr, size_t len)
1421{
1422 unsigned int nbytes = xdr_align_size(len);
1423
1424 xdr->buf->len -= nbytes;
1425 xdr->nwords -= XDR_QUADLEN(nbytes);
1426}
1427EXPORT_SYMBOL_GPL(xdr_truncate_decode);
1428
1429/**
1430 * xdr_restrict_buflen - decrease available buffer space
1431 * @xdr: pointer to xdr_stream
1432 * @newbuflen: new maximum number of bytes available
1433 *
1434 * Adjust our idea of how much space is available in the buffer.
1435 * If we've already used too much space in the buffer, returns -1.
1436 * If the available space is already smaller than newbuflen, returns 0
1437 * and does nothing. Otherwise, adjusts xdr->buf->buflen to newbuflen
1438 * and ensures xdr->end is set at most offset newbuflen from the start
1439 * of the buffer.
1440 */
1441int xdr_restrict_buflen(struct xdr_stream *xdr, int newbuflen)
1442{
1443 struct xdr_buf *buf = xdr->buf;
1444 int left_in_this_buf = (void *)xdr->end - (void *)xdr->p;
1445 int end_offset = buf->len + left_in_this_buf;
1446
1447 if (newbuflen < 0 || newbuflen < buf->len)
1448 return -1;
1449 if (newbuflen > buf->buflen)
1450 return 0;
1451 if (newbuflen < end_offset)
1452 xdr->end = (void *)xdr->end + newbuflen - end_offset;
1453 buf->buflen = newbuflen;
1454 return 0;
1455}
1456EXPORT_SYMBOL(xdr_restrict_buflen);
1457
1458/**
1459 * xdr_write_pages - Insert a list of pages into an XDR buffer for sending
1460 * @xdr: pointer to xdr_stream
1461 * @pages: array of pages to insert
1462 * @base: starting offset of first data byte in @pages
1463 * @len: number of data bytes in @pages to insert
1464 *
1465 * After the @pages are added, the tail iovec is instantiated pointing to
1466 * end of the head buffer, and the stream is set up to encode subsequent
1467 * items into the tail.
1468 */
1469void xdr_write_pages(struct xdr_stream *xdr, struct page **pages, unsigned int base,
1470 unsigned int len)
1471{
1472 struct xdr_buf *buf = xdr->buf;
1473 struct kvec *tail = buf->tail;
1474
1475 buf->pages = pages;
1476 buf->page_base = base;
1477 buf->page_len = len;
1478
1479 tail->iov_base = xdr->p;
1480 tail->iov_len = 0;
1481 xdr->iov = tail;
1482
1483 if (len & 3) {
1484 unsigned int pad = 4 - (len & 3);
1485
1486 BUG_ON(xdr->p >= xdr->end);
1487 tail->iov_base = (char *)xdr->p + (len & 3);
1488 tail->iov_len += pad;
1489 len += pad;
1490 *xdr->p++ = 0;
1491 }
1492 buf->buflen += len;
1493 buf->len += len;
1494}
1495EXPORT_SYMBOL_GPL(xdr_write_pages);
1496
1497static unsigned int xdr_set_iov(struct xdr_stream *xdr, struct kvec *iov,
1498 unsigned int base, unsigned int len)
1499{
1500 if (len > iov->iov_len)
1501 len = iov->iov_len;
1502 if (unlikely(base > len))
1503 base = len;
1504 xdr->p = (__be32*)(iov->iov_base + base);
1505 xdr->end = (__be32*)(iov->iov_base + len);
1506 xdr->iov = iov;
1507 xdr->page_ptr = NULL;
1508 return len - base;
1509}
1510
1511static unsigned int xdr_set_tail_base(struct xdr_stream *xdr,
1512 unsigned int base, unsigned int len)
1513{
1514 struct xdr_buf *buf = xdr->buf;
1515
1516 xdr_stream_set_pos(xdr, base + buf->page_len + buf->head->iov_len);
1517 return xdr_set_iov(xdr, buf->tail, base, len);
1518}
1519
1520static void xdr_stream_unmap_current_page(struct xdr_stream *xdr)
1521{
1522 if (xdr->page_kaddr) {
1523 kunmap_local(xdr->page_kaddr);
1524 xdr->page_kaddr = NULL;
1525 }
1526}
1527
1528static unsigned int xdr_set_page_base(struct xdr_stream *xdr,
1529 unsigned int base, unsigned int len)
1530{
1531 unsigned int pgnr;
1532 unsigned int maxlen;
1533 unsigned int pgoff;
1534 unsigned int pgend;
1535 void *kaddr;
1536
1537 maxlen = xdr->buf->page_len;
1538 if (base >= maxlen)
1539 return 0;
1540 else
1541 maxlen -= base;
1542 if (len > maxlen)
1543 len = maxlen;
1544
1545 xdr_stream_unmap_current_page(xdr);
1546 xdr_stream_page_set_pos(xdr, base);
1547 base += xdr->buf->page_base;
1548
1549 pgnr = base >> PAGE_SHIFT;
1550 xdr->page_ptr = &xdr->buf->pages[pgnr];
1551
1552 if (PageHighMem(*xdr->page_ptr)) {
1553 xdr->page_kaddr = kmap_local_page(*xdr->page_ptr);
1554 kaddr = xdr->page_kaddr;
1555 } else
1556 kaddr = page_address(*xdr->page_ptr);
1557
1558 pgoff = base & ~PAGE_MASK;
1559 xdr->p = (__be32*)(kaddr + pgoff);
1560
1561 pgend = pgoff + len;
1562 if (pgend > PAGE_SIZE)
1563 pgend = PAGE_SIZE;
1564 xdr->end = (__be32*)(kaddr + pgend);
1565 xdr->iov = NULL;
1566 return len;
1567}
1568
1569static void xdr_set_page(struct xdr_stream *xdr, unsigned int base,
1570 unsigned int len)
1571{
1572 if (xdr_set_page_base(xdr, base, len) == 0) {
1573 base -= xdr->buf->page_len;
1574 xdr_set_tail_base(xdr, base, len);
1575 }
1576}
1577
1578static void xdr_set_next_page(struct xdr_stream *xdr)
1579{
1580 unsigned int newbase;
1581
1582 newbase = (1 + xdr->page_ptr - xdr->buf->pages) << PAGE_SHIFT;
1583 newbase -= xdr->buf->page_base;
1584 if (newbase < xdr->buf->page_len)
1585 xdr_set_page_base(xdr, newbase, xdr_stream_remaining(xdr));
1586 else
1587 xdr_set_tail_base(xdr, 0, xdr_stream_remaining(xdr));
1588}
1589
1590static bool xdr_set_next_buffer(struct xdr_stream *xdr)
1591{
1592 if (xdr->page_ptr != NULL)
1593 xdr_set_next_page(xdr);
1594 else if (xdr->iov == xdr->buf->head)
1595 xdr_set_page(xdr, 0, xdr_stream_remaining(xdr));
1596 return xdr->p != xdr->end;
1597}
1598
1599/**
1600 * xdr_init_decode - Initialize an xdr_stream for decoding data.
1601 * @xdr: pointer to xdr_stream struct
1602 * @buf: pointer to XDR buffer from which to decode data
1603 * @p: current pointer inside XDR buffer
1604 * @rqst: pointer to controlling rpc_rqst, for debugging
1605 */
1606void xdr_init_decode(struct xdr_stream *xdr, struct xdr_buf *buf, __be32 *p,
1607 struct rpc_rqst *rqst)
1608{
1609 xdr->buf = buf;
1610 xdr->page_kaddr = NULL;
1611 xdr_reset_scratch_buffer(xdr);
1612 xdr->nwords = XDR_QUADLEN(buf->len);
1613 if (xdr_set_iov(xdr, buf->head, 0, buf->len) == 0 &&
1614 xdr_set_page_base(xdr, 0, buf->len) == 0)
1615 xdr_set_iov(xdr, buf->tail, 0, buf->len);
1616 if (p != NULL && p > xdr->p && xdr->end >= p) {
1617 xdr->nwords -= p - xdr->p;
1618 xdr->p = p;
1619 }
1620 xdr->rqst = rqst;
1621}
1622EXPORT_SYMBOL_GPL(xdr_init_decode);
1623
1624/**
1625 * xdr_init_decode_pages - Initialize an xdr_stream for decoding into pages
1626 * @xdr: pointer to xdr_stream struct
1627 * @buf: pointer to XDR buffer from which to decode data
1628 * @pages: list of pages to decode into
1629 * @len: length in bytes of buffer in pages
1630 */
1631void xdr_init_decode_pages(struct xdr_stream *xdr, struct xdr_buf *buf,
1632 struct page **pages, unsigned int len)
1633{
1634 memset(buf, 0, sizeof(*buf));
1635 buf->pages = pages;
1636 buf->page_len = len;
1637 buf->buflen = len;
1638 buf->len = len;
1639 xdr_init_decode(xdr, buf, NULL, NULL);
1640}
1641EXPORT_SYMBOL_GPL(xdr_init_decode_pages);
1642
1643/**
1644 * xdr_finish_decode - Clean up the xdr_stream after decoding data.
1645 * @xdr: pointer to xdr_stream struct
1646 */
1647void xdr_finish_decode(struct xdr_stream *xdr)
1648{
1649 xdr_stream_unmap_current_page(xdr);
1650}
1651EXPORT_SYMBOL(xdr_finish_decode);
1652
1653static __be32 * __xdr_inline_decode(struct xdr_stream *xdr, size_t nbytes)
1654{
1655 unsigned int nwords = XDR_QUADLEN(nbytes);
1656 __be32 *p = xdr->p;
1657 __be32 *q = p + nwords;
1658
1659 if (unlikely(nwords > xdr->nwords || q > xdr->end || q < p))
1660 return NULL;
1661 xdr->p = q;
1662 xdr->nwords -= nwords;
1663 return p;
1664}
1665
1666static __be32 *xdr_copy_to_scratch(struct xdr_stream *xdr, size_t nbytes)
1667{
1668 __be32 *p;
1669 char *cpdest = xdr->scratch.iov_base;
1670 size_t cplen = (char *)xdr->end - (char *)xdr->p;
1671
1672 if (nbytes > xdr->scratch.iov_len)
1673 goto out_overflow;
1674 p = __xdr_inline_decode(xdr, cplen);
1675 if (p == NULL)
1676 return NULL;
1677 memcpy(cpdest, p, cplen);
1678 if (!xdr_set_next_buffer(xdr))
1679 goto out_overflow;
1680 cpdest += cplen;
1681 nbytes -= cplen;
1682 p = __xdr_inline_decode(xdr, nbytes);
1683 if (p == NULL)
1684 return NULL;
1685 memcpy(cpdest, p, nbytes);
1686 return xdr->scratch.iov_base;
1687out_overflow:
1688 trace_rpc_xdr_overflow(xdr, nbytes);
1689 return NULL;
1690}
1691
1692/**
1693 * xdr_inline_decode - Retrieve XDR data to decode
1694 * @xdr: pointer to xdr_stream struct
1695 * @nbytes: number of bytes of data to decode
1696 *
1697 * Check if the input buffer is long enough to enable us to decode
1698 * 'nbytes' more bytes of data starting at the current position.
1699 * If so return the current pointer, then update the current
1700 * pointer position.
1701 */
1702__be32 * xdr_inline_decode(struct xdr_stream *xdr, size_t nbytes)
1703{
1704 __be32 *p;
1705
1706 if (unlikely(nbytes == 0))
1707 return xdr->p;
1708 if (xdr->p == xdr->end && !xdr_set_next_buffer(xdr))
1709 goto out_overflow;
1710 p = __xdr_inline_decode(xdr, nbytes);
1711 if (p != NULL)
1712 return p;
1713 return xdr_copy_to_scratch(xdr, nbytes);
1714out_overflow:
1715 trace_rpc_xdr_overflow(xdr, nbytes);
1716 return NULL;
1717}
1718EXPORT_SYMBOL_GPL(xdr_inline_decode);
1719
1720static void xdr_realign_pages(struct xdr_stream *xdr)
1721{
1722 struct xdr_buf *buf = xdr->buf;
1723 struct kvec *iov = buf->head;
1724 unsigned int cur = xdr_stream_pos(xdr);
1725 unsigned int copied;
1726
1727 /* Realign pages to current pointer position */
1728 if (iov->iov_len > cur) {
1729 copied = xdr_shrink_bufhead(buf, cur);
1730 trace_rpc_xdr_alignment(xdr, cur, copied);
1731 xdr_set_page(xdr, 0, buf->page_len);
1732 }
1733}
1734
1735static unsigned int xdr_align_pages(struct xdr_stream *xdr, unsigned int len)
1736{
1737 struct xdr_buf *buf = xdr->buf;
1738 unsigned int nwords = XDR_QUADLEN(len);
1739 unsigned int copied;
1740
1741 if (xdr->nwords == 0)
1742 return 0;
1743
1744 xdr_realign_pages(xdr);
1745 if (nwords > xdr->nwords) {
1746 nwords = xdr->nwords;
1747 len = nwords << 2;
1748 }
1749 if (buf->page_len <= len)
1750 len = buf->page_len;
1751 else if (nwords < xdr->nwords) {
1752 /* Truncate page data and move it into the tail */
1753 copied = xdr_shrink_pagelen(buf, len);
1754 trace_rpc_xdr_alignment(xdr, len, copied);
1755 }
1756 return len;
1757}
1758
1759/**
1760 * xdr_read_pages - align page-based XDR data to current pointer position
1761 * @xdr: pointer to xdr_stream struct
1762 * @len: number of bytes of page data
1763 *
1764 * Moves data beyond the current pointer position from the XDR head[] buffer
1765 * into the page list. Any data that lies beyond current position + @len
1766 * bytes is moved into the XDR tail[]. The xdr_stream current position is
1767 * then advanced past that data to align to the next XDR object in the tail.
1768 *
1769 * Returns the number of XDR encoded bytes now contained in the pages
1770 */
1771unsigned int xdr_read_pages(struct xdr_stream *xdr, unsigned int len)
1772{
1773 unsigned int nwords = XDR_QUADLEN(len);
1774 unsigned int base, end, pglen;
1775
1776 pglen = xdr_align_pages(xdr, nwords << 2);
1777 if (pglen == 0)
1778 return 0;
1779
1780 base = (nwords << 2) - pglen;
1781 end = xdr_stream_remaining(xdr) - pglen;
1782
1783 xdr_set_tail_base(xdr, base, end);
1784 return len <= pglen ? len : pglen;
1785}
1786EXPORT_SYMBOL_GPL(xdr_read_pages);
1787
1788/**
1789 * xdr_set_pagelen - Sets the length of the XDR pages
1790 * @xdr: pointer to xdr_stream struct
1791 * @len: new length of the XDR page data
1792 *
1793 * Either grows or shrinks the length of the xdr pages by setting pagelen to
1794 * @len bytes. When shrinking, any extra data is moved into buf->tail, whereas
1795 * when growing any data beyond the current pointer is moved into the tail.
1796 *
1797 * Returns True if the operation was successful, and False otherwise.
1798 */
1799void xdr_set_pagelen(struct xdr_stream *xdr, unsigned int len)
1800{
1801 struct xdr_buf *buf = xdr->buf;
1802 size_t remaining = xdr_stream_remaining(xdr);
1803 size_t base = 0;
1804
1805 if (len < buf->page_len) {
1806 base = buf->page_len - len;
1807 xdr_shrink_pagelen(buf, len);
1808 } else {
1809 xdr_buf_head_shift_right(buf, xdr_stream_pos(xdr),
1810 buf->page_len, remaining);
1811 if (len > buf->page_len)
1812 xdr_buf_try_expand(buf, len - buf->page_len);
1813 }
1814 xdr_set_tail_base(xdr, base, remaining);
1815}
1816EXPORT_SYMBOL_GPL(xdr_set_pagelen);
1817
1818/**
1819 * xdr_enter_page - decode data from the XDR page
1820 * @xdr: pointer to xdr_stream struct
1821 * @len: number of bytes of page data
1822 *
1823 * Moves data beyond the current pointer position from the XDR head[] buffer
1824 * into the page list. Any data that lies beyond current position + "len"
1825 * bytes is moved into the XDR tail[]. The current pointer is then
1826 * repositioned at the beginning of the first XDR page.
1827 */
1828void xdr_enter_page(struct xdr_stream *xdr, unsigned int len)
1829{
1830 len = xdr_align_pages(xdr, len);
1831 /*
1832 * Position current pointer at beginning of tail, and
1833 * set remaining message length.
1834 */
1835 if (len != 0)
1836 xdr_set_page_base(xdr, 0, len);
1837}
1838EXPORT_SYMBOL_GPL(xdr_enter_page);
1839
1840static const struct kvec empty_iov = {.iov_base = NULL, .iov_len = 0};
1841
1842void xdr_buf_from_iov(const struct kvec *iov, struct xdr_buf *buf)
1843{
1844 buf->head[0] = *iov;
1845 buf->tail[0] = empty_iov;
1846 buf->page_len = 0;
1847 buf->buflen = buf->len = iov->iov_len;
1848}
1849EXPORT_SYMBOL_GPL(xdr_buf_from_iov);
1850
1851/**
1852 * xdr_buf_subsegment - set subbuf to a portion of buf
1853 * @buf: an xdr buffer
1854 * @subbuf: the result buffer
1855 * @base: beginning of range in bytes
1856 * @len: length of range in bytes
1857 *
1858 * sets @subbuf to an xdr buffer representing the portion of @buf of
1859 * length @len starting at offset @base.
1860 *
1861 * @buf and @subbuf may be pointers to the same struct xdr_buf.
1862 *
1863 * Returns -1 if base or length are out of bounds.
1864 */
1865int xdr_buf_subsegment(const struct xdr_buf *buf, struct xdr_buf *subbuf,
1866 unsigned int base, unsigned int len)
1867{
1868 subbuf->buflen = subbuf->len = len;
1869 if (base < buf->head[0].iov_len) {
1870 subbuf->head[0].iov_base = buf->head[0].iov_base + base;
1871 subbuf->head[0].iov_len = min_t(unsigned int, len,
1872 buf->head[0].iov_len - base);
1873 len -= subbuf->head[0].iov_len;
1874 base = 0;
1875 } else {
1876 base -= buf->head[0].iov_len;
1877 subbuf->head[0].iov_base = buf->head[0].iov_base;
1878 subbuf->head[0].iov_len = 0;
1879 }
1880
1881 if (base < buf->page_len) {
1882 subbuf->page_len = min(buf->page_len - base, len);
1883 base += buf->page_base;
1884 subbuf->page_base = base & ~PAGE_MASK;
1885 subbuf->pages = &buf->pages[base >> PAGE_SHIFT];
1886 len -= subbuf->page_len;
1887 base = 0;
1888 } else {
1889 base -= buf->page_len;
1890 subbuf->pages = buf->pages;
1891 subbuf->page_base = 0;
1892 subbuf->page_len = 0;
1893 }
1894
1895 if (base < buf->tail[0].iov_len) {
1896 subbuf->tail[0].iov_base = buf->tail[0].iov_base + base;
1897 subbuf->tail[0].iov_len = min_t(unsigned int, len,
1898 buf->tail[0].iov_len - base);
1899 len -= subbuf->tail[0].iov_len;
1900 base = 0;
1901 } else {
1902 base -= buf->tail[0].iov_len;
1903 subbuf->tail[0].iov_base = buf->tail[0].iov_base;
1904 subbuf->tail[0].iov_len = 0;
1905 }
1906
1907 if (base || len)
1908 return -1;
1909 return 0;
1910}
1911EXPORT_SYMBOL_GPL(xdr_buf_subsegment);
1912
1913/**
1914 * xdr_stream_subsegment - set @subbuf to a portion of @xdr
1915 * @xdr: an xdr_stream set up for decoding
1916 * @subbuf: the result buffer
1917 * @nbytes: length of @xdr to extract, in bytes
1918 *
1919 * Sets up @subbuf to represent a portion of @xdr. The portion
1920 * starts at the current offset in @xdr, and extends for a length
1921 * of @nbytes. If this is successful, @xdr is advanced to the next
1922 * XDR data item following that portion.
1923 *
1924 * Return values:
1925 * %true: @subbuf has been initialized, and @xdr has been advanced.
1926 * %false: a bounds error has occurred
1927 */
1928bool xdr_stream_subsegment(struct xdr_stream *xdr, struct xdr_buf *subbuf,
1929 unsigned int nbytes)
1930{
1931 unsigned int start = xdr_stream_pos(xdr);
1932 unsigned int remaining, len;
1933
1934 /* Extract @subbuf and bounds-check the fn arguments */
1935 if (xdr_buf_subsegment(xdr->buf, subbuf, start, nbytes))
1936 return false;
1937
1938 /* Advance @xdr by @nbytes */
1939 for (remaining = nbytes; remaining;) {
1940 if (xdr->p == xdr->end && !xdr_set_next_buffer(xdr))
1941 return false;
1942
1943 len = (char *)xdr->end - (char *)xdr->p;
1944 if (remaining <= len) {
1945 xdr->p = (__be32 *)((char *)xdr->p +
1946 (remaining + xdr_pad_size(nbytes)));
1947 break;
1948 }
1949
1950 xdr->p = (__be32 *)((char *)xdr->p + len);
1951 xdr->end = xdr->p;
1952 remaining -= len;
1953 }
1954
1955 xdr_stream_set_pos(xdr, start + nbytes);
1956 return true;
1957}
1958EXPORT_SYMBOL_GPL(xdr_stream_subsegment);
1959
1960/**
1961 * xdr_stream_move_subsegment - Move part of a stream to another position
1962 * @xdr: the source xdr_stream
1963 * @offset: the source offset of the segment
1964 * @target: the target offset of the segment
1965 * @length: the number of bytes to move
1966 *
1967 * Moves @length bytes from @offset to @target in the xdr_stream, overwriting
1968 * anything in its space. Returns the number of bytes in the segment.
1969 */
1970unsigned int xdr_stream_move_subsegment(struct xdr_stream *xdr, unsigned int offset,
1971 unsigned int target, unsigned int length)
1972{
1973 struct xdr_buf buf;
1974 unsigned int shift;
1975
1976 if (offset < target) {
1977 shift = target - offset;
1978 if (xdr_buf_subsegment(xdr->buf, &buf, offset, shift + length) < 0)
1979 return 0;
1980 xdr_buf_head_shift_right(&buf, 0, length, shift);
1981 } else if (offset > target) {
1982 shift = offset - target;
1983 if (xdr_buf_subsegment(xdr->buf, &buf, target, shift + length) < 0)
1984 return 0;
1985 xdr_buf_head_shift_left(&buf, shift, length, shift);
1986 }
1987 return length;
1988}
1989EXPORT_SYMBOL_GPL(xdr_stream_move_subsegment);
1990
1991/**
1992 * xdr_stream_zero - zero out a portion of an xdr_stream
1993 * @xdr: an xdr_stream to zero out
1994 * @offset: the starting point in the stream
1995 * @length: the number of bytes to zero
1996 */
1997unsigned int xdr_stream_zero(struct xdr_stream *xdr, unsigned int offset,
1998 unsigned int length)
1999{
2000 struct xdr_buf buf;
2001
2002 if (xdr_buf_subsegment(xdr->buf, &buf, offset, length) < 0)
2003 return 0;
2004 if (buf.head[0].iov_len)
2005 xdr_buf_iov_zero(buf.head, 0, buf.head[0].iov_len);
2006 if (buf.page_len > 0)
2007 xdr_buf_pages_zero(&buf, 0, buf.page_len);
2008 if (buf.tail[0].iov_len)
2009 xdr_buf_iov_zero(buf.tail, 0, buf.tail[0].iov_len);
2010 return length;
2011}
2012EXPORT_SYMBOL_GPL(xdr_stream_zero);
2013
2014/**
2015 * xdr_buf_trim - lop at most "len" bytes off the end of "buf"
2016 * @buf: buf to be trimmed
2017 * @len: number of bytes to reduce "buf" by
2018 *
2019 * Trim an xdr_buf by the given number of bytes by fixing up the lengths. Note
2020 * that it's possible that we'll trim less than that amount if the xdr_buf is
2021 * too small, or if (for instance) it's all in the head and the parser has
2022 * already read too far into it.
2023 */
2024void xdr_buf_trim(struct xdr_buf *buf, unsigned int len)
2025{
2026 size_t cur;
2027 unsigned int trim = len;
2028
2029 if (buf->tail[0].iov_len) {
2030 cur = min_t(size_t, buf->tail[0].iov_len, trim);
2031 buf->tail[0].iov_len -= cur;
2032 trim -= cur;
2033 if (!trim)
2034 goto fix_len;
2035 }
2036
2037 if (buf->page_len) {
2038 cur = min_t(unsigned int, buf->page_len, trim);
2039 buf->page_len -= cur;
2040 trim -= cur;
2041 if (!trim)
2042 goto fix_len;
2043 }
2044
2045 if (buf->head[0].iov_len) {
2046 cur = min_t(size_t, buf->head[0].iov_len, trim);
2047 buf->head[0].iov_len -= cur;
2048 trim -= cur;
2049 }
2050fix_len:
2051 buf->len -= min_t(unsigned int, buf->len, len - trim);
2052}
2053EXPORT_SYMBOL_GPL(xdr_buf_trim);
2054
2055static void __read_bytes_from_xdr_buf(const struct xdr_buf *subbuf,
2056 void *obj, unsigned int len)
2057{
2058 unsigned int this_len;
2059
2060 this_len = min_t(unsigned int, len, subbuf->head[0].iov_len);
2061 memcpy(obj, subbuf->head[0].iov_base, this_len);
2062 len -= this_len;
2063 obj += this_len;
2064 this_len = min_t(unsigned int, len, subbuf->page_len);
2065 _copy_from_pages(obj, subbuf->pages, subbuf->page_base, this_len);
2066 len -= this_len;
2067 obj += this_len;
2068 this_len = min_t(unsigned int, len, subbuf->tail[0].iov_len);
2069 memcpy(obj, subbuf->tail[0].iov_base, this_len);
2070}
2071
2072/* obj is assumed to point to allocated memory of size at least len: */
2073int read_bytes_from_xdr_buf(const struct xdr_buf *buf, unsigned int base,
2074 void *obj, unsigned int len)
2075{
2076 struct xdr_buf subbuf;
2077 int status;
2078
2079 status = xdr_buf_subsegment(buf, &subbuf, base, len);
2080 if (status != 0)
2081 return status;
2082 __read_bytes_from_xdr_buf(&subbuf, obj, len);
2083 return 0;
2084}
2085EXPORT_SYMBOL_GPL(read_bytes_from_xdr_buf);
2086
2087static void __write_bytes_to_xdr_buf(const struct xdr_buf *subbuf,
2088 void *obj, unsigned int len)
2089{
2090 unsigned int this_len;
2091
2092 this_len = min_t(unsigned int, len, subbuf->head[0].iov_len);
2093 memcpy(subbuf->head[0].iov_base, obj, this_len);
2094 len -= this_len;
2095 obj += this_len;
2096 this_len = min_t(unsigned int, len, subbuf->page_len);
2097 _copy_to_pages(subbuf->pages, subbuf->page_base, obj, this_len);
2098 len -= this_len;
2099 obj += this_len;
2100 this_len = min_t(unsigned int, len, subbuf->tail[0].iov_len);
2101 memcpy(subbuf->tail[0].iov_base, obj, this_len);
2102}
2103
2104/* obj is assumed to point to allocated memory of size at least len: */
2105int write_bytes_to_xdr_buf(const struct xdr_buf *buf, unsigned int base,
2106 void *obj, unsigned int len)
2107{
2108 struct xdr_buf subbuf;
2109 int status;
2110
2111 status = xdr_buf_subsegment(buf, &subbuf, base, len);
2112 if (status != 0)
2113 return status;
2114 __write_bytes_to_xdr_buf(&subbuf, obj, len);
2115 return 0;
2116}
2117EXPORT_SYMBOL_GPL(write_bytes_to_xdr_buf);
2118
2119int xdr_decode_word(const struct xdr_buf *buf, unsigned int base, u32 *obj)
2120{
2121 __be32 raw;
2122 int status;
2123
2124 status = read_bytes_from_xdr_buf(buf, base, &raw, sizeof(*obj));
2125 if (status)
2126 return status;
2127 *obj = be32_to_cpu(raw);
2128 return 0;
2129}
2130EXPORT_SYMBOL_GPL(xdr_decode_word);
2131
2132int xdr_encode_word(const struct xdr_buf *buf, unsigned int base, u32 obj)
2133{
2134 __be32 raw = cpu_to_be32(obj);
2135
2136 return write_bytes_to_xdr_buf(buf, base, &raw, sizeof(obj));
2137}
2138EXPORT_SYMBOL_GPL(xdr_encode_word);
2139
2140/* Returns 0 on success, or else a negative error code. */
2141static int xdr_xcode_array2(const struct xdr_buf *buf, unsigned int base,
2142 struct xdr_array2_desc *desc, int encode)
2143{
2144 char *elem = NULL, *c;
2145 unsigned int copied = 0, todo, avail_here;
2146 struct page **ppages = NULL;
2147 int err;
2148
2149 if (encode) {
2150 if (xdr_encode_word(buf, base, desc->array_len) != 0)
2151 return -EINVAL;
2152 } else {
2153 if (xdr_decode_word(buf, base, &desc->array_len) != 0 ||
2154 desc->array_len > desc->array_maxlen ||
2155 (unsigned long) base + 4 + desc->array_len *
2156 desc->elem_size > buf->len)
2157 return -EINVAL;
2158 }
2159 base += 4;
2160
2161 if (!desc->xcode)
2162 return 0;
2163
2164 todo = desc->array_len * desc->elem_size;
2165
2166 /* process head */
2167 if (todo && base < buf->head->iov_len) {
2168 c = buf->head->iov_base + base;
2169 avail_here = min_t(unsigned int, todo,
2170 buf->head->iov_len - base);
2171 todo -= avail_here;
2172
2173 while (avail_here >= desc->elem_size) {
2174 err = desc->xcode(desc, c);
2175 if (err)
2176 goto out;
2177 c += desc->elem_size;
2178 avail_here -= desc->elem_size;
2179 }
2180 if (avail_here) {
2181 if (!elem) {
2182 elem = kmalloc(desc->elem_size, GFP_KERNEL);
2183 err = -ENOMEM;
2184 if (!elem)
2185 goto out;
2186 }
2187 if (encode) {
2188 err = desc->xcode(desc, elem);
2189 if (err)
2190 goto out;
2191 memcpy(c, elem, avail_here);
2192 } else
2193 memcpy(elem, c, avail_here);
2194 copied = avail_here;
2195 }
2196 base = buf->head->iov_len; /* align to start of pages */
2197 }
2198
2199 /* process pages array */
2200 base -= buf->head->iov_len;
2201 if (todo && base < buf->page_len) {
2202 unsigned int avail_page;
2203
2204 avail_here = min(todo, buf->page_len - base);
2205 todo -= avail_here;
2206
2207 base += buf->page_base;
2208 ppages = buf->pages + (base >> PAGE_SHIFT);
2209 base &= ~PAGE_MASK;
2210 avail_page = min_t(unsigned int, PAGE_SIZE - base,
2211 avail_here);
2212 c = kmap(*ppages) + base;
2213
2214 while (avail_here) {
2215 avail_here -= avail_page;
2216 if (copied || avail_page < desc->elem_size) {
2217 unsigned int l = min(avail_page,
2218 desc->elem_size - copied);
2219 if (!elem) {
2220 elem = kmalloc(desc->elem_size,
2221 GFP_KERNEL);
2222 err = -ENOMEM;
2223 if (!elem)
2224 goto out;
2225 }
2226 if (encode) {
2227 if (!copied) {
2228 err = desc->xcode(desc, elem);
2229 if (err)
2230 goto out;
2231 }
2232 memcpy(c, elem + copied, l);
2233 copied += l;
2234 if (copied == desc->elem_size)
2235 copied = 0;
2236 } else {
2237 memcpy(elem + copied, c, l);
2238 copied += l;
2239 if (copied == desc->elem_size) {
2240 err = desc->xcode(desc, elem);
2241 if (err)
2242 goto out;
2243 copied = 0;
2244 }
2245 }
2246 avail_page -= l;
2247 c += l;
2248 }
2249 while (avail_page >= desc->elem_size) {
2250 err = desc->xcode(desc, c);
2251 if (err)
2252 goto out;
2253 c += desc->elem_size;
2254 avail_page -= desc->elem_size;
2255 }
2256 if (avail_page) {
2257 unsigned int l = min(avail_page,
2258 desc->elem_size - copied);
2259 if (!elem) {
2260 elem = kmalloc(desc->elem_size,
2261 GFP_KERNEL);
2262 err = -ENOMEM;
2263 if (!elem)
2264 goto out;
2265 }
2266 if (encode) {
2267 if (!copied) {
2268 err = desc->xcode(desc, elem);
2269 if (err)
2270 goto out;
2271 }
2272 memcpy(c, elem + copied, l);
2273 copied += l;
2274 if (copied == desc->elem_size)
2275 copied = 0;
2276 } else {
2277 memcpy(elem + copied, c, l);
2278 copied += l;
2279 if (copied == desc->elem_size) {
2280 err = desc->xcode(desc, elem);
2281 if (err)
2282 goto out;
2283 copied = 0;
2284 }
2285 }
2286 }
2287 if (avail_here) {
2288 kunmap(*ppages);
2289 ppages++;
2290 c = kmap(*ppages);
2291 }
2292
2293 avail_page = min(avail_here,
2294 (unsigned int) PAGE_SIZE);
2295 }
2296 base = buf->page_len; /* align to start of tail */
2297 }
2298
2299 /* process tail */
2300 base -= buf->page_len;
2301 if (todo) {
2302 c = buf->tail->iov_base + base;
2303 if (copied) {
2304 unsigned int l = desc->elem_size - copied;
2305
2306 if (encode)
2307 memcpy(c, elem + copied, l);
2308 else {
2309 memcpy(elem + copied, c, l);
2310 err = desc->xcode(desc, elem);
2311 if (err)
2312 goto out;
2313 }
2314 todo -= l;
2315 c += l;
2316 }
2317 while (todo) {
2318 err = desc->xcode(desc, c);
2319 if (err)
2320 goto out;
2321 c += desc->elem_size;
2322 todo -= desc->elem_size;
2323 }
2324 }
2325 err = 0;
2326
2327out:
2328 kfree(elem);
2329 if (ppages)
2330 kunmap(*ppages);
2331 return err;
2332}
2333
2334int xdr_decode_array2(const struct xdr_buf *buf, unsigned int base,
2335 struct xdr_array2_desc *desc)
2336{
2337 if (base >= buf->len)
2338 return -EINVAL;
2339
2340 return xdr_xcode_array2(buf, base, desc, 0);
2341}
2342EXPORT_SYMBOL_GPL(xdr_decode_array2);
2343
2344int xdr_encode_array2(const struct xdr_buf *buf, unsigned int base,
2345 struct xdr_array2_desc *desc)
2346{
2347 if ((unsigned long) base + 4 + desc->array_len * desc->elem_size >
2348 buf->head->iov_len + buf->page_len + buf->tail->iov_len)
2349 return -EINVAL;
2350
2351 return xdr_xcode_array2(buf, base, desc, 1);
2352}
2353EXPORT_SYMBOL_GPL(xdr_encode_array2);
2354
2355/**
2356 * xdr_stream_decode_string_dup - Decode and duplicate variable length string
2357 * @xdr: pointer to xdr_stream
2358 * @str: location to store pointer to string
2359 * @maxlen: maximum acceptable string length
2360 * @gfp_flags: GFP mask to use
2361 *
2362 * Return values:
2363 * On success, returns length of NUL-terminated string stored in *@ptr
2364 * %-EBADMSG on XDR buffer overflow
2365 * %-EMSGSIZE if the size of the string would exceed @maxlen
2366 * %-ENOMEM on memory allocation failure
2367 */
2368ssize_t xdr_stream_decode_string_dup(struct xdr_stream *xdr, char **str,
2369 size_t maxlen, gfp_t gfp_flags)
2370{
2371 void *p;
2372 ssize_t ret;
2373
2374 ret = xdr_stream_decode_opaque_inline(xdr, &p, maxlen);
2375 if (ret > 0) {
2376 char *s = kmemdup_nul(p, ret, gfp_flags);
2377 if (s != NULL) {
2378 *str = s;
2379 return strlen(s);
2380 }
2381 ret = -ENOMEM;
2382 }
2383 *str = NULL;
2384 return ret;
2385}
2386EXPORT_SYMBOL_GPL(xdr_stream_decode_string_dup);
2387
2388/**
2389 * xdr_stream_decode_opaque_auth - Decode struct opaque_auth (RFC5531 S8.2)
2390 * @xdr: pointer to xdr_stream
2391 * @flavor: location to store decoded flavor
2392 * @body: location to store decode body
2393 * @body_len: location to store length of decoded body
2394 *
2395 * Return values:
2396 * On success, returns the number of buffer bytes consumed
2397 * %-EBADMSG on XDR buffer overflow
2398 * %-EMSGSIZE if the decoded size of the body field exceeds 400 octets
2399 */
2400ssize_t xdr_stream_decode_opaque_auth(struct xdr_stream *xdr, u32 *flavor,
2401 void **body, unsigned int *body_len)
2402{
2403 ssize_t ret, len;
2404
2405 len = xdr_stream_decode_u32(xdr, flavor);
2406 if (unlikely(len < 0))
2407 return len;
2408 ret = xdr_stream_decode_opaque_inline(xdr, body, RPC_MAX_AUTH_SIZE);
2409 if (unlikely(ret < 0))
2410 return ret;
2411 *body_len = ret;
2412 return len + ret;
2413}
2414EXPORT_SYMBOL_GPL(xdr_stream_decode_opaque_auth);
2415
2416/**
2417 * xdr_stream_encode_opaque_auth - Encode struct opaque_auth (RFC5531 S8.2)
2418 * @xdr: pointer to xdr_stream
2419 * @flavor: verifier flavor to encode
2420 * @body: content of body to encode
2421 * @body_len: length of body to encode
2422 *
2423 * Return values:
2424 * On success, returns length in bytes of XDR buffer consumed
2425 * %-EBADMSG on XDR buffer overflow
2426 * %-EMSGSIZE if the size of @body exceeds 400 octets
2427 */
2428ssize_t xdr_stream_encode_opaque_auth(struct xdr_stream *xdr, u32 flavor,
2429 void *body, unsigned int body_len)
2430{
2431 ssize_t ret, len;
2432
2433 if (unlikely(body_len > RPC_MAX_AUTH_SIZE))
2434 return -EMSGSIZE;
2435 len = xdr_stream_encode_u32(xdr, flavor);
2436 if (unlikely(len < 0))
2437 return len;
2438 ret = xdr_stream_encode_opaque(xdr, body, body_len);
2439 if (unlikely(ret < 0))
2440 return ret;
2441 return len + ret;
2442}
2443EXPORT_SYMBOL_GPL(xdr_stream_encode_opaque_auth);