| 1 |
|
| 2 | #ifndef __LINUX_UACCESS_H__
|
| 3 | #define __LINUX_UACCESS_H__
|
| 4 |
|
| 5 | #include <linux/cleanup.h>
|
| 6 | #include <linux/fault-inject-usercopy.h>
|
| 7 | #include <linux/instrumented.h>
|
| 8 | #include <linux/minmax.h>
|
| 9 | #include <linux/nospec.h>
|
| 10 | #include <linux/sched.h>
|
| 11 | #include <linux/ucopysize.h>
|
| 12 |
|
| 13 | #include <asm/uaccess.h>
|
| 14 |
|
| 15 | |
| 16 | * Architectures that support memory tagging (assigning tags to memory regions,
|
| 17 | * embedding these tags into addresses that point to these memory regions, and
|
| 18 | * checking that the memory and the pointer tags match on memory accesses)
|
| 19 | * redefine this macro to strip tags from pointers.
|
| 20 | *
|
| 21 | * Passing down mm_struct allows to define untagging rules on per-process
|
| 22 | * basis.
|
| 23 | *
|
| 24 | * It's defined as noop for architectures that don't support memory tagging.
|
| 25 |
|
| 26 | #ifndef untagged_addr
|
| 27 | #define untagged_addr(addr) (addr)
|
| 28 | #endif
|
| 29 |
|
| 30 | #ifndef untagged_addr_remote
|
| 31 | #define untagged_addr_remote(mm, addr) ({ \
|
| 32 | mmap_assert_locked(mm); \
|
| 33 | untagged_addr(addr); \
|
| 34 | })
|
| 35 | #endif
|
| 36 |
|
| 37 | #ifdef masked_user_access_begin
|
| 38 | #define can_do_masked_user_access() 1
|
| 39 | # ifndef masked_user_write_access_begin
|
| 40 | # define masked_user_write_access_begin masked_user_access_begin
|
| 41 | # endif
|
| 42 | # ifndef masked_user_read_access_begin
|
| 43 | # define masked_user_read_access_begin masked_user_access_begin
|
| 44 | #endif
|
| 45 | #else
|
| 46 | #define can_do_masked_user_access() 0
|
| 47 | #define masked_user_access_begin(src) NULL
|
| 48 | #define masked_user_read_access_begin(src) NULL
|
| 49 | #define masked_user_write_access_begin(src) NULL
|
| 50 | #define mask_user_address(src) (src)
|
| 51 | #endif
|
| 52 |
|
| 53 | |
| 54 | * Architectures should provide two primitives (raw_copy_{to,from}_user())
|
| 55 | * and get rid of their private instances of copy_{to,from}_user() and
|
| 56 | * __copy_{to,from}_user{,_inatomic}().
|
| 57 | *
|
| 58 | * raw_copy_{to,from}_user(to, from, size) should copy up to size bytes and
|
| 59 | * return the amount left to copy. They should assume that access_ok() has
|
| 60 | * already been checked (and succeeded); they should *not* zero-pad anything.
|
| 61 | * No KASAN or object size checks either - those belong here.
|
| 62 | *
|
| 63 | * Both of these functions should attempt to copy size bytes starting at from
|
| 64 | * into the area starting at to. They must not fetch or store anything
|
| 65 | * outside of those areas. Return value must be between 0 (everything
|
| 66 | * copied successfully) and size (nothing copied).
|
| 67 | *
|
| 68 | * If raw_copy_{to,from}_user(to, from, size) returns N, size - N bytes starting
|
| 69 | * at to must become equal to the bytes fetched from the corresponding area
|
| 70 | * starting at from. All data past to + size - N must be left unmodified.
|
| 71 | *
|
| 72 | * If copying succeeds, the return value must be 0. If some data cannot be
|
| 73 | * fetched, it is permitted to copy less than had been fetched; the only
|
| 74 | * hard requirement is that not storing anything at all (i.e. returning size)
|
| 75 | * should happen only when nothing could be copied. In other words, you don't
|
| 76 | * have to squeeze as much as possible - it is allowed, but not necessary.
|
| 77 | *
|
| 78 | * For raw_copy_from_user() to always points to kernel memory and no faults
|
| 79 | * on store should happen. Interpretation of from is affected by set_fs().
|
| 80 | * For raw_copy_to_user() it's the other way round.
|
| 81 | *
|
| 82 | * Both can be inlined - it's up to architectures whether it wants to bother
|
| 83 | * with that. They should not be used directly; they are used to implement
|
| 84 | * the 6 functions (copy_{to,from}_user(), __copy_{to,from}_user_inatomic())
|
| 85 | * that are used instead. Out of those, __... ones are inlined. Plain
|
| 86 | * copy_{to,from}_user() might or might not be inlined. If you want them
|
| 87 | * inlined, have asm/uaccess.h define INLINE_COPY_USER.
|
| 88 | *
|
| 89 | * NOTE: only copy_from_user() zero-pads the destination in case of short copy.
|
| 90 | * Neither __copy_from_user() nor __copy_from_user_inatomic() zero anything
|
| 91 | * at all; their callers absolutely must check the return value.
|
| 92 | *
|
| 93 | * Biarch ones should also provide raw_copy_in_user() - similar to the above,
|
| 94 | * but both source and destination are __user pointers (affected by set_fs()
|
| 95 | * as usual) and both source and destination can trigger faults.
|
| 96 |
|
| 97 |
|
| 98 | static __always_inline __must_check unsigned long
|
| 99 | __copy_from_user_inatomic(void *to, const void __user *from, unsigned long n)
|
| 100 | {
|
| 101 | unsigned long res;
|
| 102 |
|
| 103 | instrument_copy_from_user_before(to, from, n);
|
| 104 | check_object_size(to, n, false);
|
| 105 | res = raw_copy_from_user(to, from, n);
|
| 106 | instrument_copy_from_user_after(to, from, n, res);
|
| 107 | return res;
|
| 108 | }
|
| 109 |
|
| 110 | static __always_inline __must_check unsigned long
|
| 111 | __copy_from_user(void *to, const void __user *from, unsigned long n)
|
| 112 | {
|
| 113 | unsigned long res;
|
| 114 |
|
| 115 | might_fault();
|
| 116 | instrument_copy_from_user_before(to, from, n);
|
| 117 | if (should_fail_usercopy())
|
| 118 | return n;
|
| 119 | check_object_size(to, n, false);
|
| 120 | res = raw_copy_from_user(to, from, n);
|
| 121 | instrument_copy_from_user_after(to, from, n, res);
|
| 122 | return res;
|
| 123 | }
|
| 124 |
|
| 125 | |
| 126 | * __copy_to_user_inatomic: - Copy a block of data into user space, with less checking.
|
| 127 | * @to: Destination address, in user space.
|
| 128 | * @from: Source address, in kernel space.
|
| 129 | * @n: Number of bytes to copy.
|
| 130 | *
|
| 131 | * Context: User context only.
|
| 132 | *
|
| 133 | * Copy data from kernel space to user space. Caller must check
|
| 134 | * the specified block with access_ok() before calling this function.
|
| 135 | * The caller should also make sure he pins the user space address
|
| 136 | * so that we don't result in page fault and sleep.
|
| 137 |
|
| 138 | static __always_inline __must_check unsigned long
|
| 139 | __copy_to_user_inatomic(void __user *to, const void *from, unsigned long n)
|
| 140 | {
|
| 141 | if (should_fail_usercopy())
|
| 142 | return n;
|
| 143 | instrument_copy_to_user(to, from, n);
|
| 144 | check_object_size(from, n, true);
|
| 145 | return raw_copy_to_user(to, from, n);
|
| 146 | }
|
| 147 |
|
| 148 | static __always_inline __must_check unsigned long
|
| 149 | __copy_to_user(void __user *to, const void *from, unsigned long n)
|
| 150 | {
|
| 151 | might_fault();
|
| 152 | if (should_fail_usercopy())
|
| 153 | return n;
|
| 154 | instrument_copy_to_user(to, from, n);
|
| 155 | check_object_size(from, n, true);
|
| 156 | return raw_copy_to_user(to, from, n);
|
| 157 | }
|
| 158 |
|
| 159 | |
| 160 | * Architectures that #define INLINE_COPY_USER use this function
|
| 161 | * directly in the normal copy_to/from_user(), the other ones go
|
| 162 | * through an extern _copy_to/from_user(), which expands the same code
|
| 163 | * here.
|
| 164 |
|
| 165 | static inline __must_check unsigned long
|
| 166 | _inline_copy_from_user(void *to, const void __user *from, unsigned long n)
|
| 167 | {
|
| 168 | unsigned long res = n;
|
| 169 | might_fault();
|
| 170 | if (should_fail_usercopy())
|
| 171 | goto fail;
|
| 172 | if (can_do_masked_user_access())
|
| 173 | from = mask_user_address(from);
|
| 174 | else {
|
| 175 | if (!access_ok(from, n))
|
| 176 | goto fail;
|
| 177 | |
| 178 | * Ensure that bad access_ok() speculation will not
|
| 179 | * lead to nasty side effects *after* the copy is
|
| 180 | * finished:
|
| 181 |
|
| 182 | barrier_nospec();
|
| 183 | }
|
| 184 | instrument_copy_from_user_before(to, from, n);
|
| 185 | res = raw_copy_from_user(to, from, n);
|
| 186 | instrument_copy_from_user_after(to, from, n, res);
|
| 187 | if (likely(!res))
|
| 188 | return 0;
|
| 189 | fail:
|
| 190 | memset(to + (n - res), 0, res);
|
| 191 | return res;
|
| 192 | }
|
| 193 |
|
| 194 | static inline __must_check unsigned long
|
| 195 | _inline_copy_to_user(void __user *to, const void *from, unsigned long n)
|
| 196 | {
|
| 197 | might_fault();
|
| 198 | if (should_fail_usercopy())
|
| 199 | return n;
|
| 200 | if (access_ok(to, n)) {
|
| 201 | instrument_copy_to_user(to, from, n);
|
| 202 | n = raw_copy_to_user(to, from, n);
|
| 203 | }
|
| 204 | return n;
|
| 205 | }
|
| 206 | #ifdef INLINE_COPY_USER
|
| 207 | # define _copy_to_user _inline_copy_to_user
|
| 208 | # define _copy_from_user _inline_copy_from_user
|
| 209 | #else
|
| 210 | extern __must_check unsigned long
|
| 211 | _copy_from_user(void *, const void __user *, unsigned long);
|
| 212 |
|
| 213 | extern __must_check unsigned long
|
| 214 | _copy_to_user(void __user *, const void *, unsigned long);
|
| 215 | #endif
|
| 216 |
|
| 217 | static __always_inline unsigned long __must_check
|
| 218 | copy_from_user(void *to, const void __user *from, unsigned long n)
|
| 219 | {
|
| 220 | if (!check_copy_size(to, n, false))
|
| 221 | return n;
|
| 222 | return _copy_from_user(to, from, n);
|
| 223 | }
|
| 224 |
|
| 225 | static __always_inline unsigned long __must_check
|
| 226 | copy_to_user(void __user *to, const void *from, unsigned long n)
|
| 227 | {
|
| 228 | if (!check_copy_size(from, n, true))
|
| 229 | return n;
|
| 230 | return _copy_to_user(to, from, n);
|
| 231 | }
|
| 232 |
|
| 233 | #ifndef copy_mc_to_kernel
|
| 234 | |
| 235 | * Without arch opt-in this generic copy_mc_to_kernel() will not handle
|
| 236 | * #MC (or arch equivalent) during source read.
|
| 237 |
|
| 238 | static inline unsigned long __must_check
|
| 239 | copy_mc_to_kernel(void *dst, const void *src, size_t cnt)
|
| 240 | {
|
| 241 | memcpy(dst, src, cnt);
|
| 242 | return 0;
|
| 243 | }
|
| 244 | #endif
|
| 245 |
|
| 246 | static __always_inline void pagefault_disabled_inc(void)
|
| 247 | {
|
| 248 | current->pagefault_disabled++;
|
| 249 | }
|
| 250 |
|
| 251 | static __always_inline void pagefault_disabled_dec(void)
|
| 252 | {
|
| 253 | current->pagefault_disabled--;
|
| 254 | }
|
| 255 |
|
| 256 | |
| 257 | * These routines enable/disable the pagefault handler. If disabled, it will
|
| 258 | * not take any locks and go straight to the fixup table.
|
| 259 | *
|
| 260 | * User access methods will not sleep when called from a pagefault_disabled()
|
| 261 | * environment.
|
| 262 |
|
| 263 | static inline void pagefault_disable(void)
|
| 264 | {
|
| 265 | pagefault_disabled_inc();
|
| 266 | |
| 267 | * make sure to have issued the store before a pagefault
|
| 268 | * can hit.
|
| 269 |
|
| 270 | barrier();
|
| 271 | }
|
| 272 |
|
| 273 | static inline void pagefault_enable(void)
|
| 274 | {
|
| 275 | |
| 276 | * make sure to issue those last loads/stores before enabling
|
| 277 | * the pagefault handler again.
|
| 278 |
|
| 279 | barrier();
|
| 280 | pagefault_disabled_dec();
|
| 281 | }
|
| 282 |
|
| 283 | |
| 284 | * Is the pagefault handler disabled? If so, user access methods will not sleep.
|
| 285 |
|
| 286 | static inline bool pagefault_disabled(void)
|
| 287 | {
|
| 288 | return current->pagefault_disabled != 0;
|
| 289 | }
|
| 290 |
|
| 291 | |
| 292 | * The pagefault handler is in general disabled by pagefault_disable() or
|
| 293 | * when in irq context (via in_atomic()).
|
| 294 | *
|
| 295 | * This function should only be used by the fault handlers. Other users should
|
| 296 | * stick to pagefault_disabled().
|
| 297 | * Please NEVER use preempt_disable() to disable the fault handler. With
|
| 298 | * !CONFIG_PREEMPT_COUNT, this is like a NOP. So the handler won't be disabled.
|
| 299 | * in_atomic() will report different values based on !CONFIG_PREEMPT_COUNT.
|
| 300 |
|
| 301 | #define faulthandler_disabled() (pagefault_disabled() || in_atomic())
|
| 302 |
|
| 303 | DEFINE_LOCK_GUARD_0(pagefault, pagefault_disable(), pagefault_enable())
|
| 304 |
|
| 305 | #ifndef CONFIG_ARCH_HAS_SUBPAGE_FAULTS
|
| 306 |
|
| 307 | |
| 308 | * probe_subpage_writeable: probe the user range for write faults at sub-page
|
| 309 | * granularity (e.g. arm64 MTE)
|
| 310 | * @uaddr: start of address range
|
| 311 | * @size: size of address range
|
| 312 | *
|
| 313 | * Returns 0 on success, the number of bytes not probed on fault.
|
| 314 | *
|
| 315 | * It is expected that the caller checked for the write permission of each
|
| 316 | * page in the range either by put_user() or GUP. The architecture port can
|
| 317 | * implement a more efficient get_user() probing if the same sub-page faults
|
| 318 | * are triggered by either a read or a write.
|
| 319 |
|
| 320 | static inline size_t probe_subpage_writeable(char __user *uaddr, size_t size)
|
| 321 | {
|
| 322 | return 0;
|
| 323 | }
|
| 324 |
|
| 325 | #endif
|
| 326 |
|
| 327 | #ifndef ARCH_HAS_NONTEMPORAL_UACCESS
|
| 328 |
|
| 329 | static inline __must_check unsigned long
|
| 330 | copy_from_user_inatomic_nontemporal(void *to, const void __user *from,
|
| 331 | unsigned long n)
|
| 332 | {
|
| 333 | if (can_do_masked_user_access())
|
| 334 | from = mask_user_address(from);
|
| 335 | else
|
| 336 | if (!access_ok(from, n))
|
| 337 | return n;
|
| 338 | return __copy_from_user_inatomic(to, from, n);
|
| 339 | }
|
| 340 |
|
| 341 | #endif
|
| 342 |
|
| 343 | extern __must_check int check_zeroed_user(const void __user *from, size_t size);
|
| 344 |
|
| 345 | |
| 346 | * copy_struct_from_user: copy a struct from userspace
|
| 347 | * @dst: Destination address, in kernel space. This buffer must be @ksize
|
| 348 | * bytes long.
|
| 349 | * @ksize: Size of @dst struct.
|
| 350 | * @src: Source address, in userspace.
|
| 351 | * @usize: (Alleged) size of @src struct.
|
| 352 | *
|
| 353 | * Copies a struct from userspace to kernel space, in a way that guarantees
|
| 354 | * backwards-compatibility for struct syscall arguments (as long as future
|
| 355 | * struct extensions are made such that all new fields are *appended* to the
|
| 356 | * old struct, and zeroed-out new fields have the same meaning as the old
|
| 357 | * struct).
|
| 358 | *
|
| 359 | * @ksize is just sizeof(*dst), and @usize should've been passed by userspace.
|
| 360 | * The recommended usage is something like the following:
|
| 361 | *
|
| 362 | * SYSCALL_DEFINE2(foobar, const struct foo __user *, uarg, size_t, usize)
|
| 363 | * {
|
| 364 | * int err;
|
| 365 | * struct foo karg = {};
|
| 366 | *
|
| 367 | * if (usize > PAGE_SIZE)
|
| 368 | * return -E2BIG;
|
| 369 | * if (usize < FOO_SIZE_VER0)
|
| 370 | * return -EINVAL;
|
| 371 | *
|
| 372 | * err = copy_struct_from_user(&karg, sizeof(karg), uarg, usize);
|
| 373 | * if (err)
|
| 374 | * return err;
|
| 375 | *
|
| 376 | * // ...
|
| 377 | * }
|
| 378 | *
|
| 379 | * There are three cases to consider:
|
| 380 | * * If @usize == @ksize, then it's copied verbatim.
|
| 381 | * * If @usize < @ksize, then the userspace has passed an old struct to a
|
| 382 | * newer kernel. The rest of the trailing bytes in @dst (@ksize - @usize)
|
| 383 | * are to be zero-filled.
|
| 384 | * * If @usize > @ksize, then the userspace has passed a new struct to an
|
| 385 | * older kernel. The trailing bytes unknown to the kernel (@usize - @ksize)
|
| 386 | * are checked to ensure they are zeroed, otherwise -E2BIG is returned.
|
| 387 | *
|
| 388 | * Returns (in all cases, some data may have been copied):
|
| 389 | * * -E2BIG: (@usize > @ksize) and there are non-zero trailing bytes in @src.
|
| 390 | * * -EFAULT: access to userspace failed.
|
| 391 |
|
| 392 | static __always_inline __must_check int
|
| 393 | copy_struct_from_user(void *dst, size_t ksize, const void __user *src,
|
| 394 | size_t usize)
|
| 395 | {
|
| 396 | size_t size = min(ksize, usize);
|
| 397 | size_t rest = max(ksize, usize) - size;
|
| 398 |
|
| 399 |
|
| 400 | if (WARN_ON_ONCE(ksize > __builtin_object_size(dst, 1)))
|
| 401 | return -E2BIG;
|
| 402 |
|
| 403 |
|
| 404 | if (usize < ksize) {
|
| 405 | memset(dst + size, 0, rest);
|
| 406 | } else if (usize > ksize) {
|
| 407 | int ret = check_zeroed_user(src + size, rest);
|
| 408 | if (ret <= 0)
|
| 409 | return ret ?: -E2BIG;
|
| 410 | }
|
| 411 |
|
| 412 | if (copy_from_user(dst, src, size))
|
| 413 | return -EFAULT;
|
| 414 | return 0;
|
| 415 | }
|
| 416 |
|
| 417 | |
| 418 | * copy_struct_to_user: copy a struct to userspace
|
| 419 | * @dst: Destination address, in userspace. This buffer must be @ksize
|
| 420 | * bytes long.
|
| 421 | * @usize: (Alleged) size of @dst struct.
|
| 422 | * @src: Source address, in kernel space.
|
| 423 | * @ksize: Size of @src struct.
|
| 424 | * @ignored_trailing: Set to %true if there was a non-zero byte in @src that
|
| 425 | * userspace cannot see because they are using an smaller struct.
|
| 426 | *
|
| 427 | * Copies a struct from kernel space to userspace, in a way that guarantees
|
| 428 | * backwards-compatibility for struct syscall arguments (as long as future
|
| 429 | * struct extensions are made such that all new fields are *appended* to the
|
| 430 | * old struct, and zeroed-out new fields have the same meaning as the old
|
| 431 | * struct).
|
| 432 | *
|
| 433 | * Some syscalls may wish to make sure that userspace knows about everything in
|
| 434 | * the struct, and if there is a non-zero value that userspce doesn't know
|
| 435 | * about, they want to return an error (such as -EMSGSIZE) or have some other
|
| 436 | * fallback (such as adding a "you're missing some information" flag). If
|
| 437 | * @ignored_trailing is non-%NULL, it will be set to %true if there was a
|
| 438 | * non-zero byte that could not be copied to userspace (ie. was past @usize).
|
| 439 | *
|
| 440 | * While unconditionally returning an error in this case is the simplest
|
| 441 | * solution, for maximum backward compatibility you should try to only return
|
| 442 | * -EMSGSIZE if the user explicitly requested the data that couldn't be copied.
|
| 443 | * Note that structure sizes can change due to header changes and simple
|
| 444 | * recompilations without code changes(!), so if you care about
|
| 445 | * @ignored_trailing you probably want to make sure that any new field data is
|
| 446 | * associated with a flag. Otherwise you might assume that a program knows
|
| 447 | * about data it does not.
|
| 448 | *
|
| 449 | * @ksize is just sizeof(*src), and @usize should've been passed by userspace.
|
| 450 | * The recommended usage is something like the following:
|
| 451 | *
|
| 452 | * SYSCALL_DEFINE2(foobar, struct foo __user *, uarg, size_t, usize)
|
| 453 | * {
|
| 454 | * int err;
|
| 455 | * bool ignored_trailing;
|
| 456 | * struct foo karg = {};
|
| 457 | *
|
| 458 | * if (usize > PAGE_SIZE)
|
| 459 | * return -E2BIG;
|
| 460 | * if (usize < FOO_SIZE_VER0)
|
| 461 | * return -EINVAL;
|
| 462 | *
|
| 463 | * // ... modify karg somehow ...
|
| 464 | *
|
| 465 | * err = copy_struct_to_user(uarg, usize, &karg, sizeof(karg),
|
| 466 | * &ignored_trailing);
|
| 467 | * if (err)
|
| 468 | * return err;
|
| 469 | * if (ignored_trailing)
|
| 470 | * return -EMSGSIZE:
|
| 471 | *
|
| 472 | * // ...
|
| 473 | * }
|
| 474 | *
|
| 475 | * There are three cases to consider:
|
| 476 | * * If @usize == @ksize, then it's copied verbatim.
|
| 477 | * * If @usize < @ksize, then the kernel is trying to pass userspace a newer
|
| 478 | * struct than it supports. Thus we only copy the interoperable portions
|
| 479 | * (@usize) and ignore the rest (but @ignored_trailing is set to %true if
|
| 480 | * any of the trailing (@ksize - @usize) bytes are non-zero).
|
| 481 | * * If @usize > @ksize, then the kernel is trying to pass userspace an older
|
| 482 | * struct than userspace supports. In order to make sure the
|
| 483 | * unknown-to-the-kernel fields don't contain garbage values, we zero the
|
| 484 | * trailing (@usize - @ksize) bytes.
|
| 485 | *
|
| 486 | * Returns (in all cases, some data may have been copied):
|
| 487 | * * -EFAULT: access to userspace failed.
|
| 488 |
|
| 489 | static __always_inline __must_check int
|
| 490 | copy_struct_to_user(void __user *dst, size_t usize, const void *src,
|
| 491 | size_t ksize, bool *ignored_trailing)
|
| 492 | {
|
| 493 | size_t size = min(ksize, usize);
|
| 494 | size_t rest = max(ksize, usize) - size;
|
| 495 |
|
| 496 |
|
| 497 | if (WARN_ON_ONCE(ksize > __builtin_object_size(src, 1)))
|
| 498 | return -E2BIG;
|
| 499 |
|
| 500 |
|
| 501 | if (usize > ksize) {
|
| 502 | if (clear_user(dst + size, rest))
|
| 503 | return -EFAULT;
|
| 504 | }
|
| 505 | if (ignored_trailing)
|
| 506 | *ignored_trailing = usize < ksize &&
|
| 507 | memchr_inv(src + size, 0, rest) != NULL;
|
| 508 |
|
| 509 | if (copy_to_user(dst, src, size))
|
| 510 | return -EFAULT;
|
| 511 | return 0;
|
| 512 | }
|
| 513 |
|
| 514 | static __always_inline void
|
| 515 | __copy_struct_generic_bounce_buffer(void *dst, size_t dstsize,
|
| 516 | const void *src, size_t srcsize,
|
| 517 | bool *ignored_trailing)
|
| 518 | {
|
| 519 | size_t size = min(dstsize, srcsize);
|
| 520 | size_t rest = max(dstsize, srcsize) - size;
|
| 521 |
|
| 522 |
|
| 523 | if (dstsize > srcsize)
|
| 524 | memset(dst + size, 0, rest);
|
| 525 | if (ignored_trailing)
|
| 526 | *ignored_trailing = dstsize < srcsize &&
|
| 527 | memchr_inv(src + size, 0, rest) != NULL;
|
| 528 |
|
| 529 | memcpy(dst, src, size);
|
| 530 | }
|
| 531 |
|
| 532 | |
| 533 | * This is like copy_struct_from_user(), but the
|
| 534 | * src buffer was already copied into a kernel
|
| 535 | * bounce buffer, so it will never return -EFAULT.
|
| 536 |
|
| 537 | static __always_inline __must_check int
|
| 538 | copy_struct_from_bounce_buffer(void *dst, size_t dstsize,
|
| 539 | const void *src, size_t srcsize)
|
| 540 | {
|
| 541 | bool ignored_trailing;
|
| 542 |
|
| 543 |
|
| 544 | if (WARN_ON_ONCE(dstsize > __builtin_object_size(dst, 1)))
|
| 545 | return -E2BIG;
|
| 546 |
|
| 547 | __copy_struct_generic_bounce_buffer(dst, dstsize,
|
| 548 | src, srcsize,
|
| 549 | &ignored_trailing);
|
| 550 | if (unlikely(ignored_trailing))
|
| 551 | return -E2BIG;
|
| 552 |
|
| 553 | return 0;
|
| 554 | }
|
| 555 |
|
| 556 | |
| 557 | * This is like copy_struct_to_user(), but the
|
| 558 | * dst buffer is a kernel bounce buffer instead
|
| 559 | * of a direct userspace buffer, so it will never return -EFAULT.
|
| 560 |
|
| 561 | static __always_inline __must_check int
|
| 562 | copy_struct_to_bounce_buffer(void *dst, size_t dstsize,
|
| 563 | const void *src,
|
| 564 | size_t srcsize,
|
| 565 | bool *ignored_trailing)
|
| 566 | {
|
| 567 |
|
| 568 | if (WARN_ON_ONCE(srcsize > __builtin_object_size(src, 1)))
|
| 569 | return -E2BIG;
|
| 570 |
|
| 571 | __copy_struct_generic_bounce_buffer(dst, dstsize,
|
| 572 | src, srcsize,
|
| 573 | ignored_trailing);
|
| 574 | return 0;
|
| 575 | }
|
| 576 |
|
| 577 | bool copy_from_kernel_nofault_allowed(const void *unsafe_src, size_t size);
|
| 578 |
|
| 579 | long copy_from_kernel_nofault(void *dst, const void *src, size_t size);
|
| 580 | long notrace copy_to_kernel_nofault(void *dst, const void *src, size_t size);
|
| 581 |
|
| 582 | long copy_from_user_nofault(void *dst, const void __user *src, size_t size);
|
| 583 | long notrace copy_to_user_nofault(void __user *dst, const void *src,
|
| 584 | size_t size);
|
| 585 |
|
| 586 | long strncpy_from_kernel_nofault(char *dst, const void *unsafe_addr,
|
| 587 | long count);
|
| 588 |
|
| 589 | long strncpy_from_user_nofault(char *dst, const void __user *unsafe_addr,
|
| 590 | long count);
|
| 591 | long strnlen_user_nofault(const void __user *unsafe_addr, long count);
|
| 592 |
|
| 593 | #ifdef arch_get_kernel_nofault
|
| 594 | |
| 595 | * Wrap the architecture implementation so that @label can be outside of a
|
| 596 | * cleanup() scope. A regular C goto works correctly, but ASM goto does
|
| 597 | * not. Clang rejects such an attempt, but GCC silently emits buggy code.
|
| 598 |
|
| 599 | #define __get_kernel_nofault(dst, src, type, label) \
|
| 600 | do { \
|
| 601 | __label__ local_label; \
|
| 602 | arch_get_kernel_nofault(dst, src, type, local_label); \
|
| 603 | if (0) { \
|
| 604 | local_label: \
|
| 605 | goto label; \
|
| 606 | } \
|
| 607 | } while (0)
|
| 608 |
|
| 609 | #define __put_kernel_nofault(dst, src, type, label) \
|
| 610 | do { \
|
| 611 | __label__ local_label; \
|
| 612 | arch_put_kernel_nofault(dst, src, type, local_label); \
|
| 613 | if (0) { \
|
| 614 | local_label: \
|
| 615 | goto label; \
|
| 616 | } \
|
| 617 | } while (0)
|
| 618 |
|
| 619 | #elif !defined(__get_kernel_nofault)
|
| 620 |
|
| 621 | #define __get_kernel_nofault(dst, src, type, label) \
|
| 622 | do { \
|
| 623 | type __user *p = (type __force __user *)(src); \
|
| 624 | type data; \
|
| 625 | if (__get_user(data, p)) \
|
| 626 | goto label; \
|
| 627 | *(type *)dst = data; \
|
| 628 | } while (0)
|
| 629 |
|
| 630 | #define __put_kernel_nofault(dst, src, type, label) \
|
| 631 | do { \
|
| 632 | type __user *p = (type __force __user *)(dst); \
|
| 633 | type data = *(type *)src; \
|
| 634 | if (__put_user(data, p)) \
|
| 635 | goto label; \
|
| 636 | } while (0)
|
| 637 |
|
| 638 | #endif
|
| 639 |
|
| 640 | |
| 641 | * get_kernel_nofault(): safely attempt to read from a location
|
| 642 | * @val: read into this variable
|
| 643 | * @ptr: address to read from
|
| 644 | *
|
| 645 | * Returns 0 on success, or -EFAULT.
|
| 646 |
|
| 647 | #define get_kernel_nofault(val, ptr) ({ \
|
| 648 | const typeof(val) *__gk_ptr = (ptr); \
|
| 649 | copy_from_kernel_nofault(&(val), __gk_ptr, sizeof(val));\
|
| 650 | })
|
| 651 |
|
| 652 | #ifdef user_access_begin
|
| 653 |
|
| 654 | #ifdef arch_unsafe_get_user
|
| 655 | |
| 656 | * Wrap the architecture implementation so that @label can be outside of a
|
| 657 | * cleanup() scope. A regular C goto works correctly, but ASM goto does
|
| 658 | * not. Clang rejects such an attempt, but GCC silently emits buggy code.
|
| 659 | *
|
| 660 | * Some architectures use internal local labels already, but this extra
|
| 661 | * indirection here is harmless because the compiler optimizes it out
|
| 662 | * completely in any case. This construct just ensures that the ASM GOTO
|
| 663 | * target is always in the local scope. The C goto 'label' works correctly
|
| 664 | * when leaving a cleanup() scope.
|
| 665 |
|
| 666 | #define unsafe_get_user(x, ptr, label) \
|
| 667 | do { \
|
| 668 | __label__ local_label; \
|
| 669 | arch_unsafe_get_user(x, ptr, local_label); \
|
| 670 | if (0) { \
|
| 671 | local_label: \
|
| 672 | goto label; \
|
| 673 | } \
|
| 674 | } while (0)
|
| 675 |
|
| 676 | #define unsafe_put_user(x, ptr, label) \
|
| 677 | do { \
|
| 678 | __label__ local_label; \
|
| 679 | arch_unsafe_put_user(x, ptr, local_label); \
|
| 680 | if (0) { \
|
| 681 | local_label: \
|
| 682 | goto label; \
|
| 683 | } \
|
| 684 | } while (0)
|
| 685 | #endif
|
| 686 |
|
| 687 | #else
|
| 688 | #define user_access_begin(ptr,len) access_ok(ptr, len)
|
| 689 | #define user_access_end() do { } while (0)
|
| 690 | #define unsafe_op_wrap(op, err) do { if (unlikely(op)) goto err; } while (0)
|
| 691 | #define unsafe_get_user(x,p,e) unsafe_op_wrap(__get_user(x,p),e)
|
| 692 | #define unsafe_put_user(x,p,e) unsafe_op_wrap(__put_user(x,p),e)
|
| 693 | #define unsafe_copy_to_user(d,s,l,e) unsafe_op_wrap(__copy_to_user(d,s,l),e)
|
| 694 | #define unsafe_copy_from_user(d,s,l,e) unsafe_op_wrap(__copy_from_user(d,s,l),e)
|
| 695 | static inline unsigned long user_access_save(void) { return 0UL; }
|
| 696 | static inline void user_access_restore(unsigned long flags) { }
|
| 697 | #endif
|
| 698 |
|
| 699 | #ifndef user_write_access_begin
|
| 700 | #define user_write_access_begin user_access_begin
|
| 701 | #define user_write_access_end user_access_end
|
| 702 | #endif
|
| 703 | #ifndef user_read_access_begin
|
| 704 | #define user_read_access_begin user_access_begin
|
| 705 | #define user_read_access_end user_access_end
|
| 706 | #endif
|
| 707 |
|
| 708 | #ifndef unsafe_atomic_store_release_user
|
| 709 | # define unsafe_atomic_store_release_user(val, uptr, elbl) \
|
| 710 | do { \
|
| 711 | if (!IS_ENABLED(CONFIG_ARCH_MEMORY_ORDER_TSO)) \
|
| 712 | smp_mb(); \
|
| 713 | else \
|
| 714 | barrier(); \
|
| 715 | unsafe_put_user(val, uptr, elbl); \
|
| 716 | } while (0)
|
| 717 | #endif
|
| 718 |
|
| 719 |
|
| 720 | #define masked_user_rw_access_begin(u) masked_user_access_begin(u)
|
| 721 | #define user_rw_access_begin(u, s) user_access_begin(u, s)
|
| 722 |
|
| 723 |
|
| 724 |
|
| 725 |
|
| 726 | static __always_inline void __scoped_user_read_access_end(const void *p)
|
| 727 | {
|
| 728 | user_read_access_end();
|
| 729 | };
|
| 730 | static __always_inline void __scoped_user_write_access_end(const void *p)
|
| 731 | {
|
| 732 | user_write_access_end();
|
| 733 | };
|
| 734 | static __always_inline void __scoped_user_rw_access_end(const void *p)
|
| 735 | {
|
| 736 | user_access_end();
|
| 737 | };
|
| 738 |
|
| 739 | |
| 740 | * __scoped_user_access_begin - Start a scoped user access
|
| 741 | * @mode: The mode of the access class (read, write, rw)
|
| 742 | * @uptr: The pointer to access user space memory
|
| 743 | * @size: Size of the access
|
| 744 | * @elbl: Error label to goto when the access region is rejected
|
| 745 | *
|
| 746 | * Internal helper for __scoped_user_access(). Don't use directly.
|
| 747 |
|
| 748 | #define __scoped_user_access_begin(mode, uptr, size, elbl) \
|
| 749 | ({ \
|
| 750 | typeof(uptr) __retptr; \
|
| 751 | \
|
| 752 | if (can_do_masked_user_access()) { \
|
| 753 | __retptr = masked_user_##mode##_access_begin(uptr); \
|
| 754 | } else { \
|
| 755 | __retptr = uptr; \
|
| 756 | if (!user_##mode##_access_begin(uptr, size)) \
|
| 757 | goto elbl; \
|
| 758 | } \
|
| 759 | __retptr; \
|
| 760 | })
|
| 761 |
|
| 762 | |
| 763 | * __scoped_user_access - Open a scope for user access
|
| 764 | * @mode: The mode of the access class (read, write, rw)
|
| 765 | * @uptr: The pointer to access user space memory
|
| 766 | * @size: Size of the access
|
| 767 | * @elbl: Error label to goto when the access region is rejected. It
|
| 768 | * must be placed outside the scope
|
| 769 | *
|
| 770 | * If the user access function inside the scope requires a fault label, it
|
| 771 | * can use @elbl or a different label outside the scope, which requires
|
| 772 | * that user access which is implemented with ASM GOTO has been properly
|
| 773 | * wrapped. See unsafe_get_user() for reference.
|
| 774 | *
|
| 775 | * scoped_user_rw_access(ptr, efault) {
|
| 776 | * unsafe_get_user(rval, &ptr->rval, efault);
|
| 777 | * unsafe_put_user(wval, &ptr->wval, efault);
|
| 778 | * }
|
| 779 | * return 0;
|
| 780 | * efault:
|
| 781 | * return -EFAULT;
|
| 782 | *
|
| 783 | * The scope is internally implemented as a autoterminating nested for()
|
| 784 | * loop, which can be left with 'return', 'break' and 'goto' at any
|
| 785 | * point.
|
| 786 | *
|
| 787 | * When the scope is left user_##@_mode##_access_end() is automatically
|
| 788 | * invoked.
|
| 789 | *
|
| 790 | * When the architecture supports masked user access and the access region
|
| 791 | * which is determined by @uptr and @size is not a valid user space
|
| 792 | * address, i.e. < TASK_SIZE, the scope sets the pointer to a faulting user
|
| 793 | * space address and does not terminate early. This optimizes for the good
|
| 794 | * case and lets the performance uncritical bad case go through the fault.
|
| 795 | *
|
| 796 | * The eventual modification of the pointer is limited to the scope.
|
| 797 | * Outside of the scope the original pointer value is unmodified, so that
|
| 798 | * the original pointer value is available for diagnostic purposes in an
|
| 799 | * out of scope fault path.
|
| 800 | *
|
| 801 | * Nesting scoped user access into a user access scope is invalid and fails
|
| 802 | * the build. Nesting into other guards, e.g. pagefault is safe.
|
| 803 | *
|
| 804 | * The masked variant does not check the size of the access and relies on a
|
| 805 | * mapping hole (e.g. guard page) to catch an out of range pointer, the
|
| 806 | * first access to user memory inside the scope has to be within
|
| 807 | * @uptr ... @uptr + PAGE_SIZE - 1
|
| 808 | *
|
| 809 | * Don't use directly. Use scoped_masked_user_$MODE_access() instead.
|
| 810 |
|
| 811 | #define __scoped_user_access(mode, uptr, size, elbl) \
|
| 812 | for (bool done = false; !done; done = true) \
|
| 813 | for (auto _tmpptr = __scoped_user_access_begin(mode, uptr, size, elbl); \
|
| 814 | !done; done = true) \
|
| 815 | \
|
| 816 | for (const auto uptr __cleanup(__scoped_user_##mode##_access_end) = \
|
| 817 | _tmpptr; !done; done = true)
|
| 818 |
|
| 819 | |
| 820 | * scoped_user_read_access_size - Start a scoped user read access with given size
|
| 821 | * @usrc: Pointer to the user space address to read from
|
| 822 | * @size: Size of the access starting from @usrc
|
| 823 | * @elbl: Error label to goto when the access region is rejected
|
| 824 | *
|
| 825 | * For further information see __scoped_user_access() above.
|
| 826 |
|
| 827 | #define scoped_user_read_access_size(usrc, size, elbl) \
|
| 828 | __scoped_user_access(read, usrc, size, elbl)
|
| 829 |
|
| 830 | |
| 831 | * scoped_user_read_access - Start a scoped user read access
|
| 832 | * @usrc: Pointer to the user space address to read from
|
| 833 | * @elbl: Error label to goto when the access region is rejected
|
| 834 | *
|
| 835 | * The size of the access starting from @usrc is determined via sizeof(*@usrc)).
|
| 836 | *
|
| 837 | * For further information see __scoped_user_access() above.
|
| 838 |
|
| 839 | #define scoped_user_read_access(usrc, elbl) \
|
| 840 | scoped_user_read_access_size(usrc, sizeof(*(usrc)), elbl)
|
| 841 |
|
| 842 | |
| 843 | * scoped_user_write_access_size - Start a scoped user write access with given size
|
| 844 | * @udst: Pointer to the user space address to write to
|
| 845 | * @size: Size of the access starting from @udst
|
| 846 | * @elbl: Error label to goto when the access region is rejected
|
| 847 | *
|
| 848 | * For further information see __scoped_user_access() above.
|
| 849 |
|
| 850 | #define scoped_user_write_access_size(udst, size, elbl) \
|
| 851 | __scoped_user_access(write, udst, size, elbl)
|
| 852 |
|
| 853 | |
| 854 | * scoped_user_write_access - Start a scoped user write access
|
| 855 | * @udst: Pointer to the user space address to write to
|
| 856 | * @elbl: Error label to goto when the access region is rejected
|
| 857 | *
|
| 858 | * The size of the access starting from @udst is determined via sizeof(*@udst)).
|
| 859 | *
|
| 860 | * For further information see __scoped_user_access() above.
|
| 861 |
|
| 862 | #define scoped_user_write_access(udst, elbl) \
|
| 863 | scoped_user_write_access_size(udst, sizeof(*(udst)), elbl)
|
| 864 |
|
| 865 | |
| 866 | * scoped_user_rw_access_size - Start a scoped user read/write access with given size
|
| 867 | * @uptr: Pointer to the user space address to read from and write to
|
| 868 | * @size: Size of the access starting from @uptr
|
| 869 | * @elbl: Error label to goto when the access region is rejected
|
| 870 | *
|
| 871 | * For further information see __scoped_user_access() above.
|
| 872 |
|
| 873 | #define scoped_user_rw_access_size(uptr, size, elbl) \
|
| 874 | __scoped_user_access(rw, uptr, size, elbl)
|
| 875 |
|
| 876 | |
| 877 | * scoped_user_rw_access - Start a scoped user read/write access
|
| 878 | * @uptr: Pointer to the user space address to read from and write to
|
| 879 | * @elbl: Error label to goto when the access region is rejected
|
| 880 | *
|
| 881 | * The size of the access starting from @uptr is determined via sizeof(*@uptr)).
|
| 882 | *
|
| 883 | * For further information see __scoped_user_access() above.
|
| 884 |
|
| 885 | #define scoped_user_rw_access(uptr, elbl) \
|
| 886 | scoped_user_rw_access_size(uptr, sizeof(*(uptr)), elbl)
|
| 887 |
|
| 888 | |
| 889 | * get_user_inline - Read user data inlined
|
| 890 | * @val: The variable to store the value read from user memory
|
| 891 | * @usrc: Pointer to the user space memory to read from
|
| 892 | *
|
| 893 | * Return: 0 if successful, -EFAULT when faulted
|
| 894 | *
|
| 895 | * Inlined variant of get_user(). Only use when there is a demonstrable
|
| 896 | * performance reason.
|
| 897 |
|
| 898 | #define get_user_inline(val, usrc) \
|
| 899 | ({ \
|
| 900 | __label__ efault; \
|
| 901 | typeof(usrc) _tmpsrc = usrc; \
|
| 902 | int _ret = 0; \
|
| 903 | \
|
| 904 | scoped_user_read_access(_tmpsrc, efault) \
|
| 905 | unsafe_get_user(val, _tmpsrc, efault); \
|
| 906 | if (0) { \
|
| 907 | efault: \
|
| 908 | _ret = -EFAULT; \
|
| 909 | } \
|
| 910 | _ret; \
|
| 911 | })
|
| 912 |
|
| 913 | |
| 914 | * put_user_inline - Write to user memory inlined
|
| 915 | * @val: The value to write
|
| 916 | * @udst: Pointer to the user space memory to write to
|
| 917 | *
|
| 918 | * Return: 0 if successful, -EFAULT when faulted
|
| 919 | *
|
| 920 | * Inlined variant of put_user(). Only use when there is a demonstrable
|
| 921 | * performance reason.
|
| 922 |
|
| 923 | #define put_user_inline(val, udst) \
|
| 924 | ({ \
|
| 925 | __label__ efault; \
|
| 926 | typeof(udst) _tmpdst = udst; \
|
| 927 | int _ret = 0; \
|
| 928 | \
|
| 929 | scoped_user_write_access(_tmpdst, efault) \
|
| 930 | unsafe_put_user(val, _tmpdst, efault); \
|
| 931 | if (0) { \
|
| 932 | efault: \
|
| 933 | _ret = -EFAULT; \
|
| 934 | } \
|
| 935 | _ret; \
|
| 936 | })
|
| 937 |
|
| 938 | #ifdef CONFIG_HARDENED_USERCOPY
|
| 939 | void __noreturn usercopy_abort(const char *name, const char *detail,
|
| 940 | bool to_user, unsigned long offset,
|
| 941 | unsigned long len);
|
| 942 | #endif
|
| 943 |
|
| 944 | #endif
|