| 1 |
|
| 2 | #ifndef __X86_KERNEL_FPU_CONTEXT_H
|
| 3 | #define __X86_KERNEL_FPU_CONTEXT_H
|
| 4 |
|
| 5 | #include <asm/fpu/xstate.h>
|
| 6 | #include <asm/trace/fpu.h>
|
| 7 |
|
| 8 |
|
| 9 |
|
| 10 | |
| 11 | * The in-register FPU state for an FPU context on a CPU is assumed to be
|
| 12 | * valid if the fpu->last_cpu matches the CPU, and the fpu_fpregs_owner_ctx
|
| 13 | * matches the FPU.
|
| 14 | *
|
| 15 | * If the FPU register state is valid, the kernel can skip restoring the
|
| 16 | * FPU state from memory.
|
| 17 | *
|
| 18 | * Any code that clobbers the FPU registers or updates the in-memory
|
| 19 | * FPU state for a task MUST let the rest of the kernel know that the
|
| 20 | * FPU registers are no longer valid for this task.
|
| 21 | *
|
| 22 | * Invalidate a resource you control: CPU if using the CPU for something else
|
| 23 | * (with preemption disabled), FPU for the current task, or a task that
|
| 24 | * is prevented from running by the current task.
|
| 25 |
|
| 26 | static inline void __cpu_invalidate_fpregs_state(void)
|
| 27 | {
|
| 28 | __this_cpu_write(fpu_fpregs_owner_ctx, NULL);
|
| 29 | }
|
| 30 |
|
| 31 | static inline void __fpu_invalidate_fpregs_state(struct fpu *fpu)
|
| 32 | {
|
| 33 | fpu->last_cpu = -1;
|
| 34 | }
|
| 35 |
|
| 36 | static inline int fpregs_state_valid(struct fpu *fpu, unsigned int cpu)
|
| 37 | {
|
| 38 | return fpu == this_cpu_read(fpu_fpregs_owner_ctx) && cpu == fpu->last_cpu;
|
| 39 | }
|
| 40 |
|
| 41 | static inline void fpregs_deactivate(struct fpu *fpu)
|
| 42 | {
|
| 43 | __this_cpu_write(fpu_fpregs_owner_ctx, NULL);
|
| 44 | trace_x86_fpu_regs_deactivated(fpu);
|
| 45 | }
|
| 46 |
|
| 47 | static inline void fpregs_activate(struct fpu *fpu)
|
| 48 | {
|
| 49 | __this_cpu_write(fpu_fpregs_owner_ctx, fpu);
|
| 50 | trace_x86_fpu_regs_activated(fpu);
|
| 51 | }
|
| 52 |
|
| 53 |
|
| 54 | static inline void fpregs_restore_userregs(void)
|
| 55 | {
|
| 56 | struct fpu *fpu = x86_task_fpu(current);
|
| 57 | int cpu = smp_processor_id();
|
| 58 |
|
| 59 | if (WARN_ON_ONCE(current->flags & (PF_KTHREAD | PF_USER_WORKER)))
|
| 60 | return;
|
| 61 |
|
| 62 | if (!fpregs_state_valid(fpu, cpu)) {
|
| 63 | |
| 64 | * This restores _all_ xstate which has not been
|
| 65 | * established yet.
|
| 66 | *
|
| 67 | * If PKRU is enabled, then the PKRU value is already
|
| 68 | * correct because it was either set in switch_to() or in
|
| 69 | * flush_thread(). So it is excluded because it might be
|
| 70 | * not up to date in current->thread.fpu->xsave state.
|
| 71 | *
|
| 72 | * XFD state is handled in restore_fpregs_from_fpstate().
|
| 73 |
|
| 74 | restore_fpregs_from_fpstate(fpu->fpstate, XFEATURE_MASK_FPSTATE);
|
| 75 |
|
| 76 | fpregs_activate(fpu);
|
| 77 | fpu->last_cpu = cpu;
|
| 78 | }
|
| 79 | clear_thread_flag(TIF_NEED_FPU_LOAD);
|
| 80 | }
|
| 81 |
|
| 82 | #endif
|