| 1 |
|
| 2 | #ifndef _KERNEL_TIME_MIGRATION_H
|
| 3 | #define _KERNEL_TIME_MIGRATION_H
|
| 4 |
|
| 5 |
|
| 6 | #define TMIGR_CHILDREN_PER_GROUP 8
|
| 7 |
|
| 8 | |
| 9 | * struct tmigr_hierarchy - a hierarchy associated to a given CPU capacity.
|
| 10 | * Homogeneous systems have only one hierarchy.
|
| 11 | * Heterogenous have one hierarchy per CPU capacity.
|
| 12 | * @cpumask: CPUs belonging to this hierarchy
|
| 13 | * @root: The current root of the hierarchy
|
| 14 | * @capacity: CPU capacity associated to this hierarchy
|
| 15 | * @node: Node in the global hierarchy list
|
| 16 | * @level_list: Per level lists of tmigr groups
|
| 17 |
|
| 18 | struct tmigr_hierarchy {
|
| 19 | struct cpumask *cpumask;
|
| 20 | struct tmigr_group *root;
|
| 21 | unsigned long capacity;
|
| 22 | struct list_head node;
|
| 23 | struct list_head level_list[];
|
| 24 | };
|
| 25 |
|
| 26 | |
| 27 | * struct tmigr_event - a timer event associated to a CPU
|
| 28 | * @nextevt: The node to enqueue an event in the parent group queue
|
| 29 | * @cpu: The CPU to which this event belongs
|
| 30 | * @ignore: Hint whether the event could be ignored; it is set when
|
| 31 | * CPU or group is active;
|
| 32 |
|
| 33 | struct tmigr_event {
|
| 34 | struct timerqueue_node nextevt;
|
| 35 | unsigned int cpu;
|
| 36 | bool ignore;
|
| 37 | };
|
| 38 |
|
| 39 | |
| 40 | * struct tmigr_group - timer migration hierarchy group
|
| 41 | * @lock: Lock protecting the event information and group hierarchy
|
| 42 | * information during setup
|
| 43 | * @parent: Pointer to the parent group. Pointer is updated when a
|
| 44 | * new hierarchy level is added because of a CPU coming
|
| 45 | * online the first time. Once it is set, the pointer will
|
| 46 | * not be removed or updated. When accessing parent pointer
|
| 47 | * lock less to decide whether to abort a propagation or
|
| 48 | * not, it is not a problem. The worst outcome is an
|
| 49 | * unnecessary/early CPU wake up. But do not access parent
|
| 50 | * pointer several times in the same 'action' (like
|
| 51 | * activation, deactivation, check for remote expiry,...)
|
| 52 | * without holding the lock as it is not ensured that value
|
| 53 | * will not change.
|
| 54 | * @groupevt: Next event of the group which is only used when the
|
| 55 | * group is !active. The group event is then queued into
|
| 56 | * the parent timer queue.
|
| 57 | * Ignore bit of @groupevt is set when the group is active.
|
| 58 | * @next_expiry: Base monotonic expiry time of the next event of the
|
| 59 | * group; It is used for the racy lockless check whether a
|
| 60 | * remote expiry is required; it is always reliable
|
| 61 | * @events: Timer queue for child events queued in the group
|
| 62 | * @migr_state: State of the group (see union tmigr_state)
|
| 63 | * @level: Hierarchy level of the group; Required during setup
|
| 64 | * @numa_node: Required for setup only to make sure CPU and low level
|
| 65 | * group information is NUMA local. It is set to NUMA node
|
| 66 | * as long as the group level is per NUMA node (level <
|
| 67 | * tmigr_crossnode_level); otherwise it is set to
|
| 68 | * NUMA_NO_NODE
|
| 69 | * @num_children: Counter of group children to make sure the group is only
|
| 70 | * filled with TMIGR_CHILDREN_PER_GROUP; Required for setup
|
| 71 | * only
|
| 72 | * @groupmask: mask of the group in the parent group; is set during
|
| 73 | * setup and will never change; can be read lockless
|
| 74 | * @list: List head that is added to the per level
|
| 75 | * tmigr_level_list; is required during setup when a
|
| 76 | * new group needs to be connected to the existing
|
| 77 | * hierarchy groups
|
| 78 |
|
| 79 | struct tmigr_group {
|
| 80 | raw_spinlock_t lock;
|
| 81 | struct tmigr_group *parent;
|
| 82 | struct tmigr_event groupevt;
|
| 83 | u64 next_expiry;
|
| 84 | struct timerqueue_head events;
|
| 85 | atomic_t migr_state;
|
| 86 | unsigned int level;
|
| 87 | int numa_node;
|
| 88 | unsigned int num_children;
|
| 89 | u8 groupmask;
|
| 90 | struct list_head list;
|
| 91 | };
|
| 92 |
|
| 93 | |
| 94 | * struct tmigr_cpu - timer migration per CPU group
|
| 95 | * @lock: Lock protecting the tmigr_cpu group information
|
| 96 | * @available: Indicates whether the CPU is available for handling
|
| 97 | * global timers. In the deactivate path it is required to
|
| 98 | * know whether the migrator in the top level group is to
|
| 99 | * be set offline, while a timer is pending. Then another
|
| 100 | * available CPU needs to be notified to take over the
|
| 101 | * migrator role. Furthermore the information is required
|
| 102 | * in the CPU hotplug path as the CPU is able to go idle
|
| 103 | * before the timer migration hierarchy hotplug callback is
|
| 104 | * reached. During this phase, the CPU has to handle the
|
| 105 | * global timers on its own and must not act as a migrator.
|
| 106 | *
|
| 107 | * @idle: Indicates whether the CPU is idle in the timer migration
|
| 108 | * hierarchy
|
| 109 | * @remote: Is set when timers of the CPU are expired remotely
|
| 110 | * @tmgroup: Pointer to the parent group
|
| 111 | * @groupmask: mask of tmigr_cpu in the parent group
|
| 112 | * @wakeup: Stores the first timer when the timer migration
|
| 113 | * hierarchy is completely idle and remote expiry was done;
|
| 114 | * is returned to timer code in the idle path and is only
|
| 115 | * used in idle path.
|
| 116 | * @cpuevt: CPU event which could be enqueued into the parent group
|
| 117 |
|
| 118 | struct tmigr_cpu {
|
| 119 | raw_spinlock_t lock;
|
| 120 | bool available;
|
| 121 | bool idle;
|
| 122 | bool remote;
|
| 123 | struct tmigr_group *tmgroup;
|
| 124 | u8 groupmask;
|
| 125 | u64 wakeup;
|
| 126 | struct tmigr_event cpuevt;
|
| 127 | };
|
| 128 |
|
| 129 | |
| 130 | * union tmigr_state - state of tmigr_group
|
| 131 | * @state: Combined version of the state - only used for atomic
|
| 132 | * read/cmpxchg function
|
| 133 | * &anon struct: Split version of the state - only use the struct members to
|
| 134 | * update information to stay independent of endianness
|
| 135 | * @active: Contains each mask bit of the active children
|
| 136 | * @migrator: Contains mask of the child which is migrator
|
| 137 | * @seq: Sequence counter needs to be increased when an update
|
| 138 | * to the tmigr_state is done. It prevents a race when
|
| 139 | * updates in the child groups are propagated in changed
|
| 140 | * order. Detailed information about the scenario is
|
| 141 | * given in the documentation at the begin of
|
| 142 | * timer_migration.c.
|
| 143 |
|
| 144 | union tmigr_state {
|
| 145 | u32 state;
|
| 146 | struct {
|
| 147 | u8 active;
|
| 148 | u8 migrator;
|
| 149 | u16 seq;
|
| 150 | } __packed;
|
| 151 | };
|
| 152 |
|
| 153 | #if defined(CONFIG_SMP) && defined(CONFIG_NO_HZ_COMMON)
|
| 154 | extern void tmigr_handle_remote(void);
|
| 155 | extern bool tmigr_requires_handle_remote(void);
|
| 156 | extern void tmigr_cpu_activate(void);
|
| 157 | extern u64 tmigr_cpu_deactivate(u64 nextevt);
|
| 158 | extern u64 tmigr_cpu_new_timer(u64 nextevt);
|
| 159 | extern u64 tmigr_quick_check(u64 nextevt);
|
| 160 | #else
|
| 161 | static inline void tmigr_handle_remote(void) { }
|
| 162 | static inline bool tmigr_requires_handle_remote(void) { return false; }
|
| 163 | static inline void tmigr_cpu_activate(void) { }
|
| 164 | #endif
|
| 165 |
|
| 166 | #endif
|