projects/linux·blob master · refs

784 lines · 25.1 KiB

1// SPDX-License-Identifier: GPL-2.0-or-later
2/*
3 * coupled.c - helper functions to enter the same idle state on multiple cpus
4 *
5 * Copyright (c) 2011 Google, Inc.
6 *
7 * Author: Colin Cross <[email protected]>
8 */
9
10#include <linux/kernel.h>
11#include <linux/cpu.h>
12#include <linux/cpuidle.h>
13#include <linux/mutex.h>
14#include <linux/sched.h>
15#include <linux/slab.h>
16#include <linux/spinlock.h>
17
18#include "cpuidle.h"
19
20/**
21 * DOC: Coupled cpuidle states
22 *
23 * On some ARM SMP SoCs (OMAP4460, Tegra 2, and probably more), the
24 * cpus cannot be independently powered down, either due to
25 * sequencing restrictions (on Tegra 2, cpu 0 must be the last to
26 * power down), or due to HW bugs (on OMAP4460, a cpu powering up
27 * will corrupt the gic state unless the other cpu runs a work
28 * around). Each cpu has a power state that it can enter without
29 * coordinating with the other cpu (usually Wait For Interrupt, or
30 * WFI), and one or more "coupled" power states that affect blocks
31 * shared between the cpus (L2 cache, interrupt controller, and
32 * sometimes the whole SoC). Entering a coupled power state must
33 * be tightly controlled on both cpus.
34 *
35 * This file implements a solution, where each cpu will wait in the
36 * WFI state until all cpus are ready to enter a coupled state, at
37 * which point the coupled state function will be called on all
38 * cpus at approximately the same time.
39 *
40 * Once all cpus are ready to enter idle, they are woken by an smp
41 * cross call. At this point, there is a chance that one of the
42 * cpus will find work to do, and choose not to enter idle. A
43 * final pass is needed to guarantee that all cpus will call the
44 * power state enter function at the same time. During this pass,
45 * each cpu will increment the ready counter, and continue once the
46 * ready counter matches the number of online coupled cpus. If any
47 * cpu exits idle, the other cpus will decrement their counter and
48 * retry.
49 *
50 * requested_state stores the deepest coupled idle state each cpu
51 * is ready for. It is assumed that the states are indexed from
52 * shallowest (highest power, lowest exit latency) to deepest
53 * (lowest power, highest exit latency). The requested_state
54 * variable is not locked. It is only written from the cpu that
55 * it stores (or by the on/offlining cpu if that cpu is offline),
56 * and only read after all the cpus are ready for the coupled idle
57 * state are no longer updating it.
58 *
59 * Three atomic counters are used. alive_count tracks the number
60 * of cpus in the coupled set that are currently or soon will be
61 * online. waiting_count tracks the number of cpus that are in
62 * the waiting loop, in the ready loop, or in the coupled idle state.
63 * ready_count tracks the number of cpus that are in the ready loop
64 * or in the coupled idle state.
65 *
66 * To use coupled cpuidle states, a cpuidle driver must:
67 *
68 * Set struct cpuidle_device.coupled_cpus to the mask of all
69 * coupled cpus, usually the same as cpu_possible_mask if all cpus
70 * are part of the same cluster. The coupled_cpus mask must be
71 * set in the struct cpuidle_device for each cpu.
72 *
73 * Set struct cpuidle_device.safe_state to a state that is not a
74 * coupled state. This is usually WFI.
75 *
76 * Set CPUIDLE_FLAG_COUPLED in struct cpuidle_state.flags for each
77 * state that affects multiple cpus.
78 *
79 * Provide a struct cpuidle_state.enter function for each state
80 * that affects multiple cpus. This function is guaranteed to be
81 * called on all cpus at approximately the same time. The driver
82 * should ensure that the cpus all abort together if any cpu tries
83 * to abort once the function is called. The function should return
84 * with interrupts still disabled.
85 */
86
87/**
88 * struct cpuidle_coupled - data for set of cpus that share a coupled idle state
89 * @coupled_cpus: mask of cpus that are part of the coupled set
90 * @requested_state: array of requested states for cpus in the coupled set
91 * @ready_waiting_counts: combined count of cpus in ready or waiting loops
92 * @abort_barrier: synchronisation point for abort cases
93 * @online_count: count of cpus that are online
94 * @refcnt: reference count of cpuidle devices that are using this struct
95 * @prevent: flag to prevent coupled idle while a cpu is hotplugging
96 */
97struct cpuidle_coupled {
98 cpumask_t coupled_cpus;
99 int requested_state[NR_CPUS];
100 atomic_t ready_waiting_counts;
101 atomic_t abort_barrier;
102 int online_count;
103 int refcnt;
104 int prevent;
105};
106
107#define WAITING_BITS 16
108#define MAX_WAITING_CPUS (1 << WAITING_BITS)
109#define WAITING_MASK (MAX_WAITING_CPUS - 1)
110#define READY_MASK (~WAITING_MASK)
111
112#define CPUIDLE_COUPLED_NOT_IDLE (-1)
113
114static DEFINE_PER_CPU(call_single_data_t, cpuidle_coupled_poke_cb);
115
116/*
117 * The cpuidle_coupled_poke_pending mask is used to avoid calling
118 * __smp_call_function_single with the per cpu call_single_data_t struct already
119 * in use. This prevents a deadlock where two cpus are waiting for each others
120 * call_single_data_t struct to be available
121 */
122static cpumask_t cpuidle_coupled_poke_pending;
123
124/*
125 * The cpuidle_coupled_poked mask is used to ensure that each cpu has been poked
126 * once to minimize entering the ready loop with a poke pending, which would
127 * require aborting and retrying.
128 */
129static cpumask_t cpuidle_coupled_poked;
130
131/**
132 * cpuidle_coupled_parallel_barrier - synchronize all online coupled cpus
133 * @dev: cpuidle_device of the calling cpu
134 * @a: atomic variable to hold the barrier
135 *
136 * No caller to this function will return from this function until all online
137 * cpus in the same coupled group have called this function. Once any caller
138 * has returned from this function, the barrier is immediately available for
139 * reuse.
140 *
141 * The atomic variable must be initialized to 0 before any cpu calls
142 * this function, will be reset to 0 before any cpu returns from this function.
143 *
144 * Must only be called from within a coupled idle state handler
145 * (state.enter when state.flags has CPUIDLE_FLAG_COUPLED set).
146 *
147 * Provides full smp barrier semantics before and after calling.
148 */
149void cpuidle_coupled_parallel_barrier(struct cpuidle_device *dev, atomic_t *a)
150{
151 int n = dev->coupled->online_count;
152
153 smp_mb__before_atomic();
154 atomic_inc(a);
155
156 while (atomic_read(a) < n)
157 cpu_relax();
158
159 if (atomic_inc_return(a) == n * 2) {
160 atomic_set(a, 0);
161 return;
162 }
163
164 while (atomic_read(a) > n)
165 cpu_relax();
166}
167
168/**
169 * cpuidle_state_is_coupled - check if a state is part of a coupled set
170 * @drv: struct cpuidle_driver for the platform
171 * @state: index of the target state in drv->states
172 *
173 * Returns true if the target state is coupled with cpus besides this one
174 */
175bool cpuidle_state_is_coupled(struct cpuidle_driver *drv, int state)
176{
177 return drv->states[state].flags & CPUIDLE_FLAG_COUPLED;
178}
179
180/**
181 * cpuidle_coupled_state_verify - check if the coupled states are correctly set.
182 * @drv: struct cpuidle_driver for the platform
183 *
184 * Returns 0 for valid state values, a negative error code otherwise:
185 * * -EINVAL if any coupled state(safe_state_index) is wrongly set.
186 */
187int cpuidle_coupled_state_verify(struct cpuidle_driver *drv)
188{
189 int i;
190
191 for (i = drv->state_count - 1; i >= 0; i--) {
192 if (cpuidle_state_is_coupled(drv, i) &&
193 (drv->safe_state_index == i ||
194 drv->safe_state_index < 0 ||
195 drv->safe_state_index >= drv->state_count))
196 return -EINVAL;
197 }
198
199 return 0;
200}
201
202/**
203 * cpuidle_coupled_set_ready - mark a cpu as ready
204 * @coupled: the struct coupled that contains the current cpu
205 */
206static inline void cpuidle_coupled_set_ready(struct cpuidle_coupled *coupled)
207{
208 atomic_add(MAX_WAITING_CPUS, &coupled->ready_waiting_counts);
209}
210
211/**
212 * cpuidle_coupled_set_not_ready - mark a cpu as not ready
213 * @coupled: the struct coupled that contains the current cpu
214 *
215 * Decrements the ready counter, unless the ready (and thus the waiting) counter
216 * is equal to the number of online cpus. Prevents a race where one cpu
217 * decrements the waiting counter and then re-increments it just before another
218 * cpu has decremented its ready counter, leading to the ready counter going
219 * down from the number of online cpus without going through the coupled idle
220 * state.
221 *
222 * Returns 0 if the counter was decremented successfully, -EINVAL if the ready
223 * counter was equal to the number of online cpus.
224 */
225static
226inline int cpuidle_coupled_set_not_ready(struct cpuidle_coupled *coupled)
227{
228 int all;
229 int ret;
230
231 all = coupled->online_count | (coupled->online_count << WAITING_BITS);
232 ret = atomic_add_unless(&coupled->ready_waiting_counts,
233 -MAX_WAITING_CPUS, all);
234
235 return ret ? 0 : -EINVAL;
236}
237
238/**
239 * cpuidle_coupled_no_cpus_ready - check if no cpus in a coupled set are ready
240 * @coupled: the struct coupled that contains the current cpu
241 *
242 * Returns true if all of the cpus in a coupled set are out of the ready loop.
243 */
244static inline int cpuidle_coupled_no_cpus_ready(struct cpuidle_coupled *coupled)
245{
246 int r = atomic_read(&coupled->ready_waiting_counts) >> WAITING_BITS;
247 return r == 0;
248}
249
250/**
251 * cpuidle_coupled_cpus_ready - check if all cpus in a coupled set are ready
252 * @coupled: the struct coupled that contains the current cpu
253 *
254 * Returns true if all cpus coupled to this target state are in the ready loop
255 */
256static inline bool cpuidle_coupled_cpus_ready(struct cpuidle_coupled *coupled)
257{
258 int r = atomic_read(&coupled->ready_waiting_counts) >> WAITING_BITS;
259 return r == coupled->online_count;
260}
261
262/**
263 * cpuidle_coupled_cpus_waiting - check if all cpus in a coupled set are waiting
264 * @coupled: the struct coupled that contains the current cpu
265 *
266 * Returns true if all cpus coupled to this target state are in the wait loop
267 */
268static inline bool cpuidle_coupled_cpus_waiting(struct cpuidle_coupled *coupled)
269{
270 int w = atomic_read(&coupled->ready_waiting_counts) & WAITING_MASK;
271 return w == coupled->online_count;
272}
273
274/**
275 * cpuidle_coupled_no_cpus_waiting - check if no cpus in coupled set are waiting
276 * @coupled: the struct coupled that contains the current cpu
277 *
278 * Returns true if all of the cpus in a coupled set are out of the waiting loop.
279 */
280static inline int cpuidle_coupled_no_cpus_waiting(struct cpuidle_coupled *coupled)
281{
282 int w = atomic_read(&coupled->ready_waiting_counts) & WAITING_MASK;
283 return w == 0;
284}
285
286/**
287 * cpuidle_coupled_get_state - determine the deepest idle state
288 * @dev: struct cpuidle_device for this cpu
289 * @coupled: the struct coupled that contains the current cpu
290 *
291 * Returns the deepest idle state that all coupled cpus can enter
292 */
293static inline int cpuidle_coupled_get_state(struct cpuidle_device *dev,
294 struct cpuidle_coupled *coupled)
295{
296 int i;
297 int state = INT_MAX;
298
299 /*
300 * Read barrier ensures that read of requested_state is ordered after
301 * reads of ready_count. Matches the write barriers
302 * cpuidle_set_state_waiting.
303 */
304 smp_rmb();
305
306 for_each_cpu(i, &coupled->coupled_cpus)
307 if (cpu_online(i) && coupled->requested_state[i] < state)
308 state = coupled->requested_state[i];
309
310 return state;
311}
312
313static void cpuidle_coupled_handle_poke(void *info)
314{
315 int cpu = (unsigned long)info;
316 cpumask_set_cpu(cpu, &cpuidle_coupled_poked);
317 cpumask_clear_cpu(cpu, &cpuidle_coupled_poke_pending);
318}
319
320/**
321 * cpuidle_coupled_poke - wake up a cpu that may be waiting
322 * @cpu: target cpu
323 *
324 * Ensures that the target cpu exits it's waiting idle state (if it is in it)
325 * and will see updates to waiting_count before it re-enters it's waiting idle
326 * state.
327 *
328 * If cpuidle_coupled_poked_mask is already set for the target cpu, that cpu
329 * either has or will soon have a pending IPI that will wake it out of idle,
330 * or it is currently processing the IPI and is not in idle.
331 */
332static void cpuidle_coupled_poke(int cpu)
333{
334 call_single_data_t *csd = &per_cpu(cpuidle_coupled_poke_cb, cpu);
335
336 if (!cpumask_test_and_set_cpu(cpu, &cpuidle_coupled_poke_pending))
337 smp_call_function_single_async(cpu, csd);
338}
339
340/**
341 * cpuidle_coupled_poke_others - wake up all other cpus that may be waiting
342 * @this_cpu: target cpu
343 * @coupled: the struct coupled that contains the current cpu
344 *
345 * Calls cpuidle_coupled_poke on all other online cpus.
346 */
347static void cpuidle_coupled_poke_others(int this_cpu,
348 struct cpuidle_coupled *coupled)
349{
350 int cpu;
351
352 for_each_cpu(cpu, &coupled->coupled_cpus)
353 if (cpu != this_cpu && cpu_online(cpu))
354 cpuidle_coupled_poke(cpu);
355}
356
357/**
358 * cpuidle_coupled_set_waiting - mark this cpu as in the wait loop
359 * @cpu: target cpu
360 * @coupled: the struct coupled that contains the current cpu
361 * @next_state: the index in drv->states of the requested state for this cpu
362 *
363 * Updates the requested idle state for the specified cpuidle device.
364 * Returns the number of waiting cpus.
365 */
366static int cpuidle_coupled_set_waiting(int cpu,
367 struct cpuidle_coupled *coupled, int next_state)
368{
369 coupled->requested_state[cpu] = next_state;
370
371 /*
372 * The atomic_inc_return provides a write barrier to order the write
373 * to requested_state with the later write that increments ready_count.
374 */
375 return atomic_inc_return(&coupled->ready_waiting_counts) & WAITING_MASK;
376}
377
378/**
379 * cpuidle_coupled_set_not_waiting - mark this cpu as leaving the wait loop
380 * @cpu: target cpu
381 * @coupled: the struct coupled that contains the current cpu
382 *
383 * Removes the requested idle state for the specified cpuidle device.
384 */
385static void cpuidle_coupled_set_not_waiting(int cpu,
386 struct cpuidle_coupled *coupled)
387{
388 /*
389 * Decrementing waiting count can race with incrementing it in
390 * cpuidle_coupled_set_waiting, but that's OK. Worst case, some
391 * cpus will increment ready_count and then spin until they
392 * notice that this cpu has cleared it's requested_state.
393 */
394 atomic_dec(&coupled->ready_waiting_counts);
395
396 coupled->requested_state[cpu] = CPUIDLE_COUPLED_NOT_IDLE;
397}
398
399/**
400 * cpuidle_coupled_set_done - mark this cpu as leaving the ready loop
401 * @cpu: the current cpu
402 * @coupled: the struct coupled that contains the current cpu
403 *
404 * Marks this cpu as no longer in the ready and waiting loops. Decrements
405 * the waiting count first to prevent another cpu looping back in and seeing
406 * this cpu as waiting just before it exits idle.
407 */
408static void cpuidle_coupled_set_done(int cpu, struct cpuidle_coupled *coupled)
409{
410 cpuidle_coupled_set_not_waiting(cpu, coupled);
411 atomic_sub(MAX_WAITING_CPUS, &coupled->ready_waiting_counts);
412}
413
414/**
415 * cpuidle_coupled_clear_pokes - spin until the poke interrupt is processed
416 * @cpu: this cpu
417 *
418 * Turns on interrupts and spins until any outstanding poke interrupts have
419 * been processed and the poke bit has been cleared.
420 *
421 * Other interrupts may also be processed while interrupts are enabled, so
422 * need_resched() must be tested after this function returns to make sure
423 * the interrupt didn't schedule work that should take the cpu out of idle.
424 *
425 * Returns 0 if no poke was pending, 1 if a poke was cleared.
426 */
427static int cpuidle_coupled_clear_pokes(int cpu)
428{
429 if (!cpumask_test_cpu(cpu, &cpuidle_coupled_poke_pending))
430 return 0;
431
432 local_irq_enable();
433 while (cpumask_test_cpu(cpu, &cpuidle_coupled_poke_pending))
434 cpu_relax();
435 local_irq_disable();
436
437 return 1;
438}
439
440static bool cpuidle_coupled_any_pokes_pending(struct cpuidle_coupled *coupled)
441{
442 return cpumask_first_and_and(cpu_online_mask, &coupled->coupled_cpus,
443 &cpuidle_coupled_poke_pending) < nr_cpu_ids;
444}
445
446/**
447 * cpuidle_enter_state_coupled - attempt to enter a state with coupled cpus
448 * @dev: struct cpuidle_device for the current cpu
449 * @drv: struct cpuidle_driver for the platform
450 * @next_state: index of the requested state in drv->states
451 *
452 * Coordinate with coupled cpus to enter the target state. This is a two
453 * stage process. In the first stage, the cpus are operating independently,
454 * and may call into cpuidle_enter_state_coupled at completely different times.
455 * To save as much power as possible, the first cpus to call this function will
456 * go to an intermediate state (the cpuidle_device's safe state), and wait for
457 * all the other cpus to call this function. Once all coupled cpus are idle,
458 * the second stage will start. Each coupled cpu will spin until all cpus have
459 * guaranteed that they will call the target_state.
460 *
461 * This function must be called with interrupts disabled. It may enable
462 * interrupts while preparing for idle, and it will always return with
463 * interrupts enabled.
464 */
465int cpuidle_enter_state_coupled(struct cpuidle_device *dev,
466 struct cpuidle_driver *drv, int next_state)
467{
468 int entered_state = -1;
469 struct cpuidle_coupled *coupled = dev->coupled;
470 int w;
471
472 if (!coupled)
473 return -EINVAL;
474
475 while (coupled->prevent) {
476 cpuidle_coupled_clear_pokes(dev->cpu);
477 if (need_resched()) {
478 local_irq_enable();
479 return entered_state;
480 }
481 entered_state = cpuidle_enter_state(dev, drv,
482 drv->safe_state_index);
483 local_irq_disable();
484 }
485
486 /* Read barrier ensures online_count is read after prevent is cleared */
487 smp_rmb();
488
489reset:
490 cpumask_clear_cpu(dev->cpu, &cpuidle_coupled_poked);
491
492 w = cpuidle_coupled_set_waiting(dev->cpu, coupled, next_state);
493 /*
494 * If this is the last cpu to enter the waiting state, poke
495 * all the other cpus out of their waiting state so they can
496 * enter a deeper state. This can race with one of the cpus
497 * exiting the waiting state due to an interrupt and
498 * decrementing waiting_count, see comment below.
499 */
500 if (w == coupled->online_count) {
501 cpumask_set_cpu(dev->cpu, &cpuidle_coupled_poked);
502 cpuidle_coupled_poke_others(dev->cpu, coupled);
503 }
504
505retry:
506 /*
507 * Wait for all coupled cpus to be idle, using the deepest state
508 * allowed for a single cpu. If this was not the poking cpu, wait
509 * for at least one poke before leaving to avoid a race where
510 * two cpus could arrive at the waiting loop at the same time,
511 * but the first of the two to arrive could skip the loop without
512 * processing the pokes from the last to arrive.
513 */
514 while (!cpuidle_coupled_cpus_waiting(coupled) ||
515 !cpumask_test_cpu(dev->cpu, &cpuidle_coupled_poked)) {
516 if (cpuidle_coupled_clear_pokes(dev->cpu))
517 continue;
518
519 if (need_resched()) {
520 cpuidle_coupled_set_not_waiting(dev->cpu, coupled);
521 goto out;
522 }
523
524 if (coupled->prevent) {
525 cpuidle_coupled_set_not_waiting(dev->cpu, coupled);
526 goto out;
527 }
528
529 entered_state = cpuidle_enter_state(dev, drv,
530 drv->safe_state_index);
531 local_irq_disable();
532 }
533
534 cpuidle_coupled_clear_pokes(dev->cpu);
535 if (need_resched()) {
536 cpuidle_coupled_set_not_waiting(dev->cpu, coupled);
537 goto out;
538 }
539
540 /*
541 * Make sure final poke status for this cpu is visible before setting
542 * cpu as ready.
543 */
544 smp_wmb();
545
546 /*
547 * All coupled cpus are probably idle. There is a small chance that
548 * one of the other cpus just became active. Increment the ready count,
549 * and spin until all coupled cpus have incremented the counter. Once a
550 * cpu has incremented the ready counter, it cannot abort idle and must
551 * spin until either all cpus have incremented the ready counter, or
552 * another cpu leaves idle and decrements the waiting counter.
553 */
554
555 cpuidle_coupled_set_ready(coupled);
556 while (!cpuidle_coupled_cpus_ready(coupled)) {
557 /* Check if any other cpus bailed out of idle. */
558 if (!cpuidle_coupled_cpus_waiting(coupled))
559 if (!cpuidle_coupled_set_not_ready(coupled))
560 goto retry;
561
562 cpu_relax();
563 }
564
565 /*
566 * Make sure read of all cpus ready is done before reading pending pokes
567 */
568 smp_rmb();
569
570 /*
571 * There is a small chance that a cpu left and reentered idle after this
572 * cpu saw that all cpus were waiting. The cpu that reentered idle will
573 * have sent this cpu a poke, which will still be pending after the
574 * ready loop. The pending interrupt may be lost by the interrupt
575 * controller when entering the deep idle state. It's not possible to
576 * clear a pending interrupt without turning interrupts on and handling
577 * it, and it's too late to turn on interrupts here, so reset the
578 * coupled idle state of all cpus and retry.
579 */
580 if (cpuidle_coupled_any_pokes_pending(coupled)) {
581 cpuidle_coupled_set_done(dev->cpu, coupled);
582 /* Wait for all cpus to see the pending pokes */
583 cpuidle_coupled_parallel_barrier(dev, &coupled->abort_barrier);
584 goto reset;
585 }
586
587 /* all cpus have acked the coupled state */
588 next_state = cpuidle_coupled_get_state(dev, coupled);
589
590 entered_state = cpuidle_enter_state(dev, drv, next_state);
591
592 cpuidle_coupled_set_done(dev->cpu, coupled);
593
594out:
595 /*
596 * Normal cpuidle states are expected to return with irqs enabled.
597 * That leads to an inefficiency where a cpu receiving an interrupt
598 * that brings it out of idle will process that interrupt before
599 * exiting the idle enter function and decrementing ready_count. All
600 * other cpus will need to spin waiting for the cpu that is processing
601 * the interrupt. If the driver returns with interrupts disabled,
602 * all other cpus will loop back into the safe idle state instead of
603 * spinning, saving power.
604 *
605 * Calling local_irq_enable here allows coupled states to return with
606 * interrupts disabled, but won't cause problems for drivers that
607 * exit with interrupts enabled.
608 */
609 local_irq_enable();
610
611 /*
612 * Wait until all coupled cpus have exited idle. There is no risk that
613 * a cpu exits and re-enters the ready state because this cpu has
614 * already decremented its waiting_count.
615 */
616 while (!cpuidle_coupled_no_cpus_ready(coupled))
617 cpu_relax();
618
619 return entered_state;
620}
621
622static void cpuidle_coupled_update_online_cpus(struct cpuidle_coupled *coupled)
623{
624 coupled->online_count = cpumask_weight_and(cpu_online_mask, &coupled->coupled_cpus);
625}
626
627/**
628 * cpuidle_coupled_register_device - register a coupled cpuidle device
629 * @dev: struct cpuidle_device for the current cpu
630 *
631 * Called from cpuidle_register_device to handle coupled idle init. Finds the
632 * cpuidle_coupled struct for this set of coupled cpus, or creates one if none
633 * exists yet.
634 */
635int cpuidle_coupled_register_device(struct cpuidle_device *dev)
636{
637 int cpu;
638 struct cpuidle_device *other_dev;
639 call_single_data_t *csd;
640 struct cpuidle_coupled *coupled;
641
642 if (cpumask_empty(&dev->coupled_cpus))
643 return 0;
644
645 for_each_cpu(cpu, &dev->coupled_cpus) {
646 other_dev = per_cpu(cpuidle_devices, cpu);
647 if (other_dev && other_dev->coupled) {
648 coupled = other_dev->coupled;
649 goto have_coupled;
650 }
651 }
652
653 /* No existing coupled info found, create a new one */
654 coupled = kzalloc_obj(struct cpuidle_coupled);
655 if (!coupled)
656 return -ENOMEM;
657
658 coupled->coupled_cpus = dev->coupled_cpus;
659
660have_coupled:
661 dev->coupled = coupled;
662 if (WARN_ON(!cpumask_equal(&dev->coupled_cpus, &coupled->coupled_cpus)))
663 coupled->prevent++;
664
665 cpuidle_coupled_update_online_cpus(coupled);
666
667 coupled->refcnt++;
668
669 csd = &per_cpu(cpuidle_coupled_poke_cb, dev->cpu);
670 INIT_CSD(csd, cpuidle_coupled_handle_poke, (void *)(unsigned long)dev->cpu);
671
672 return 0;
673}
674
675/**
676 * cpuidle_coupled_unregister_device - unregister a coupled cpuidle device
677 * @dev: struct cpuidle_device for the current cpu
678 *
679 * Called from cpuidle_unregister_device to tear down coupled idle. Removes the
680 * cpu from the coupled idle set, and frees the cpuidle_coupled_info struct if
681 * this was the last cpu in the set.
682 */
683void cpuidle_coupled_unregister_device(struct cpuidle_device *dev)
684{
685 struct cpuidle_coupled *coupled = dev->coupled;
686
687 if (cpumask_empty(&dev->coupled_cpus))
688 return;
689
690 if (--coupled->refcnt)
691 kfree(coupled);
692 dev->coupled = NULL;
693}
694
695/**
696 * cpuidle_coupled_prevent_idle - prevent cpus from entering a coupled state
697 * @coupled: the struct coupled that contains the cpu that is changing state
698 *
699 * Disables coupled cpuidle on a coupled set of cpus. Used to ensure that
700 * cpu_online_mask doesn't change while cpus are coordinating coupled idle.
701 */
702static void cpuidle_coupled_prevent_idle(struct cpuidle_coupled *coupled)
703{
704 int cpu = get_cpu();
705
706 /* Force all cpus out of the waiting loop. */
707 coupled->prevent++;
708 cpuidle_coupled_poke_others(cpu, coupled);
709 put_cpu();
710 while (!cpuidle_coupled_no_cpus_waiting(coupled))
711 cpu_relax();
712}
713
714/**
715 * cpuidle_coupled_allow_idle - allows cpus to enter a coupled state
716 * @coupled: the struct coupled that contains the cpu that is changing state
717 *
718 * Enables coupled cpuidle on a coupled set of cpus. Used to ensure that
719 * cpu_online_mask doesn't change while cpus are coordinating coupled idle.
720 */
721static void cpuidle_coupled_allow_idle(struct cpuidle_coupled *coupled)
722{
723 int cpu = get_cpu();
724
725 /*
726 * Write barrier ensures readers see the new online_count when they
727 * see prevent == 0.
728 */
729 smp_wmb();
730 coupled->prevent--;
731 /* Force cpus out of the prevent loop. */
732 cpuidle_coupled_poke_others(cpu, coupled);
733 put_cpu();
734}
735
736static int coupled_cpu_online(unsigned int cpu)
737{
738 struct cpuidle_device *dev;
739
740 mutex_lock(&cpuidle_lock);
741
742 dev = per_cpu(cpuidle_devices, cpu);
743 if (dev && dev->coupled) {
744 cpuidle_coupled_update_online_cpus(dev->coupled);
745 cpuidle_coupled_allow_idle(dev->coupled);
746 }
747
748 mutex_unlock(&cpuidle_lock);
749 return 0;
750}
751
752static int coupled_cpu_up_prepare(unsigned int cpu)
753{
754 struct cpuidle_device *dev;
755
756 mutex_lock(&cpuidle_lock);
757
758 dev = per_cpu(cpuidle_devices, cpu);
759 if (dev && dev->coupled)
760 cpuidle_coupled_prevent_idle(dev->coupled);
761
762 mutex_unlock(&cpuidle_lock);
763 return 0;
764}
765
766static int __init cpuidle_coupled_init(void)
767{
768 int ret;
769
770 ret = cpuhp_setup_state_nocalls(CPUHP_CPUIDLE_COUPLED_PREPARE,
771 "cpuidle/coupled:prepare",
772 coupled_cpu_up_prepare,
773 coupled_cpu_online);
774 if (ret)
775 return ret;
776 ret = cpuhp_setup_state_nocalls(CPUHP_AP_ONLINE_DYN,
777 "cpuidle/coupled:online",
778 coupled_cpu_online,
779 coupled_cpu_up_prepare);
780 if (ret < 0)
781 cpuhp_remove_state_nocalls(CPUHP_CPUIDLE_COUPLED_PREPARE);
782 return ret;
783}
784core_initcall(cpuidle_coupled_init);