projects/linux·blob master · refs

2442 lines · 60.9 KiB

1// SPDX-License-Identifier: GPL-2.0
2/*
3 * drivers/base/power/main.c - Where the driver meets power management.
4 *
5 * Copyright (c) 2003 Patrick Mochel
6 * Copyright (c) 2003 Open Source Development Lab
7 *
8 * The driver model core calls device_pm_add() when a device is registered.
9 * This will initialize the embedded device_pm_info object in the device
10 * and add it to the list of power-controlled devices. sysfs entries for
11 * controlling device power management will also be added.
12 *
13 * A separate list is used for keeping track of power info, because the power
14 * domain dependencies may differ from the ancestral dependencies that the
15 * subsystem list maintains.
16 */
17
18#define pr_fmt(fmt) "PM: " fmt
19#define dev_fmt pr_fmt
20
21#include <linux/device.h>
22#include <linux/export.h>
23#include <linux/mutex.h>
24#include <linux/pm.h>
25#include <linux/pm_runtime.h>
26#include <linux/pm-trace.h>
27#include <linux/pm_wakeirq.h>
28#include <linux/interrupt.h>
29#include <linux/sched.h>
30#include <linux/sched/debug.h>
31#include <linux/sysctl.h>
32#include <linux/async.h>
33#include <linux/suspend.h>
34#include <trace/events/power.h>
35#include <linux/cpufreq.h>
36#include <linux/devfreq.h>
37#include <linux/thermal.h>
38#include <linux/timer.h>
39#include <linux/nmi.h>
40
41#include "../base.h"
42#include "power.h"
43
44#undef MODULE_PARAM_PREFIX
45#define MODULE_PARAM_PREFIX "pm."
46
47typedef int (*pm_callback_t)(struct device *);
48
49/*
50 * The entries in the dpm_list list are in a depth first order, simply
51 * because children are guaranteed to be discovered after parents, and
52 * are inserted at the back of the list on discovery.
53 *
54 * Since device_pm_add() may be called with a device lock held,
55 * we must never try to acquire a device lock while holding
56 * dpm_list_mutex.
57 */
58
59LIST_HEAD(dpm_list);
60static LIST_HEAD(dpm_prepared_list);
61static LIST_HEAD(dpm_suspended_list);
62static LIST_HEAD(dpm_late_early_list);
63static LIST_HEAD(dpm_noirq_list);
64
65static DEFINE_MUTEX(dpm_list_mtx);
66static pm_message_t pm_transition;
67
68static DEFINE_MUTEX(async_wip_mtx);
69static int async_error;
70
71/**
72 * pm_hibernate_is_recovering - if recovering from hibernate due to error.
73 *
74 * Used to query if dev_pm_ops.thaw() is called for normal hibernation case or
75 * recovering from some error.
76 *
77 * Return: true for error case, false for normal case.
78 */
79bool pm_hibernate_is_recovering(void)
80{
81 return pm_transition.event == PM_EVENT_RECOVER;
82}
83EXPORT_SYMBOL_GPL(pm_hibernate_is_recovering);
84
85static const char *pm_verb(int event)
86{
87 switch (event) {
88 case PM_EVENT_SUSPEND:
89 return "suspend";
90 case PM_EVENT_RESUME:
91 return "resume";
92 case PM_EVENT_FREEZE:
93 return "freeze";
94 case PM_EVENT_QUIESCE:
95 return "quiesce";
96 case PM_EVENT_HIBERNATE:
97 return "hibernate";
98 case PM_EVENT_THAW:
99 return "thaw";
100 case PM_EVENT_RESTORE:
101 return "restore";
102 case PM_EVENT_RECOVER:
103 return "recover";
104 case PM_EVENT_POWEROFF:
105 return "poweroff";
106 default:
107 return "(unknown PM event)";
108 }
109}
110
111/**
112 * device_pm_sleep_init - Initialize system suspend-related device fields.
113 * @dev: Device object being initialized.
114 */
115void device_pm_sleep_init(struct device *dev)
116{
117 dev->power.is_prepared = false;
118 dev->power.is_suspended = false;
119 dev->power.is_noirq_suspended = false;
120 dev->power.is_late_suspended = false;
121 init_completion(&dev->power.completion);
122 complete(&dev->power.completion);
123 dev->power.wakeup = NULL;
124 INIT_LIST_HEAD(&dev->power.entry);
125}
126
127/**
128 * device_pm_lock - Lock the list of active devices used by the PM core.
129 */
130void device_pm_lock(void)
131{
132 mutex_lock(&dpm_list_mtx);
133}
134
135/**
136 * device_pm_unlock - Unlock the list of active devices used by the PM core.
137 */
138void device_pm_unlock(void)
139{
140 mutex_unlock(&dpm_list_mtx);
141}
142
143/**
144 * device_pm_add - Add a device to the PM core's list of active devices.
145 * @dev: Device to add to the list.
146 */
147void device_pm_add(struct device *dev)
148{
149 /* Skip PM setup/initialization. */
150 if (device_pm_not_required(dev))
151 return;
152
153 pr_debug("Adding info for %s:%s\n",
154 dev->bus ? dev->bus->name : "No Bus", dev_name(dev));
155 device_pm_check_callbacks(dev);
156 mutex_lock(&dpm_list_mtx);
157 if (dev->parent && dev->parent->power.is_prepared)
158 dev_warn(dev, "parent %s should not be sleeping\n",
159 dev_name(dev->parent));
160 list_add_tail(&dev->power.entry, &dpm_list);
161 dev->power.in_dpm_list = true;
162 mutex_unlock(&dpm_list_mtx);
163}
164
165/**
166 * device_pm_remove - Remove a device from the PM core's list of active devices.
167 * @dev: Device to be removed from the list.
168 */
169void device_pm_remove(struct device *dev)
170{
171 if (device_pm_not_required(dev))
172 return;
173
174 pr_debug("Removing info for %s:%s\n",
175 dev->bus ? dev->bus->name : "No Bus", dev_name(dev));
176 complete_all(&dev->power.completion);
177 mutex_lock(&dpm_list_mtx);
178 list_del_init(&dev->power.entry);
179 dev->power.in_dpm_list = false;
180 mutex_unlock(&dpm_list_mtx);
181 device_wakeup_disable(dev);
182 pm_runtime_remove(dev);
183 device_pm_check_callbacks(dev);
184}
185
186/**
187 * device_pm_move_before - Move device in the PM core's list of active devices.
188 * @deva: Device to move in dpm_list.
189 * @devb: Device @deva should come before.
190 */
191void device_pm_move_before(struct device *deva, struct device *devb)
192{
193 pr_debug("Moving %s:%s before %s:%s\n",
194 deva->bus ? deva->bus->name : "No Bus", dev_name(deva),
195 devb->bus ? devb->bus->name : "No Bus", dev_name(devb));
196 /* Delete deva from dpm_list and reinsert before devb. */
197 list_move_tail(&deva->power.entry, &devb->power.entry);
198}
199
200/**
201 * device_pm_move_after - Move device in the PM core's list of active devices.
202 * @deva: Device to move in dpm_list.
203 * @devb: Device @deva should come after.
204 */
205void device_pm_move_after(struct device *deva, struct device *devb)
206{
207 pr_debug("Moving %s:%s after %s:%s\n",
208 deva->bus ? deva->bus->name : "No Bus", dev_name(deva),
209 devb->bus ? devb->bus->name : "No Bus", dev_name(devb));
210 /* Delete deva from dpm_list and reinsert after devb. */
211 list_move(&deva->power.entry, &devb->power.entry);
212}
213
214/**
215 * device_pm_move_last - Move device to end of the PM core's list of devices.
216 * @dev: Device to move in dpm_list.
217 */
218void device_pm_move_last(struct device *dev)
219{
220 pr_debug("Moving %s:%s to end of list\n",
221 dev->bus ? dev->bus->name : "No Bus", dev_name(dev));
222 list_move_tail(&dev->power.entry, &dpm_list);
223}
224
225static ktime_t initcall_debug_start(struct device *dev, void *cb)
226{
227 if (!pm_print_times_enabled)
228 return 0;
229
230 dev_info(dev, "calling %ps @ %i, parent: %s\n", cb,
231 task_pid_nr(current),
232 dev->parent ? dev_name(dev->parent) : "none");
233 return ktime_get();
234}
235
236static void initcall_debug_report(struct device *dev, ktime_t calltime,
237 void *cb, int error)
238{
239 ktime_t rettime;
240
241 if (!pm_print_times_enabled)
242 return;
243
244 rettime = ktime_get();
245 dev_info(dev, "%ps returned %d after %Ld usecs\n", cb, error,
246 (unsigned long long)ktime_us_delta(rettime, calltime));
247}
248
249/**
250 * dpm_wait - Wait for a PM operation to complete.
251 * @dev: Device to wait for.
252 * @async: If unset, wait only if the device's power.async_suspend flag is set.
253 */
254static void dpm_wait(struct device *dev, bool async)
255{
256 if (!dev)
257 return;
258
259 /* Devices with no PM support don't use the completion. */
260 if (dev->power.no_pm)
261 return;
262
263 if (async || (pm_async_enabled && dev->power.async_suspend))
264 wait_for_completion(&dev->power.completion);
265}
266
267static int dpm_wait_fn(struct device *dev, void *async_ptr)
268{
269 dpm_wait(dev, *((bool *)async_ptr));
270 return 0;
271}
272
273static void dpm_wait_for_children(struct device *dev, bool async)
274{
275 device_for_each_child(dev, &async, dpm_wait_fn);
276}
277
278static void dpm_wait_for_suppliers(struct device *dev, bool async)
279{
280 struct device_link *link;
281 int idx;
282
283 idx = device_links_read_lock();
284
285 /*
286 * If the supplier goes away right after we've checked the link to it,
287 * we'll wait for its completion to change the state, but that's fine,
288 * because the only things that will block as a result are the SRCU
289 * callbacks freeing the link objects for the links in the list we're
290 * walking.
291 */
292 dev_for_each_link_to_supplier(link, dev)
293 if (READ_ONCE(link->status) != DL_STATE_DORMANT &&
294 !device_link_flag_is_sync_state_only(link->flags))
295 dpm_wait(link->supplier, async);
296
297 device_links_read_unlock(idx);
298}
299
300static bool dpm_wait_for_superior(struct device *dev, bool async)
301{
302 struct device *parent;
303
304 /*
305 * If the device is resumed asynchronously and the parent's callback
306 * deletes both the device and the parent itself, the parent object may
307 * be freed while this function is running, so avoid that by reference
308 * counting the parent once more unless the device has been deleted
309 * already (in which case return right away).
310 */
311 mutex_lock(&dpm_list_mtx);
312
313 if (!device_pm_initialized(dev)) {
314 mutex_unlock(&dpm_list_mtx);
315 return false;
316 }
317
318 parent = get_device(dev->parent);
319
320 mutex_unlock(&dpm_list_mtx);
321
322 dpm_wait(parent, async);
323 put_device(parent);
324
325 dpm_wait_for_suppliers(dev, async);
326
327 /*
328 * If the parent's callback has deleted the device, attempting to resume
329 * it would be invalid, so avoid doing that then.
330 */
331 return device_pm_initialized(dev);
332}
333
334static void dpm_wait_for_consumers(struct device *dev, bool async)
335{
336 struct device_link *link;
337 int idx;
338
339 idx = device_links_read_lock();
340
341 /*
342 * The status of a device link can only be changed from "dormant" by a
343 * probe, but that cannot happen during system suspend/resume. In
344 * theory it can change to "dormant" at that time, but then it is
345 * reasonable to wait for the target device anyway (eg. if it goes
346 * away, it's better to wait for it to go away completely and then
347 * continue instead of trying to continue in parallel with its
348 * unregistration).
349 */
350 dev_for_each_link_to_consumer(link, dev)
351 if (READ_ONCE(link->status) != DL_STATE_DORMANT &&
352 !device_link_flag_is_sync_state_only(link->flags))
353 dpm_wait(link->consumer, async);
354
355 device_links_read_unlock(idx);
356}
357
358static void dpm_wait_for_subordinate(struct device *dev, bool async)
359{
360 dpm_wait_for_children(dev, async);
361 dpm_wait_for_consumers(dev, async);
362}
363
364/**
365 * pm_op - Return the PM operation appropriate for given PM event.
366 * @ops: PM operations to choose from.
367 * @state: PM transition of the system being carried out.
368 */
369static pm_callback_t pm_op(const struct dev_pm_ops *ops, pm_message_t state)
370{
371 switch (state.event) {
372#ifdef CONFIG_SUSPEND
373 case PM_EVENT_SUSPEND:
374 return ops->suspend;
375 case PM_EVENT_RESUME:
376 return ops->resume;
377#endif /* CONFIG_SUSPEND */
378#ifdef CONFIG_HIBERNATE_CALLBACKS
379 case PM_EVENT_FREEZE:
380 case PM_EVENT_QUIESCE:
381 return ops->freeze;
382 case PM_EVENT_POWEROFF:
383 case PM_EVENT_HIBERNATE:
384 return ops->poweroff;
385 case PM_EVENT_THAW:
386 case PM_EVENT_RECOVER:
387 return ops->thaw;
388 case PM_EVENT_RESTORE:
389 return ops->restore;
390#endif /* CONFIG_HIBERNATE_CALLBACKS */
391 }
392
393 return NULL;
394}
395
396/**
397 * pm_late_early_op - Return the PM operation appropriate for given PM event.
398 * @ops: PM operations to choose from.
399 * @state: PM transition of the system being carried out.
400 *
401 * Runtime PM is disabled for @dev while this function is being executed.
402 */
403static pm_callback_t pm_late_early_op(const struct dev_pm_ops *ops,
404 pm_message_t state)
405{
406 switch (state.event) {
407#ifdef CONFIG_SUSPEND
408 case PM_EVENT_SUSPEND:
409 return ops->suspend_late;
410 case PM_EVENT_RESUME:
411 return ops->resume_early;
412#endif /* CONFIG_SUSPEND */
413#ifdef CONFIG_HIBERNATE_CALLBACKS
414 case PM_EVENT_FREEZE:
415 case PM_EVENT_QUIESCE:
416 return ops->freeze_late;
417 case PM_EVENT_POWEROFF:
418 case PM_EVENT_HIBERNATE:
419 return ops->poweroff_late;
420 case PM_EVENT_THAW:
421 case PM_EVENT_RECOVER:
422 return ops->thaw_early;
423 case PM_EVENT_RESTORE:
424 return ops->restore_early;
425#endif /* CONFIG_HIBERNATE_CALLBACKS */
426 }
427
428 return NULL;
429}
430
431/**
432 * pm_noirq_op - Return the PM operation appropriate for given PM event.
433 * @ops: PM operations to choose from.
434 * @state: PM transition of the system being carried out.
435 *
436 * The driver of @dev will not receive interrupts while this function is being
437 * executed.
438 */
439static pm_callback_t pm_noirq_op(const struct dev_pm_ops *ops, pm_message_t state)
440{
441 switch (state.event) {
442#ifdef CONFIG_SUSPEND
443 case PM_EVENT_SUSPEND:
444 return ops->suspend_noirq;
445 case PM_EVENT_RESUME:
446 return ops->resume_noirq;
447#endif /* CONFIG_SUSPEND */
448#ifdef CONFIG_HIBERNATE_CALLBACKS
449 case PM_EVENT_FREEZE:
450 case PM_EVENT_QUIESCE:
451 return ops->freeze_noirq;
452 case PM_EVENT_POWEROFF:
453 case PM_EVENT_HIBERNATE:
454 return ops->poweroff_noirq;
455 case PM_EVENT_THAW:
456 case PM_EVENT_RECOVER:
457 return ops->thaw_noirq;
458 case PM_EVENT_RESTORE:
459 return ops->restore_noirq;
460#endif /* CONFIG_HIBERNATE_CALLBACKS */
461 }
462
463 return NULL;
464}
465
466static void pm_dev_dbg(struct device *dev, pm_message_t state, const char *info)
467{
468 dev_dbg(dev, "%s%s%s driver flags: %x\n", info, pm_verb(state.event),
469 ((state.event & PM_EVENT_SLEEP) && device_may_wakeup(dev)) ?
470 ", may wakeup" : "", dev->power.driver_flags);
471}
472
473static void pm_dev_err(struct device *dev, pm_message_t state, const char *info,
474 int error)
475{
476 dev_err(dev, "failed to %s%s: error %d\n", pm_verb(state.event), info,
477 error);
478}
479
480static void dpm_show_time(ktime_t starttime, pm_message_t state, int error,
481 const char *info)
482{
483 ktime_t calltime;
484 u64 usecs64;
485 int usecs;
486
487 calltime = ktime_get();
488 usecs64 = ktime_to_ns(ktime_sub(calltime, starttime));
489 do_div(usecs64, NSEC_PER_USEC);
490 usecs = usecs64;
491 if (usecs == 0)
492 usecs = 1;
493
494 pm_pr_dbg("%s%s%s of devices %s after %ld.%03ld msecs\n",
495 info ?: "", info ? " " : "", pm_verb(state.event),
496 error ? "aborted" : "complete",
497 usecs / USEC_PER_MSEC, usecs % USEC_PER_MSEC);
498}
499
500static int dpm_run_callback(pm_callback_t cb, struct device *dev,
501 pm_message_t state, const char *info)
502{
503 ktime_t calltime;
504 int error;
505
506 if (!cb)
507 return 0;
508
509 calltime = initcall_debug_start(dev, cb);
510
511 pm_dev_dbg(dev, state, info);
512 trace_device_pm_callback_start(dev, info, state.event);
513 error = cb(dev);
514 trace_device_pm_callback_end(dev, error);
515 suspend_report_result(dev, cb, error);
516
517 initcall_debug_report(dev, calltime, cb, error);
518
519 return error;
520}
521
522#ifdef CONFIG_DPM_WATCHDOG
523struct dpm_watchdog {
524 struct device *dev;
525 struct task_struct *tsk;
526 struct timer_list timer;
527 bool fatal;
528};
529
530#define DECLARE_DPM_WATCHDOG_ON_STACK(wd) \
531 struct dpm_watchdog wd
532
533static bool __read_mostly dpm_watchdog_all_cpu_backtrace;
534module_param(dpm_watchdog_all_cpu_backtrace, bool, 0644);
535MODULE_PARM_DESC(dpm_watchdog_all_cpu_backtrace,
536 "Backtrace all CPUs on DPM watchdog timeout");
537
538static bool __read_mostly dpm_watchdog_enabled =
539 IS_ENABLED(CONFIG_DPM_WATCHDOG_ENABLED);
540module_param(dpm_watchdog_enabled, bool, 0644);
541MODULE_PARM_DESC(dpm_watchdog_enabled, "Enable DPM watchdog");
542
543static unsigned int __read_mostly dpm_watchdog_timeout = CONFIG_DPM_WATCHDOG_TIMEOUT;
544static unsigned int __read_mostly dpm_watchdog_warning_timeout =
545 CONFIG_DPM_WATCHDOG_WARNING_TIMEOUT;
546static const unsigned int dpm_watchdog_timeout_max = CONFIG_DPM_WATCHDOG_TIMEOUT;
547
548static int proc_dodpm_watchdog_timeout_secs(const struct ctl_table *table,
549 int write, void *buffer,
550 size_t *lenp, loff_t *ppos)
551{
552 struct ctl_table ctl = *table;
553 unsigned int val = dpm_watchdog_timeout;
554 int ret;
555
556 ctl.data = &val;
557 ret = proc_douintvec_minmax(&ctl, write, buffer, lenp, ppos);
558 if (ret || !write)
559 return ret;
560
561 if (val < dpm_watchdog_warning_timeout)
562 dpm_watchdog_warning_timeout = val;
563 dpm_watchdog_timeout = val;
564
565 return 0;
566}
567
568static const struct ctl_table dpm_watchdog_sysctls[] = {
569 {
570 .procname = "dpm_watchdog_timeout_secs",
571 .maxlen = sizeof(unsigned int),
572 .mode = 0644,
573 .proc_handler = proc_dodpm_watchdog_timeout_secs,
574 .extra1 = SYSCTL_ONE,
575 .extra2 = (void *)&dpm_watchdog_timeout_max,
576 },
577 {
578 .procname = "dpm_watchdog_warning_timeout_secs",
579 .data = &dpm_watchdog_warning_timeout,
580 .maxlen = sizeof(unsigned int),
581 .mode = 0644,
582 .proc_handler = proc_douintvec_minmax,
583 .extra1 = SYSCTL_ONE,
584 .extra2 = (void *)&dpm_watchdog_timeout,
585 },
586};
587
588static int __init dpm_watchdog_sysctl_init(void)
589{
590 register_sysctl_init("kernel", dpm_watchdog_sysctls);
591 return 0;
592}
593subsys_initcall(dpm_watchdog_sysctl_init);
594
595/**
596 * dpm_watchdog_handler - Driver suspend / resume watchdog handler.
597 * @t: The timer that PM watchdog depends on.
598 *
599 * Called when a driver has timed out suspending or resuming.
600 * There's not much we can do here to recover so panic() to
601 * capture a crash-dump in pstore.
602 */
603static void dpm_watchdog_handler(struct timer_list *t)
604{
605 struct dpm_watchdog *wd = timer_container_of(wd, t, timer);
606 struct timer_list *timer = &wd->timer;
607 unsigned int time_left;
608
609 if (wd->fatal) {
610 unsigned int this_cpu = smp_processor_id();
611
612 dev_emerg(wd->dev, "**** DPM device timeout ****\n");
613 show_stack(wd->tsk, NULL, KERN_EMERG);
614 if (dpm_watchdog_all_cpu_backtrace)
615 trigger_allbutcpu_cpu_backtrace(this_cpu);
616 panic("%s %s: unrecoverable failure\n",
617 dev_driver_string(wd->dev), dev_name(wd->dev));
618 }
619
620 time_left = dpm_watchdog_timeout - dpm_watchdog_warning_timeout;
621 dev_warn(wd->dev, "**** DPM device timeout after %u seconds; %u seconds until panic ****\n",
622 dpm_watchdog_warning_timeout, time_left);
623 show_stack(wd->tsk, NULL, KERN_WARNING);
624
625 wd->fatal = true;
626 mod_timer(timer, jiffies + HZ * time_left);
627}
628
629/**
630 * dpm_watchdog_set - Enable pm watchdog for given device.
631 * @wd: Watchdog. Must be allocated on the stack.
632 * @dev: Device to handle.
633 */
634static void dpm_watchdog_set(struct dpm_watchdog *wd, struct device *dev)
635{
636 struct timer_list *timer = &wd->timer;
637
638 if (!dpm_watchdog_enabled)
639 return;
640
641 wd->dev = dev;
642 wd->tsk = current;
643 wd->fatal = dpm_watchdog_timeout == dpm_watchdog_warning_timeout;
644
645 timer_setup_on_stack(timer, dpm_watchdog_handler, 0);
646 /* use same timeout value for both suspend and resume */
647 timer->expires = jiffies + HZ * dpm_watchdog_warning_timeout;
648 add_timer(timer);
649}
650
651/**
652 * dpm_watchdog_clear - Disable suspend/resume watchdog.
653 * @wd: Watchdog to disable.
654 */
655static void dpm_watchdog_clear(struct dpm_watchdog *wd)
656{
657 struct timer_list *timer = &wd->timer;
658
659 if (!dpm_watchdog_enabled)
660 return;
661
662 timer_delete_sync(timer);
663 timer_destroy_on_stack(timer);
664}
665#else
666#define DECLARE_DPM_WATCHDOG_ON_STACK(wd)
667#define dpm_watchdog_set(x, y)
668#define dpm_watchdog_clear(x)
669#endif
670
671/*------------------------- Resume routines -------------------------*/
672
673/**
674 * dev_pm_skip_resume - System-wide device resume optimization check.
675 * @dev: Target device.
676 *
677 * Return:
678 * - %false if the transition under way is RESTORE.
679 * - Return value of dev_pm_skip_suspend() if the transition under way is THAW.
680 * - The logical negation of %power.must_resume otherwise (that is, when the
681 * transition under way is RESUME).
682 */
683bool dev_pm_skip_resume(struct device *dev)
684{
685 if (pm_transition.event == PM_EVENT_RESTORE)
686 return false;
687
688 if (pm_transition.event == PM_EVENT_THAW)
689 return dev_pm_skip_suspend(dev);
690
691 return !dev->power.must_resume;
692}
693
694static bool is_async(struct device *dev)
695{
696 return dev->power.async_suspend && pm_async_enabled
697 && !pm_trace_is_enabled();
698}
699
700static bool __dpm_async(struct device *dev, async_func_t func)
701{
702 if (dev->power.work_in_progress)
703 return true;
704
705 if (!is_async(dev))
706 return false;
707
708 dev->power.work_in_progress = true;
709
710 get_device(dev);
711
712 if (async_schedule_dev_nocall(func, dev))
713 return true;
714
715 put_device(dev);
716
717 return false;
718}
719
720static bool dpm_async_fn(struct device *dev, async_func_t func)
721{
722 guard(mutex)(&async_wip_mtx);
723
724 return __dpm_async(dev, func);
725}
726
727static int dpm_async_with_cleanup(struct device *dev, void *fn)
728{
729 guard(mutex)(&async_wip_mtx);
730
731 if (!__dpm_async(dev, fn))
732 dev->power.work_in_progress = false;
733
734 return 0;
735}
736
737static void dpm_async_resume_children(struct device *dev, async_func_t func)
738{
739 /*
740 * Prevent racing with dpm_clear_async_state() during initial list
741 * walks in dpm_noirq_resume_devices(), dpm_resume_early(), and
742 * dpm_resume().
743 */
744 guard(mutex)(&dpm_list_mtx);
745
746 /*
747 * Start processing "async" children of the device unless it's been
748 * started already for them.
749 */
750 device_for_each_child(dev, func, dpm_async_with_cleanup);
751}
752
753static void dpm_async_resume_subordinate(struct device *dev, async_func_t func)
754{
755 struct device_link *link;
756 int idx;
757
758 dpm_async_resume_children(dev, func);
759
760 idx = device_links_read_lock();
761
762 /* Start processing the device's "async" consumers. */
763 dev_for_each_link_to_consumer(link, dev)
764 if (READ_ONCE(link->status) != DL_STATE_DORMANT)
765 dpm_async_with_cleanup(link->consumer, func);
766
767 device_links_read_unlock(idx);
768}
769
770static void dpm_clear_async_state(struct device *dev)
771{
772 reinit_completion(&dev->power.completion);
773 dev->power.work_in_progress = false;
774}
775
776static bool dpm_root_device(struct device *dev)
777{
778 lockdep_assert_held(&dpm_list_mtx);
779
780 /*
781 * Since this function is required to run under dpm_list_mtx, the
782 * list_empty() below will only return true if the device's list of
783 * consumers is actually empty before calling it.
784 */
785 return !dev->parent && list_empty(&dev->links.suppliers);
786}
787
788static void async_resume_noirq(void *data, async_cookie_t cookie);
789
790/**
791 * device_resume_noirq - Execute a "noirq resume" callback for given device.
792 * @dev: Device to handle.
793 * @state: PM transition of the system being carried out.
794 * @async: If true, the device is being resumed asynchronously.
795 *
796 * The driver of @dev will not receive interrupts while this function is being
797 * executed.
798 */
799static void device_resume_noirq(struct device *dev, pm_message_t state, bool async)
800{
801 pm_callback_t callback = NULL;
802 const char *info = NULL;
803 bool skip_resume;
804 int error = 0;
805
806 TRACE_DEVICE(dev);
807 TRACE_RESUME(0);
808
809 if (dev->power.syscore || dev->power.direct_complete)
810 goto Out;
811
812 if (!dev->power.is_noirq_suspended) {
813 /*
814 * This means that system suspend has been aborted in the noirq
815 * phase before invoking the noirq suspend callback for the
816 * device, so if device_suspend_late() has left it in suspend,
817 * device_resume_early() should leave it in suspend either in
818 * case the early resume of it depends on the noirq resume that
819 * has not run.
820 */
821 if (dev_pm_skip_suspend(dev))
822 dev->power.must_resume = false;
823
824 goto Out;
825 }
826
827 if (!dpm_wait_for_superior(dev, async))
828 goto Out;
829
830 skip_resume = dev_pm_skip_resume(dev);
831 /*
832 * If the driver callback is skipped below or by the middle layer
833 * callback and device_resume_early() also skips the driver callback for
834 * this device later, it needs to appear as "suspended" to PM-runtime,
835 * so change its status accordingly.
836 *
837 * Otherwise, the device is going to be resumed, so set its PM-runtime
838 * status to "active" unless its power.smart_suspend flag is clear, in
839 * which case it is not necessary to update its PM-runtime status.
840 */
841 if (skip_resume)
842 pm_runtime_set_suspended(dev);
843 else if (dev_pm_smart_suspend(dev))
844 pm_runtime_set_active(dev);
845
846 if (dev->pm_domain) {
847 info = "noirq power domain ";
848 callback = pm_noirq_op(&dev->pm_domain->ops, state);
849 } else if (dev->type && dev->type->pm) {
850 info = "noirq type ";
851 callback = pm_noirq_op(dev->type->pm, state);
852 } else if (dev->class && dev->class->pm) {
853 info = "noirq class ";
854 callback = pm_noirq_op(dev->class->pm, state);
855 } else if (dev->bus && dev->bus->pm) {
856 info = "noirq bus ";
857 callback = pm_noirq_op(dev->bus->pm, state);
858 }
859 if (callback)
860 goto Run;
861
862 if (skip_resume)
863 goto Skip;
864
865 if (dev->driver && dev->driver->pm) {
866 info = "noirq driver ";
867 callback = pm_noirq_op(dev->driver->pm, state);
868 }
869
870Run:
871 error = dpm_run_callback(callback, dev, state, info);
872
873Skip:
874 dev->power.is_noirq_suspended = false;
875
876Out:
877 complete_all(&dev->power.completion);
878 TRACE_RESUME(error);
879
880 if (error) {
881 WRITE_ONCE(async_error, error);
882 dpm_save_failed_dev(dev_name(dev));
883 pm_dev_err(dev, state, async ? " async noirq" : " noirq", error);
884 }
885
886 dpm_async_resume_subordinate(dev, async_resume_noirq);
887}
888
889static void async_resume_noirq(void *data, async_cookie_t cookie)
890{
891 struct device *dev = data;
892
893 device_resume_noirq(dev, pm_transition, true);
894 put_device(dev);
895}
896
897static void dpm_noirq_resume_devices(pm_message_t state)
898{
899 struct device *dev;
900 ktime_t starttime = ktime_get();
901
902 trace_suspend_resume(TPS("dpm_resume_noirq"), state.event, true);
903
904 async_error = 0;
905 pm_transition = state;
906
907 mutex_lock(&dpm_list_mtx);
908
909 /*
910 * Start processing "async" root devices upfront so they don't wait for
911 * the "sync" devices they don't depend on.
912 */
913 list_for_each_entry(dev, &dpm_noirq_list, power.entry) {
914 dpm_clear_async_state(dev);
915 if (dpm_root_device(dev))
916 dpm_async_with_cleanup(dev, async_resume_noirq);
917 }
918
919 while (!list_empty(&dpm_noirq_list)) {
920 dev = to_device(dpm_noirq_list.next);
921 list_move_tail(&dev->power.entry, &dpm_late_early_list);
922
923 if (!dpm_async_fn(dev, async_resume_noirq)) {
924 get_device(dev);
925
926 mutex_unlock(&dpm_list_mtx);
927
928 device_resume_noirq(dev, state, false);
929
930 put_device(dev);
931
932 mutex_lock(&dpm_list_mtx);
933 }
934 }
935 mutex_unlock(&dpm_list_mtx);
936 async_synchronize_full();
937 dpm_show_time(starttime, state, 0, "noirq");
938 if (READ_ONCE(async_error))
939 dpm_save_failed_step(SUSPEND_RESUME_NOIRQ);
940
941 trace_suspend_resume(TPS("dpm_resume_noirq"), state.event, false);
942}
943
944/**
945 * dpm_resume_noirq - Execute "noirq resume" callbacks for all devices.
946 * @state: PM transition of the system being carried out.
947 *
948 * Invoke the "noirq" resume callbacks for all devices in dpm_noirq_list and
949 * allow device drivers' interrupt handlers to be called.
950 */
951void dpm_resume_noirq(pm_message_t state)
952{
953 dpm_noirq_resume_devices(state);
954
955 resume_device_irqs();
956 device_wakeup_disarm_wake_irqs();
957}
958
959static void async_resume_early(void *data, async_cookie_t cookie);
960
961/**
962 * device_resume_early - Execute an "early resume" callback for given device.
963 * @dev: Device to handle.
964 * @state: PM transition of the system being carried out.
965 * @async: If true, the device is being resumed asynchronously.
966 *
967 * Runtime PM is disabled for @dev while this function is being executed.
968 */
969static void device_resume_early(struct device *dev, pm_message_t state, bool async)
970{
971 pm_callback_t callback = NULL;
972 const char *info = NULL;
973 int error = 0;
974
975 TRACE_DEVICE(dev);
976 TRACE_RESUME(0);
977
978 if (dev->power.direct_complete)
979 goto Out;
980
981 if (!dev->power.is_late_suspended)
982 goto Out;
983
984 if (dev->power.syscore)
985 goto Skip;
986
987 if (!dpm_wait_for_superior(dev, async))
988 goto Out;
989
990 if (dev->pm_domain) {
991 info = "early power domain ";
992 callback = pm_late_early_op(&dev->pm_domain->ops, state);
993 } else if (dev->type && dev->type->pm) {
994 info = "early type ";
995 callback = pm_late_early_op(dev->type->pm, state);
996 } else if (dev->class && dev->class->pm) {
997 info = "early class ";
998 callback = pm_late_early_op(dev->class->pm, state);
999 } else if (dev->bus && dev->bus->pm) {
1000 info = "early bus ";
1001 callback = pm_late_early_op(dev->bus->pm, state);
1002 }
1003 if (callback)
1004 goto Run;
1005
1006 if (dev_pm_skip_resume(dev))
1007 goto Skip;
1008
1009 if (dev->driver && dev->driver->pm) {
1010 info = "early driver ";
1011 callback = pm_late_early_op(dev->driver->pm, state);
1012 }
1013
1014Run:
1015 error = dpm_run_callback(callback, dev, state, info);
1016
1017Skip:
1018 dev->power.is_late_suspended = false;
1019 pm_runtime_enable(dev);
1020
1021Out:
1022 TRACE_RESUME(error);
1023
1024 complete_all(&dev->power.completion);
1025
1026 if (error) {
1027 WRITE_ONCE(async_error, error);
1028 dpm_save_failed_dev(dev_name(dev));
1029 pm_dev_err(dev, state, async ? " async early" : " early", error);
1030 }
1031
1032 dpm_async_resume_subordinate(dev, async_resume_early);
1033}
1034
1035static void async_resume_early(void *data, async_cookie_t cookie)
1036{
1037 struct device *dev = data;
1038
1039 device_resume_early(dev, pm_transition, true);
1040 put_device(dev);
1041}
1042
1043/**
1044 * dpm_resume_early - Execute "early resume" callbacks for all devices.
1045 * @state: PM transition of the system being carried out.
1046 */
1047void dpm_resume_early(pm_message_t state)
1048{
1049 struct device *dev;
1050 ktime_t starttime = ktime_get();
1051
1052 trace_suspend_resume(TPS("dpm_resume_early"), state.event, true);
1053
1054 async_error = 0;
1055 pm_transition = state;
1056
1057 mutex_lock(&dpm_list_mtx);
1058
1059 /*
1060 * Start processing "async" root devices upfront so they don't wait for
1061 * the "sync" devices they don't depend on.
1062 */
1063 list_for_each_entry(dev, &dpm_late_early_list, power.entry) {
1064 dpm_clear_async_state(dev);
1065 if (dpm_root_device(dev))
1066 dpm_async_with_cleanup(dev, async_resume_early);
1067 }
1068
1069 while (!list_empty(&dpm_late_early_list)) {
1070 dev = to_device(dpm_late_early_list.next);
1071 list_move_tail(&dev->power.entry, &dpm_suspended_list);
1072
1073 if (!dpm_async_fn(dev, async_resume_early)) {
1074 get_device(dev);
1075
1076 mutex_unlock(&dpm_list_mtx);
1077
1078 device_resume_early(dev, state, false);
1079
1080 put_device(dev);
1081
1082 mutex_lock(&dpm_list_mtx);
1083 }
1084 }
1085 mutex_unlock(&dpm_list_mtx);
1086 async_synchronize_full();
1087 dpm_show_time(starttime, state, 0, "early");
1088 if (READ_ONCE(async_error))
1089 dpm_save_failed_step(SUSPEND_RESUME_EARLY);
1090
1091 trace_suspend_resume(TPS("dpm_resume_early"), state.event, false);
1092}
1093
1094/**
1095 * dpm_resume_start - Execute "noirq" and "early" device callbacks.
1096 * @state: PM transition of the system being carried out.
1097 */
1098void dpm_resume_start(pm_message_t state)
1099{
1100 dpm_resume_noirq(state);
1101 dpm_resume_early(state);
1102}
1103EXPORT_SYMBOL_GPL(dpm_resume_start);
1104
1105static void async_resume(void *data, async_cookie_t cookie);
1106
1107/**
1108 * device_resume - Execute "resume" callbacks for given device.
1109 * @dev: Device to handle.
1110 * @state: PM transition of the system being carried out.
1111 * @async: If true, the device is being resumed asynchronously.
1112 */
1113static void device_resume(struct device *dev, pm_message_t state, bool async)
1114{
1115 pm_callback_t callback = NULL;
1116 const char *info = NULL;
1117 int error = 0;
1118 DECLARE_DPM_WATCHDOG_ON_STACK(wd);
1119
1120 TRACE_DEVICE(dev);
1121 TRACE_RESUME(0);
1122
1123 if (dev->power.syscore)
1124 goto Complete;
1125
1126 if (!dev->power.is_suspended)
1127 goto Complete;
1128
1129 dev->power.is_suspended = false;
1130
1131 if (dev->power.direct_complete) {
1132 /*
1133 * Allow new children to be added under the device after this
1134 * point if it has no PM callbacks.
1135 */
1136 if (dev->power.no_pm_callbacks)
1137 dev->power.is_prepared = false;
1138
1139 /* Match the pm_runtime_disable() in device_suspend(). */
1140 pm_runtime_enable(dev);
1141 goto Complete;
1142 }
1143
1144 if (!dpm_wait_for_superior(dev, async))
1145 goto Complete;
1146
1147 dpm_watchdog_set(&wd, dev);
1148 device_lock(dev);
1149
1150 /*
1151 * This is a fib. But we'll allow new children to be added below
1152 * a resumed device, even if the device hasn't been completed yet.
1153 */
1154 dev->power.is_prepared = false;
1155
1156 if (dev->pm_domain) {
1157 info = "power domain ";
1158 callback = pm_op(&dev->pm_domain->ops, state);
1159 goto Driver;
1160 }
1161
1162 if (dev->type && dev->type->pm) {
1163 info = "type ";
1164 callback = pm_op(dev->type->pm, state);
1165 goto Driver;
1166 }
1167
1168 if (dev->class && dev->class->pm) {
1169 info = "class ";
1170 callback = pm_op(dev->class->pm, state);
1171 goto Driver;
1172 }
1173
1174 if (dev->bus) {
1175 if (dev->bus->pm) {
1176 info = "bus ";
1177 callback = pm_op(dev->bus->pm, state);
1178 } else if (dev->bus->resume) {
1179 info = "legacy bus ";
1180 callback = dev->bus->resume;
1181 goto End;
1182 }
1183 }
1184
1185 Driver:
1186 if (!callback && dev->driver && dev->driver->pm) {
1187 info = "driver ";
1188 callback = pm_op(dev->driver->pm, state);
1189 }
1190
1191 End:
1192 error = dpm_run_callback(callback, dev, state, info);
1193
1194 device_unlock(dev);
1195 dpm_watchdog_clear(&wd);
1196
1197 Complete:
1198 complete_all(&dev->power.completion);
1199
1200 TRACE_RESUME(error);
1201
1202 if (error) {
1203 WRITE_ONCE(async_error, error);
1204 dpm_save_failed_dev(dev_name(dev));
1205 pm_dev_err(dev, state, async ? " async" : "", error);
1206 }
1207
1208 dpm_async_resume_subordinate(dev, async_resume);
1209}
1210
1211static void async_resume(void *data, async_cookie_t cookie)
1212{
1213 struct device *dev = data;
1214
1215 device_resume(dev, pm_transition, true);
1216 put_device(dev);
1217}
1218
1219/**
1220 * dpm_resume - Execute "resume" callbacks for non-sysdev devices.
1221 * @state: PM transition of the system being carried out.
1222 *
1223 * Execute the appropriate "resume" callback for all devices whose status
1224 * indicates that they are suspended.
1225 */
1226void dpm_resume(pm_message_t state)
1227{
1228 struct device *dev;
1229 ktime_t starttime = ktime_get();
1230
1231 trace_suspend_resume(TPS("dpm_resume"), state.event, true);
1232
1233 pm_transition = state;
1234 async_error = 0;
1235
1236 mutex_lock(&dpm_list_mtx);
1237
1238 /*
1239 * Start processing "async" root devices upfront so they don't wait for
1240 * the "sync" devices they don't depend on.
1241 */
1242 list_for_each_entry(dev, &dpm_suspended_list, power.entry) {
1243 dpm_clear_async_state(dev);
1244 if (dpm_root_device(dev))
1245 dpm_async_with_cleanup(dev, async_resume);
1246 }
1247
1248 while (!list_empty(&dpm_suspended_list)) {
1249 dev = to_device(dpm_suspended_list.next);
1250 list_move_tail(&dev->power.entry, &dpm_prepared_list);
1251
1252 if (!dpm_async_fn(dev, async_resume)) {
1253 get_device(dev);
1254
1255 mutex_unlock(&dpm_list_mtx);
1256
1257 device_resume(dev, state, false);
1258
1259 put_device(dev);
1260
1261 mutex_lock(&dpm_list_mtx);
1262 }
1263 }
1264 mutex_unlock(&dpm_list_mtx);
1265 async_synchronize_full();
1266 dpm_show_time(starttime, state, 0, NULL);
1267 if (READ_ONCE(async_error))
1268 dpm_save_failed_step(SUSPEND_RESUME);
1269
1270 cpufreq_resume();
1271 devfreq_resume();
1272 trace_suspend_resume(TPS("dpm_resume"), state.event, false);
1273}
1274
1275/**
1276 * device_complete - Complete a PM transition for given device.
1277 * @dev: Device to handle.
1278 * @state: PM transition of the system being carried out.
1279 */
1280static void device_complete(struct device *dev, pm_message_t state)
1281{
1282 void (*callback)(struct device *) = NULL;
1283 const char *info = NULL;
1284
1285 if (dev->power.syscore)
1286 goto out;
1287
1288 device_lock(dev);
1289
1290 if (dev->pm_domain) {
1291 info = "completing power domain ";
1292 callback = dev->pm_domain->ops.complete;
1293 } else if (dev->type && dev->type->pm) {
1294 info = "completing type ";
1295 callback = dev->type->pm->complete;
1296 } else if (dev->class && dev->class->pm) {
1297 info = "completing class ";
1298 callback = dev->class->pm->complete;
1299 } else if (dev->bus && dev->bus->pm) {
1300 info = "completing bus ";
1301 callback = dev->bus->pm->complete;
1302 }
1303
1304 if (!callback && dev->driver && dev->driver->pm) {
1305 info = "completing driver ";
1306 callback = dev->driver->pm->complete;
1307 }
1308
1309 if (callback) {
1310 pm_dev_dbg(dev, state, info);
1311 callback(dev);
1312 }
1313
1314 device_unlock(dev);
1315
1316out:
1317 /* If enabling runtime PM for the device is blocked, unblock it. */
1318 pm_runtime_unblock(dev);
1319 pm_runtime_put(dev);
1320}
1321
1322/**
1323 * dpm_complete - Complete a PM transition for all non-sysdev devices.
1324 * @state: PM transition of the system being carried out.
1325 *
1326 * Execute the ->complete() callbacks for all devices whose PM status is not
1327 * DPM_ON (this allows new devices to be registered).
1328 */
1329void dpm_complete(pm_message_t state)
1330{
1331 struct list_head list;
1332
1333 trace_suspend_resume(TPS("dpm_complete"), state.event, true);
1334
1335 INIT_LIST_HEAD(&list);
1336 mutex_lock(&dpm_list_mtx);
1337 while (!list_empty(&dpm_prepared_list)) {
1338 struct device *dev = to_device(dpm_prepared_list.prev);
1339
1340 get_device(dev);
1341 dev->power.is_prepared = false;
1342 list_move(&dev->power.entry, &list);
1343
1344 mutex_unlock(&dpm_list_mtx);
1345
1346 trace_device_pm_callback_start(dev, "", state.event);
1347 device_complete(dev, state);
1348 trace_device_pm_callback_end(dev, 0);
1349
1350 put_device(dev);
1351
1352 mutex_lock(&dpm_list_mtx);
1353 }
1354 list_splice(&list, &dpm_list);
1355 mutex_unlock(&dpm_list_mtx);
1356
1357 /* Start resuming thermal control */
1358 thermal_pm_complete();
1359 /* Allow device probing and trigger re-probing of deferred devices */
1360 device_unblock_probing();
1361 trace_suspend_resume(TPS("dpm_complete"), state.event, false);
1362}
1363
1364/**
1365 * dpm_resume_end - Execute "resume" callbacks and complete system transition.
1366 * @state: PM transition of the system being carried out.
1367 *
1368 * Execute "resume" callbacks for all devices and complete the PM transition of
1369 * the system.
1370 */
1371void dpm_resume_end(pm_message_t state)
1372{
1373 dpm_resume(state);
1374 pm_restore_gfp_mask();
1375 dpm_complete(state);
1376}
1377EXPORT_SYMBOL_GPL(dpm_resume_end);
1378
1379
1380/*------------------------- Suspend routines -------------------------*/
1381
1382static bool dpm_leaf_device(struct device *dev)
1383{
1384 struct device *child;
1385
1386 lockdep_assert_held(&dpm_list_mtx);
1387
1388 child = device_find_any_child(dev);
1389 if (child) {
1390 put_device(child);
1391
1392 return false;
1393 }
1394
1395 /*
1396 * Since this function is required to run under dpm_list_mtx, the
1397 * list_empty() below will only return true if the device's list of
1398 * consumers is actually empty before calling it.
1399 */
1400 return list_empty(&dev->links.consumers);
1401}
1402
1403static bool dpm_async_suspend_parent(struct device *dev, async_func_t func)
1404{
1405 guard(mutex)(&dpm_list_mtx);
1406
1407 /*
1408 * If the device is suspended asynchronously and the parent's callback
1409 * deletes both the device and the parent itself, the parent object may
1410 * be freed while this function is running, so avoid that by checking
1411 * if the device has been deleted already as the parent cannot be
1412 * deleted before it.
1413 */
1414 if (!device_pm_initialized(dev))
1415 return false;
1416
1417 /* Start processing the device's parent if it is "async". */
1418 if (dev->parent)
1419 dpm_async_with_cleanup(dev->parent, func);
1420
1421 return true;
1422}
1423
1424static void dpm_async_suspend_superior(struct device *dev, async_func_t func)
1425{
1426 struct device_link *link;
1427 int idx;
1428
1429 if (!dpm_async_suspend_parent(dev, func))
1430 return;
1431
1432 idx = device_links_read_lock();
1433
1434 /* Start processing the device's "async" suppliers. */
1435 dev_for_each_link_to_supplier(link, dev)
1436 if (READ_ONCE(link->status) != DL_STATE_DORMANT)
1437 dpm_async_with_cleanup(link->supplier, func);
1438
1439 device_links_read_unlock(idx);
1440}
1441
1442static void dpm_async_suspend_complete_all(struct list_head *device_list)
1443{
1444 struct device *dev;
1445
1446 guard(mutex)(&async_wip_mtx);
1447
1448 list_for_each_entry_reverse(dev, device_list, power.entry) {
1449 /*
1450 * In case the device is being waited for and async processing
1451 * has not started for it yet, let the waiters make progress.
1452 */
1453 if (!dev->power.work_in_progress)
1454 complete_all(&dev->power.completion);
1455 }
1456}
1457
1458/**
1459 * resume_event - Return a "resume" message for given "suspend" sleep state.
1460 * @sleep_state: PM message representing a sleep state.
1461 *
1462 * Return a PM message representing the resume event corresponding to given
1463 * sleep state.
1464 */
1465static pm_message_t resume_event(pm_message_t sleep_state)
1466{
1467 switch (sleep_state.event) {
1468 case PM_EVENT_SUSPEND:
1469 return PMSG_RESUME;
1470 case PM_EVENT_FREEZE:
1471 case PM_EVENT_QUIESCE:
1472 return PMSG_RECOVER;
1473 case PM_EVENT_HIBERNATE:
1474 return PMSG_RESTORE;
1475 }
1476 return PMSG_ON;
1477}
1478
1479static void dpm_superior_set_must_resume(struct device *dev)
1480{
1481 struct device_link *link;
1482 int idx;
1483
1484 if (dev->parent)
1485 dev->parent->power.must_resume = true;
1486
1487 idx = device_links_read_lock();
1488
1489 dev_for_each_link_to_supplier(link, dev)
1490 link->supplier->power.must_resume = true;
1491
1492 device_links_read_unlock(idx);
1493}
1494
1495static void async_suspend_noirq(void *data, async_cookie_t cookie);
1496
1497/**
1498 * device_suspend_noirq - Execute a "noirq suspend" callback for given device.
1499 * @dev: Device to handle.
1500 * @state: PM transition of the system being carried out.
1501 * @async: If true, the device is being suspended asynchronously.
1502 *
1503 * The driver of @dev will not receive interrupts while this function is being
1504 * executed.
1505 */
1506static void device_suspend_noirq(struct device *dev, pm_message_t state, bool async)
1507{
1508 pm_callback_t callback = NULL;
1509 const char *info = NULL;
1510 int error = 0;
1511
1512 TRACE_DEVICE(dev);
1513 TRACE_SUSPEND(0);
1514
1515 dpm_wait_for_subordinate(dev, async);
1516
1517 if (READ_ONCE(async_error))
1518 goto Complete;
1519
1520 if (dev->power.syscore || dev->power.direct_complete)
1521 goto Complete;
1522
1523 if (dev->pm_domain) {
1524 info = "noirq power domain ";
1525 callback = pm_noirq_op(&dev->pm_domain->ops, state);
1526 } else if (dev->type && dev->type->pm) {
1527 info = "noirq type ";
1528 callback = pm_noirq_op(dev->type->pm, state);
1529 } else if (dev->class && dev->class->pm) {
1530 info = "noirq class ";
1531 callback = pm_noirq_op(dev->class->pm, state);
1532 } else if (dev->bus && dev->bus->pm) {
1533 info = "noirq bus ";
1534 callback = pm_noirq_op(dev->bus->pm, state);
1535 }
1536 if (callback)
1537 goto Run;
1538
1539 if (dev_pm_skip_suspend(dev))
1540 goto Skip;
1541
1542 if (dev->driver && dev->driver->pm) {
1543 info = "noirq driver ";
1544 callback = pm_noirq_op(dev->driver->pm, state);
1545 }
1546
1547Run:
1548 error = dpm_run_callback(callback, dev, state, info);
1549 if (error) {
1550 WRITE_ONCE(async_error, error);
1551 dpm_save_failed_dev(dev_name(dev));
1552 pm_dev_err(dev, state, async ? " async noirq" : " noirq", error);
1553 goto Complete;
1554 }
1555
1556Skip:
1557 dev->power.is_noirq_suspended = true;
1558
1559 /*
1560 * Devices must be resumed unless they are explicitly allowed to be left
1561 * in suspend, but even in that case skipping the resume of devices that
1562 * were in use right before the system suspend (as indicated by their
1563 * runtime PM usage counters and child counters) would be suboptimal.
1564 */
1565 if (!(dev_pm_test_driver_flags(dev, DPM_FLAG_MAY_SKIP_RESUME) &&
1566 dev->power.may_skip_resume) || !pm_runtime_need_not_resume(dev))
1567 dev->power.must_resume = true;
1568
1569 if (dev->power.must_resume)
1570 dpm_superior_set_must_resume(dev);
1571
1572Complete:
1573 complete_all(&dev->power.completion);
1574 TRACE_SUSPEND(error);
1575
1576 if (error || READ_ONCE(async_error))
1577 return;
1578
1579 dpm_async_suspend_superior(dev, async_suspend_noirq);
1580}
1581
1582static void async_suspend_noirq(void *data, async_cookie_t cookie)
1583{
1584 struct device *dev = data;
1585
1586 device_suspend_noirq(dev, pm_transition, true);
1587 put_device(dev);
1588}
1589
1590static int dpm_noirq_suspend_devices(pm_message_t state)
1591{
1592 ktime_t starttime = ktime_get();
1593 struct device *dev;
1594 int error;
1595
1596 trace_suspend_resume(TPS("dpm_suspend_noirq"), state.event, true);
1597
1598 pm_transition = state;
1599 async_error = 0;
1600
1601 mutex_lock(&dpm_list_mtx);
1602
1603 /*
1604 * Start processing "async" leaf devices upfront so they don't need to
1605 * wait for the "sync" devices they don't depend on.
1606 */
1607 list_for_each_entry_reverse(dev, &dpm_late_early_list, power.entry) {
1608 dpm_clear_async_state(dev);
1609 if (dpm_leaf_device(dev))
1610 dpm_async_with_cleanup(dev, async_suspend_noirq);
1611 }
1612
1613 while (!list_empty(&dpm_late_early_list)) {
1614 dev = to_device(dpm_late_early_list.prev);
1615
1616 list_move(&dev->power.entry, &dpm_noirq_list);
1617
1618 if (dpm_async_fn(dev, async_suspend_noirq))
1619 continue;
1620
1621 get_device(dev);
1622
1623 mutex_unlock(&dpm_list_mtx);
1624
1625 device_suspend_noirq(dev, state, false);
1626
1627 put_device(dev);
1628
1629 mutex_lock(&dpm_list_mtx);
1630
1631 if (READ_ONCE(async_error)) {
1632 dpm_async_suspend_complete_all(&dpm_late_early_list);
1633 /*
1634 * Move all devices to the target list to resume them
1635 * properly.
1636 */
1637 list_splice_init(&dpm_late_early_list, &dpm_noirq_list);
1638 break;
1639 }
1640 }
1641
1642 mutex_unlock(&dpm_list_mtx);
1643
1644 async_synchronize_full();
1645
1646 error = READ_ONCE(async_error);
1647 if (error)
1648 dpm_save_failed_step(SUSPEND_SUSPEND_NOIRQ);
1649
1650 dpm_show_time(starttime, state, error, "noirq");
1651 trace_suspend_resume(TPS("dpm_suspend_noirq"), state.event, false);
1652 return error;
1653}
1654
1655/**
1656 * dpm_suspend_noirq - Execute "noirq suspend" callbacks for all devices.
1657 * @state: PM transition of the system being carried out.
1658 *
1659 * Prevent device drivers' interrupt handlers from being called and invoke
1660 * "noirq" suspend callbacks for all non-sysdev devices.
1661 */
1662int dpm_suspend_noirq(pm_message_t state)
1663{
1664 int ret;
1665
1666 device_wakeup_arm_wake_irqs();
1667 suspend_device_irqs();
1668
1669 ret = dpm_noirq_suspend_devices(state);
1670 if (ret)
1671 dpm_resume_noirq(resume_event(state));
1672
1673 return ret;
1674}
1675
1676static void dpm_propagate_wakeup_to_parent(struct device *dev)
1677{
1678 struct device *parent = dev->parent;
1679
1680 if (!parent)
1681 return;
1682
1683 spin_lock_irq(&parent->power.lock);
1684
1685 if (device_wakeup_path(dev) && !parent->power.ignore_children)
1686 parent->power.wakeup_path = true;
1687
1688 spin_unlock_irq(&parent->power.lock);
1689}
1690
1691static void async_suspend_late(void *data, async_cookie_t cookie);
1692
1693/**
1694 * device_suspend_late - Execute a "late suspend" callback for given device.
1695 * @dev: Device to handle.
1696 * @state: PM transition of the system being carried out.
1697 * @async: If true, the device is being suspended asynchronously.
1698 *
1699 * Runtime PM is disabled for @dev while this function is being executed.
1700 */
1701static void device_suspend_late(struct device *dev, pm_message_t state, bool async)
1702{
1703 pm_callback_t callback = NULL;
1704 const char *info = NULL;
1705 int error = 0;
1706
1707 TRACE_DEVICE(dev);
1708 TRACE_SUSPEND(0);
1709
1710 dpm_wait_for_subordinate(dev, async);
1711
1712 if (READ_ONCE(async_error))
1713 goto Complete;
1714
1715 if (pm_wakeup_pending()) {
1716 WRITE_ONCE(async_error, -EBUSY);
1717 goto Complete;
1718 }
1719
1720 if (dev->power.direct_complete)
1721 goto Complete;
1722
1723 /*
1724 * After this point, any runtime PM operations targeting the device
1725 * will fail until the corresponding pm_runtime_enable() call in
1726 * device_resume_early().
1727 */
1728 pm_runtime_disable(dev);
1729
1730 if (dev->power.syscore)
1731 goto Skip;
1732
1733 if (dev->pm_domain) {
1734 info = "late power domain ";
1735 callback = pm_late_early_op(&dev->pm_domain->ops, state);
1736 } else if (dev->type && dev->type->pm) {
1737 info = "late type ";
1738 callback = pm_late_early_op(dev->type->pm, state);
1739 } else if (dev->class && dev->class->pm) {
1740 info = "late class ";
1741 callback = pm_late_early_op(dev->class->pm, state);
1742 } else if (dev->bus && dev->bus->pm) {
1743 info = "late bus ";
1744 callback = pm_late_early_op(dev->bus->pm, state);
1745 }
1746 if (callback)
1747 goto Run;
1748
1749 if (dev_pm_skip_suspend(dev))
1750 goto Skip;
1751
1752 if (dev->driver && dev->driver->pm) {
1753 info = "late driver ";
1754 callback = pm_late_early_op(dev->driver->pm, state);
1755 }
1756
1757Run:
1758 error = dpm_run_callback(callback, dev, state, info);
1759 if (error) {
1760 WRITE_ONCE(async_error, error);
1761 dpm_save_failed_dev(dev_name(dev));
1762 pm_dev_err(dev, state, async ? " async late" : " late", error);
1763 pm_runtime_enable(dev);
1764 goto Complete;
1765 }
1766 dpm_propagate_wakeup_to_parent(dev);
1767
1768Skip:
1769 dev->power.is_late_suspended = true;
1770
1771Complete:
1772 TRACE_SUSPEND(error);
1773 complete_all(&dev->power.completion);
1774
1775 if (error || READ_ONCE(async_error))
1776 return;
1777
1778 dpm_async_suspend_superior(dev, async_suspend_late);
1779}
1780
1781static void async_suspend_late(void *data, async_cookie_t cookie)
1782{
1783 struct device *dev = data;
1784
1785 device_suspend_late(dev, pm_transition, true);
1786 put_device(dev);
1787}
1788
1789/**
1790 * dpm_suspend_late - Execute "late suspend" callbacks for all devices.
1791 * @state: PM transition of the system being carried out.
1792 */
1793int dpm_suspend_late(pm_message_t state)
1794{
1795 ktime_t starttime = ktime_get();
1796 struct device *dev;
1797 int error;
1798
1799 trace_suspend_resume(TPS("dpm_suspend_late"), state.event, true);
1800
1801 pm_transition = state;
1802 async_error = 0;
1803
1804 wake_up_all_idle_cpus();
1805
1806 mutex_lock(&dpm_list_mtx);
1807
1808 /*
1809 * Start processing "async" leaf devices upfront so they don't need to
1810 * wait for the "sync" devices they don't depend on.
1811 */
1812 list_for_each_entry_reverse(dev, &dpm_suspended_list, power.entry) {
1813 dpm_clear_async_state(dev);
1814 if (dpm_leaf_device(dev))
1815 dpm_async_with_cleanup(dev, async_suspend_late);
1816 }
1817
1818 while (!list_empty(&dpm_suspended_list)) {
1819 dev = to_device(dpm_suspended_list.prev);
1820
1821 list_move(&dev->power.entry, &dpm_late_early_list);
1822
1823 if (dpm_async_fn(dev, async_suspend_late))
1824 continue;
1825
1826 get_device(dev);
1827
1828 mutex_unlock(&dpm_list_mtx);
1829
1830 device_suspend_late(dev, state, false);
1831
1832 put_device(dev);
1833
1834 mutex_lock(&dpm_list_mtx);
1835
1836 if (READ_ONCE(async_error)) {
1837 dpm_async_suspend_complete_all(&dpm_suspended_list);
1838 /*
1839 * Move all devices to the target list to resume them
1840 * properly.
1841 */
1842 list_splice_init(&dpm_suspended_list, &dpm_late_early_list);
1843 break;
1844 }
1845 }
1846
1847 mutex_unlock(&dpm_list_mtx);
1848
1849 async_synchronize_full();
1850
1851 error = READ_ONCE(async_error);
1852 if (error) {
1853 dpm_save_failed_step(SUSPEND_SUSPEND_LATE);
1854 dpm_resume_early(resume_event(state));
1855 }
1856 dpm_show_time(starttime, state, error, "late");
1857 trace_suspend_resume(TPS("dpm_suspend_late"), state.event, false);
1858 return error;
1859}
1860
1861/**
1862 * dpm_suspend_end - Execute "late" and "noirq" device suspend callbacks.
1863 * @state: PM transition of the system being carried out.
1864 */
1865int dpm_suspend_end(pm_message_t state)
1866{
1867 ktime_t starttime = ktime_get();
1868 int error;
1869
1870 error = dpm_suspend_late(state);
1871 if (error)
1872 goto out;
1873
1874 error = dpm_suspend_noirq(state);
1875 if (error)
1876 dpm_resume_early(resume_event(state));
1877
1878out:
1879 dpm_show_time(starttime, state, error, "end");
1880 return error;
1881}
1882EXPORT_SYMBOL_GPL(dpm_suspend_end);
1883
1884/**
1885 * legacy_suspend - Execute a legacy (bus or class) suspend callback for device.
1886 * @dev: Device to suspend.
1887 * @state: PM transition of the system being carried out.
1888 * @cb: Suspend callback to execute.
1889 * @info: string description of caller.
1890 */
1891static int legacy_suspend(struct device *dev, pm_message_t state,
1892 int (*cb)(struct device *dev, pm_message_t state),
1893 const char *info)
1894{
1895 int error;
1896 ktime_t calltime;
1897
1898 calltime = initcall_debug_start(dev, cb);
1899
1900 trace_device_pm_callback_start(dev, info, state.event);
1901 error = cb(dev, state);
1902 trace_device_pm_callback_end(dev, error);
1903 suspend_report_result(dev, cb, error);
1904
1905 initcall_debug_report(dev, calltime, cb, error);
1906
1907 return error;
1908}
1909
1910static void dpm_clear_superiors_direct_complete(struct device *dev)
1911{
1912 struct device_link *link;
1913 int idx;
1914
1915 if (dev->parent) {
1916 spin_lock_irq(&dev->parent->power.lock);
1917 dev->parent->power.direct_complete = false;
1918 spin_unlock_irq(&dev->parent->power.lock);
1919 }
1920
1921 idx = device_links_read_lock();
1922
1923 dev_for_each_link_to_supplier(link, dev) {
1924 spin_lock_irq(&link->supplier->power.lock);
1925 link->supplier->power.direct_complete = false;
1926 spin_unlock_irq(&link->supplier->power.lock);
1927 }
1928
1929 device_links_read_unlock(idx);
1930}
1931
1932static void async_suspend(void *data, async_cookie_t cookie);
1933
1934/**
1935 * device_suspend - Execute "suspend" callbacks for given device.
1936 * @dev: Device to handle.
1937 * @state: PM transition of the system being carried out.
1938 * @async: If true, the device is being suspended asynchronously.
1939 */
1940static void device_suspend(struct device *dev, pm_message_t state, bool async)
1941{
1942 pm_callback_t callback = NULL;
1943 const char *info = NULL;
1944 int error = 0;
1945 DECLARE_DPM_WATCHDOG_ON_STACK(wd);
1946
1947 TRACE_DEVICE(dev);
1948 TRACE_SUSPEND(0);
1949
1950 dpm_wait_for_subordinate(dev, async);
1951
1952 if (READ_ONCE(async_error)) {
1953 dev->power.direct_complete = false;
1954 goto Complete;
1955 }
1956
1957 /*
1958 * Wait for possible runtime PM transitions of the device in progress
1959 * to complete and if there's a runtime resume request pending for it,
1960 * resume it before proceeding with invoking the system-wide suspend
1961 * callbacks for it.
1962 *
1963 * If the system-wide suspend callbacks below change the configuration
1964 * of the device, they must disable runtime PM for it or otherwise
1965 * ensure that its runtime-resume callbacks will not be confused by that
1966 * change in case they are invoked going forward.
1967 */
1968 pm_runtime_barrier(dev);
1969
1970 if (pm_wakeup_pending()) {
1971 dev->power.direct_complete = false;
1972 WRITE_ONCE(async_error, -EBUSY);
1973 goto Complete;
1974 }
1975
1976 if (dev->power.syscore)
1977 goto Complete;
1978
1979 /* Avoid direct_complete to let wakeup_path propagate. */
1980 if (device_may_wakeup(dev) || device_wakeup_path(dev))
1981 dev->power.direct_complete = false;
1982
1983 if (dev->power.direct_complete) {
1984 if (pm_runtime_status_suspended(dev)) {
1985 pm_runtime_disable(dev);
1986 if (pm_runtime_status_suspended(dev)) {
1987 pm_dev_dbg(dev, state, "direct-complete ");
1988 dev->power.is_suspended = true;
1989 goto Complete;
1990 }
1991
1992 pm_runtime_enable(dev);
1993 }
1994 dev->power.direct_complete = false;
1995 }
1996
1997 dev->power.may_skip_resume = true;
1998 dev->power.must_resume = !dev_pm_test_driver_flags(dev, DPM_FLAG_MAY_SKIP_RESUME);
1999
2000 dpm_watchdog_set(&wd, dev);
2001 device_lock(dev);
2002
2003 if (dev->pm_domain) {
2004 info = "power domain ";
2005 callback = pm_op(&dev->pm_domain->ops, state);
2006 goto Run;
2007 }
2008
2009 if (dev->type && dev->type->pm) {
2010 info = "type ";
2011 callback = pm_op(dev->type->pm, state);
2012 goto Run;
2013 }
2014
2015 if (dev->class && dev->class->pm) {
2016 info = "class ";
2017 callback = pm_op(dev->class->pm, state);
2018 goto Run;
2019 }
2020
2021 if (dev->bus) {
2022 if (dev->bus->pm) {
2023 info = "bus ";
2024 callback = pm_op(dev->bus->pm, state);
2025 } else if (dev->bus->suspend) {
2026 pm_dev_dbg(dev, state, "legacy bus ");
2027 error = legacy_suspend(dev, state, dev->bus->suspend,
2028 "legacy bus ");
2029 goto End;
2030 }
2031 }
2032
2033 Run:
2034 if (!callback && dev->driver && dev->driver->pm) {
2035 info = "driver ";
2036 callback = pm_op(dev->driver->pm, state);
2037 }
2038
2039 error = dpm_run_callback(callback, dev, state, info);
2040
2041 End:
2042 if (!error) {
2043 dev->power.is_suspended = true;
2044 if (device_may_wakeup(dev))
2045 dev->power.wakeup_path = true;
2046
2047 dpm_propagate_wakeup_to_parent(dev);
2048 dpm_clear_superiors_direct_complete(dev);
2049 }
2050
2051 device_unlock(dev);
2052 dpm_watchdog_clear(&wd);
2053
2054 Complete:
2055 if (error) {
2056 WRITE_ONCE(async_error, error);
2057 dpm_save_failed_dev(dev_name(dev));
2058 pm_dev_err(dev, state, async ? " async" : "", error);
2059 }
2060
2061 complete_all(&dev->power.completion);
2062 TRACE_SUSPEND(error);
2063
2064 if (error || READ_ONCE(async_error))
2065 return;
2066
2067 dpm_async_suspend_superior(dev, async_suspend);
2068}
2069
2070static void async_suspend(void *data, async_cookie_t cookie)
2071{
2072 struct device *dev = data;
2073
2074 device_suspend(dev, pm_transition, true);
2075 put_device(dev);
2076}
2077
2078/**
2079 * dpm_suspend - Execute "suspend" callbacks for all non-sysdev devices.
2080 * @state: PM transition of the system being carried out.
2081 */
2082int dpm_suspend(pm_message_t state)
2083{
2084 ktime_t starttime = ktime_get();
2085 struct device *dev;
2086 int error;
2087
2088 trace_suspend_resume(TPS("dpm_suspend"), state.event, true);
2089 might_sleep();
2090
2091 devfreq_suspend();
2092 cpufreq_suspend();
2093
2094 pm_transition = state;
2095 async_error = 0;
2096
2097 mutex_lock(&dpm_list_mtx);
2098
2099 /*
2100 * Start processing "async" leaf devices upfront so they don't need to
2101 * wait for the "sync" devices they don't depend on.
2102 */
2103 list_for_each_entry_reverse(dev, &dpm_prepared_list, power.entry) {
2104 dpm_clear_async_state(dev);
2105 if (dpm_leaf_device(dev))
2106 dpm_async_with_cleanup(dev, async_suspend);
2107 }
2108
2109 while (!list_empty(&dpm_prepared_list)) {
2110 dev = to_device(dpm_prepared_list.prev);
2111
2112 list_move(&dev->power.entry, &dpm_suspended_list);
2113
2114 if (dpm_async_fn(dev, async_suspend))
2115 continue;
2116
2117 get_device(dev);
2118
2119 mutex_unlock(&dpm_list_mtx);
2120
2121 device_suspend(dev, state, false);
2122
2123 put_device(dev);
2124
2125 mutex_lock(&dpm_list_mtx);
2126
2127 if (READ_ONCE(async_error)) {
2128 dpm_async_suspend_complete_all(&dpm_prepared_list);
2129 /*
2130 * Move all devices to the target list to resume them
2131 * properly.
2132 */
2133 list_splice_init(&dpm_prepared_list, &dpm_suspended_list);
2134 break;
2135 }
2136 }
2137
2138 mutex_unlock(&dpm_list_mtx);
2139
2140 async_synchronize_full();
2141
2142 error = READ_ONCE(async_error);
2143 if (error)
2144 dpm_save_failed_step(SUSPEND_SUSPEND);
2145
2146 dpm_show_time(starttime, state, error, NULL);
2147 trace_suspend_resume(TPS("dpm_suspend"), state.event, false);
2148 return error;
2149}
2150
2151static bool device_prepare_smart_suspend(struct device *dev)
2152{
2153 struct device_link *link;
2154 bool ret = true;
2155 int idx;
2156
2157 /*
2158 * The "smart suspend" feature is enabled for devices whose drivers ask
2159 * for it and for devices without PM callbacks.
2160 *
2161 * However, if "smart suspend" is not enabled for the device's parent
2162 * or any of its suppliers that take runtime PM into account, it cannot
2163 * be enabled for the device either.
2164 */
2165 if (!dev->power.no_pm_callbacks &&
2166 !dev_pm_test_driver_flags(dev, DPM_FLAG_SMART_SUSPEND))
2167 return false;
2168
2169 if (dev->parent && !dev_pm_smart_suspend(dev->parent) &&
2170 !dev->parent->power.ignore_children && !pm_runtime_blocked(dev->parent))
2171 return false;
2172
2173 idx = device_links_read_lock();
2174
2175 dev_for_each_link_to_supplier(link, dev) {
2176 if (!device_link_test(link, DL_FLAG_PM_RUNTIME))
2177 continue;
2178
2179 if (!dev_pm_smart_suspend(link->supplier) &&
2180 !pm_runtime_blocked(link->supplier)) {
2181 ret = false;
2182 break;
2183 }
2184 }
2185
2186 device_links_read_unlock(idx);
2187
2188 return ret;
2189}
2190
2191/**
2192 * device_prepare - Prepare a device for system power transition.
2193 * @dev: Device to handle.
2194 * @state: PM transition of the system being carried out.
2195 *
2196 * Execute the ->prepare() callback(s) for given device. No new children of the
2197 * device may be registered after this function has returned.
2198 */
2199static int device_prepare(struct device *dev, pm_message_t state)
2200{
2201 int (*callback)(struct device *) = NULL;
2202 bool smart_suspend;
2203 int ret = 0;
2204
2205 /*
2206 * If a device's parent goes into runtime suspend at the wrong time,
2207 * it won't be possible to resume the device. To prevent this we
2208 * block runtime suspend here, during the prepare phase, and allow
2209 * it again during the complete phase.
2210 */
2211 pm_runtime_get_noresume(dev);
2212 /*
2213 * If runtime PM is disabled for the device at this point and it has
2214 * never been enabled so far, it should not be enabled until this system
2215 * suspend-resume cycle is complete, so prepare to trigger a warning on
2216 * subsequent attempts to enable it.
2217 */
2218 smart_suspend = !pm_runtime_block_if_disabled(dev);
2219
2220 if (dev->power.syscore)
2221 return 0;
2222
2223 device_lock(dev);
2224
2225 dev->power.wakeup_path = false;
2226 dev->power.out_band_wakeup = false;
2227
2228 if (dev->power.no_pm_callbacks)
2229 goto unlock;
2230
2231 if (dev->pm_domain)
2232 callback = dev->pm_domain->ops.prepare;
2233 else if (dev->type && dev->type->pm)
2234 callback = dev->type->pm->prepare;
2235 else if (dev->class && dev->class->pm)
2236 callback = dev->class->pm->prepare;
2237 else if (dev->bus && dev->bus->pm)
2238 callback = dev->bus->pm->prepare;
2239
2240 if (!callback && dev->driver && dev->driver->pm)
2241 callback = dev->driver->pm->prepare;
2242
2243 if (callback)
2244 ret = callback(dev);
2245
2246unlock:
2247 device_unlock(dev);
2248
2249 if (ret < 0) {
2250 suspend_report_result(dev, callback, ret);
2251 pm_runtime_unblock(dev);
2252 pm_runtime_put(dev);
2253 return ret;
2254 }
2255 /* Do not enable "smart suspend" for devices with disabled runtime PM. */
2256 if (smart_suspend)
2257 smart_suspend = device_prepare_smart_suspend(dev);
2258
2259 spin_lock_irq(&dev->power.lock);
2260
2261 dev->power.smart_suspend = smart_suspend;
2262 /*
2263 * A positive return value from ->prepare() means "this device appears
2264 * to be runtime-suspended and its state is fine, so if it really is
2265 * runtime-suspended, you can leave it in that state provided that you
2266 * will do the same thing with all of its descendants". This only
2267 * applies to suspend transitions, however.
2268 */
2269 dev->power.direct_complete = state.event == PM_EVENT_SUSPEND &&
2270 (ret > 0 || dev->power.no_pm_callbacks) &&
2271 !dev_pm_test_driver_flags(dev, DPM_FLAG_NO_DIRECT_COMPLETE);
2272
2273 spin_unlock_irq(&dev->power.lock);
2274
2275 return 0;
2276}
2277
2278/**
2279 * dpm_prepare - Prepare all non-sysdev devices for a system PM transition.
2280 * @state: PM transition of the system being carried out.
2281 *
2282 * Execute the ->prepare() callback(s) for all devices.
2283 */
2284int dpm_prepare(pm_message_t state)
2285{
2286 int error = 0;
2287
2288 trace_suspend_resume(TPS("dpm_prepare"), state.event, true);
2289
2290 /*
2291 * Give a chance for the known devices to complete their probes, before
2292 * disable probing of devices. This sync point is important at least
2293 * at boot time + hibernation restore.
2294 */
2295 wait_for_device_probe();
2296 /*
2297 * It is unsafe if probing of devices will happen during suspend or
2298 * hibernation and system behavior will be unpredictable in this case.
2299 * So, let's prohibit device's probing here and defer their probes
2300 * instead. The normal behavior will be restored in dpm_complete().
2301 */
2302 device_block_probing();
2303 /* Suspend thermal control. */
2304 thermal_pm_prepare();
2305
2306 mutex_lock(&dpm_list_mtx);
2307 while (!list_empty(&dpm_list) && !error) {
2308 struct device *dev = to_device(dpm_list.next);
2309
2310 get_device(dev);
2311
2312 mutex_unlock(&dpm_list_mtx);
2313
2314 trace_device_pm_callback_start(dev, "", state.event);
2315 error = device_prepare(dev, state);
2316 trace_device_pm_callback_end(dev, error);
2317
2318 mutex_lock(&dpm_list_mtx);
2319
2320 if (!error) {
2321 dev->power.is_prepared = true;
2322 if (!list_empty(&dev->power.entry))
2323 list_move_tail(&dev->power.entry, &dpm_prepared_list);
2324 } else if (error == -EAGAIN) {
2325 error = 0;
2326 } else {
2327 dev_info(dev, "not prepared for power transition: code %d\n",
2328 error);
2329 }
2330
2331 mutex_unlock(&dpm_list_mtx);
2332
2333 put_device(dev);
2334
2335 mutex_lock(&dpm_list_mtx);
2336 }
2337 mutex_unlock(&dpm_list_mtx);
2338 trace_suspend_resume(TPS("dpm_prepare"), state.event, false);
2339 return error;
2340}
2341
2342/**
2343 * dpm_suspend_start - Prepare devices for PM transition and suspend them.
2344 * @state: PM transition of the system being carried out.
2345 *
2346 * Prepare all non-sysdev devices for system PM transition and execute "suspend"
2347 * callbacks for them.
2348 */
2349int dpm_suspend_start(pm_message_t state)
2350{
2351 ktime_t starttime = ktime_get();
2352 int error;
2353
2354 error = dpm_prepare(state);
2355 if (error)
2356 dpm_save_failed_step(SUSPEND_PREPARE);
2357 else {
2358 pm_restrict_gfp_mask();
2359 error = dpm_suspend(state);
2360 }
2361
2362 dpm_show_time(starttime, state, error, "start");
2363 return error;
2364}
2365EXPORT_SYMBOL_GPL(dpm_suspend_start);
2366
2367void __suspend_report_result(const char *function, struct device *dev, void *fn, int ret)
2368{
2369 if (ret)
2370 dev_err(dev, "%s(): %ps returns %d\n", function, fn, ret);
2371}
2372EXPORT_SYMBOL_GPL(__suspend_report_result);
2373
2374/**
2375 * device_pm_wait_for_dev - Wait for suspend/resume of a device to complete.
2376 * @subordinate: Device that needs to wait for @dev.
2377 * @dev: Device to wait for.
2378 */
2379int device_pm_wait_for_dev(struct device *subordinate, struct device *dev)
2380{
2381 dpm_wait(dev, subordinate->power.async_suspend);
2382 return async_error;
2383}
2384EXPORT_SYMBOL_GPL(device_pm_wait_for_dev);
2385
2386/**
2387 * dpm_for_each_dev - device iterator.
2388 * @data: data for the callback.
2389 * @fn: function to be called for each device.
2390 *
2391 * Iterate over devices in dpm_list, and call @fn for each device,
2392 * passing it @data.
2393 */
2394void dpm_for_each_dev(void *data, void (*fn)(struct device *, void *))
2395{
2396 struct device *dev;
2397
2398 if (!fn)
2399 return;
2400
2401 device_pm_lock();
2402 list_for_each_entry(dev, &dpm_list, power.entry)
2403 fn(dev, data);
2404 device_pm_unlock();
2405}
2406EXPORT_SYMBOL_GPL(dpm_for_each_dev);
2407
2408static bool pm_ops_is_empty(const struct dev_pm_ops *ops)
2409{
2410 if (!ops)
2411 return true;
2412
2413 return !ops->prepare &&
2414 !ops->suspend &&
2415 !ops->suspend_late &&
2416 !ops->suspend_noirq &&
2417 !ops->resume_noirq &&
2418 !ops->resume_early &&
2419 !ops->resume &&
2420 !ops->complete;
2421}
2422
2423void device_pm_check_callbacks(struct device *dev)
2424{
2425 unsigned long flags;
2426
2427 spin_lock_irqsave(&dev->power.lock, flags);
2428 dev->power.no_pm_callbacks =
2429 (!dev->bus || (pm_ops_is_empty(dev->bus->pm) &&
2430 !dev->bus->suspend && !dev->bus->resume)) &&
2431 (!dev->class || pm_ops_is_empty(dev->class->pm)) &&
2432 (!dev->type || pm_ops_is_empty(dev->type->pm)) &&
2433 (!dev->pm_domain || pm_ops_is_empty(&dev->pm_domain->ops)) &&
2434 (!dev->driver || (pm_ops_is_empty(dev->driver->pm) &&
2435 !dev->driver->suspend && !dev->driver->resume));
2436 spin_unlock_irqrestore(&dev->power.lock, flags);
2437}
2438
2439bool dev_pm_skip_suspend(struct device *dev)
2440{
2441 return dev_pm_smart_suspend(dev) && pm_runtime_status_suspended(dev);
2442}