projects/linux·blob v2.6.28-rc3 · refs

1918 lines · 46.9 KiB

1/*
2 * linux/kernel/hrtimer.c
3 *
4 * Copyright(C) 2005-2006, Thomas Gleixner <[email protected]>
5 * Copyright(C) 2005-2007, Red Hat, Inc., Ingo Molnar
6 * Copyright(C) 2006-2007 Timesys Corp., Thomas Gleixner
7 *
8 * High-resolution kernel timers
9 *
10 * In contrast to the low-resolution timeout API implemented in
11 * kernel/timer.c, hrtimers provide finer resolution and accuracy
12 * depending on system configuration and capabilities.
13 *
14 * These timers are currently used for:
15 * - itimers
16 * - POSIX timers
17 * - nanosleep
18 * - precise in-kernel timing
19 *
20 * Started by: Thomas Gleixner and Ingo Molnar
21 *
22 * Credits:
23 * based on kernel/timer.c
24 *
25 * Help, testing, suggestions, bugfixes, improvements were
26 * provided by:
27 *
28 * George Anzinger, Andrew Morton, Steven Rostedt, Roman Zippel
29 * et. al.
30 *
31 * For licencing details see kernel-base/COPYING
32 */
33
34#include <linux/cpu.h>
35#include <linux/irq.h>
36#include <linux/module.h>
37#include <linux/percpu.h>
38#include <linux/hrtimer.h>
39#include <linux/notifier.h>
40#include <linux/syscalls.h>
41#include <linux/kallsyms.h>
42#include <linux/interrupt.h>
43#include <linux/tick.h>
44#include <linux/seq_file.h>
45#include <linux/err.h>
46#include <linux/debugobjects.h>
47
48#include <asm/uaccess.h>
49
50/**
51 * ktime_get - get the monotonic time in ktime_t format
52 *
53 * returns the time in ktime_t format
54 */
55ktime_t ktime_get(void)
56{
57 struct timespec now;
58
59 ktime_get_ts(&now);
60
61 return timespec_to_ktime(now);
62}
63EXPORT_SYMBOL_GPL(ktime_get);
64
65/**
66 * ktime_get_real - get the real (wall-) time in ktime_t format
67 *
68 * returns the time in ktime_t format
69 */
70ktime_t ktime_get_real(void)
71{
72 struct timespec now;
73
74 getnstimeofday(&now);
75
76 return timespec_to_ktime(now);
77}
78
79EXPORT_SYMBOL_GPL(ktime_get_real);
80
81/*
82 * The timer bases:
83 *
84 * Note: If we want to add new timer bases, we have to skip the two
85 * clock ids captured by the cpu-timers. We do this by holding empty
86 * entries rather than doing math adjustment of the clock ids.
87 * This ensures that we capture erroneous accesses to these clock ids
88 * rather than moving them into the range of valid clock id's.
89 */
90DEFINE_PER_CPU(struct hrtimer_cpu_base, hrtimer_bases) =
91{
92
93 .clock_base =
94 {
95 {
96 .index = CLOCK_REALTIME,
97 .get_time = &ktime_get_real,
98 .resolution = KTIME_LOW_RES,
99 },
100 {
101 .index = CLOCK_MONOTONIC,
102 .get_time = &ktime_get,
103 .resolution = KTIME_LOW_RES,
104 },
105 }
106};
107
108/**
109 * ktime_get_ts - get the monotonic clock in timespec format
110 * @ts: pointer to timespec variable
111 *
112 * The function calculates the monotonic clock from the realtime
113 * clock and the wall_to_monotonic offset and stores the result
114 * in normalized timespec format in the variable pointed to by @ts.
115 */
116void ktime_get_ts(struct timespec *ts)
117{
118 struct timespec tomono;
119 unsigned long seq;
120
121 do {
122 seq = read_seqbegin(&xtime_lock);
123 getnstimeofday(ts);
124 tomono = wall_to_monotonic;
125
126 } while (read_seqretry(&xtime_lock, seq));
127
128 set_normalized_timespec(ts, ts->tv_sec + tomono.tv_sec,
129 ts->tv_nsec + tomono.tv_nsec);
130}
131EXPORT_SYMBOL_GPL(ktime_get_ts);
132
133/*
134 * Get the coarse grained time at the softirq based on xtime and
135 * wall_to_monotonic.
136 */
137static void hrtimer_get_softirq_time(struct hrtimer_cpu_base *base)
138{
139 ktime_t xtim, tomono;
140 struct timespec xts, tom;
141 unsigned long seq;
142
143 do {
144 seq = read_seqbegin(&xtime_lock);
145 xts = current_kernel_time();
146 tom = wall_to_monotonic;
147 } while (read_seqretry(&xtime_lock, seq));
148
149 xtim = timespec_to_ktime(xts);
150 tomono = timespec_to_ktime(tom);
151 base->clock_base[CLOCK_REALTIME].softirq_time = xtim;
152 base->clock_base[CLOCK_MONOTONIC].softirq_time =
153 ktime_add(xtim, tomono);
154}
155
156/*
157 * Functions and macros which are different for UP/SMP systems are kept in a
158 * single place
159 */
160#ifdef CONFIG_SMP
161
162/*
163 * We are using hashed locking: holding per_cpu(hrtimer_bases)[n].lock
164 * means that all timers which are tied to this base via timer->base are
165 * locked, and the base itself is locked too.
166 *
167 * So __run_timers/migrate_timers can safely modify all timers which could
168 * be found on the lists/queues.
169 *
170 * When the timer's base is locked, and the timer removed from list, it is
171 * possible to set timer->base = NULL and drop the lock: the timer remains
172 * locked.
173 */
174static
175struct hrtimer_clock_base *lock_hrtimer_base(const struct hrtimer *timer,
176 unsigned long *flags)
177{
178 struct hrtimer_clock_base *base;
179
180 for (;;) {
181 base = timer->base;
182 if (likely(base != NULL)) {
183 spin_lock_irqsave(&base->cpu_base->lock, *flags);
184 if (likely(base == timer->base))
185 return base;
186 /* The timer has migrated to another CPU: */
187 spin_unlock_irqrestore(&base->cpu_base->lock, *flags);
188 }
189 cpu_relax();
190 }
191}
192
193/*
194 * Switch the timer base to the current CPU when possible.
195 */
196static inline struct hrtimer_clock_base *
197switch_hrtimer_base(struct hrtimer *timer, struct hrtimer_clock_base *base)
198{
199 struct hrtimer_clock_base *new_base;
200 struct hrtimer_cpu_base *new_cpu_base;
201
202 new_cpu_base = &__get_cpu_var(hrtimer_bases);
203 new_base = &new_cpu_base->clock_base[base->index];
204
205 if (base != new_base) {
206 /*
207 * We are trying to schedule the timer on the local CPU.
208 * However we can't change timer's base while it is running,
209 * so we keep it on the same CPU. No hassle vs. reprogramming
210 * the event source in the high resolution case. The softirq
211 * code will take care of this when the timer function has
212 * completed. There is no conflict as we hold the lock until
213 * the timer is enqueued.
214 */
215 if (unlikely(hrtimer_callback_running(timer)))
216 return base;
217
218 /* See the comment in lock_timer_base() */
219 timer->base = NULL;
220 spin_unlock(&base->cpu_base->lock);
221 spin_lock(&new_base->cpu_base->lock);
222 timer->base = new_base;
223 }
224 return new_base;
225}
226
227#else /* CONFIG_SMP */
228
229static inline struct hrtimer_clock_base *
230lock_hrtimer_base(const struct hrtimer *timer, unsigned long *flags)
231{
232 struct hrtimer_clock_base *base = timer->base;
233
234 spin_lock_irqsave(&base->cpu_base->lock, *flags);
235
236 return base;
237}
238
239# define switch_hrtimer_base(t, b) (b)
240
241#endif /* !CONFIG_SMP */
242
243/*
244 * Functions for the union type storage format of ktime_t which are
245 * too large for inlining:
246 */
247#if BITS_PER_LONG < 64
248# ifndef CONFIG_KTIME_SCALAR
249/**
250 * ktime_add_ns - Add a scalar nanoseconds value to a ktime_t variable
251 * @kt: addend
252 * @nsec: the scalar nsec value to add
253 *
254 * Returns the sum of kt and nsec in ktime_t format
255 */
256ktime_t ktime_add_ns(const ktime_t kt, u64 nsec)
257{
258 ktime_t tmp;
259
260 if (likely(nsec < NSEC_PER_SEC)) {
261 tmp.tv64 = nsec;
262 } else {
263 unsigned long rem = do_div(nsec, NSEC_PER_SEC);
264
265 tmp = ktime_set((long)nsec, rem);
266 }
267
268 return ktime_add(kt, tmp);
269}
270
271EXPORT_SYMBOL_GPL(ktime_add_ns);
272
273/**
274 * ktime_sub_ns - Subtract a scalar nanoseconds value from a ktime_t variable
275 * @kt: minuend
276 * @nsec: the scalar nsec value to subtract
277 *
278 * Returns the subtraction of @nsec from @kt in ktime_t format
279 */
280ktime_t ktime_sub_ns(const ktime_t kt, u64 nsec)
281{
282 ktime_t tmp;
283
284 if (likely(nsec < NSEC_PER_SEC)) {
285 tmp.tv64 = nsec;
286 } else {
287 unsigned long rem = do_div(nsec, NSEC_PER_SEC);
288
289 tmp = ktime_set((long)nsec, rem);
290 }
291
292 return ktime_sub(kt, tmp);
293}
294
295EXPORT_SYMBOL_GPL(ktime_sub_ns);
296# endif /* !CONFIG_KTIME_SCALAR */
297
298/*
299 * Divide a ktime value by a nanosecond value
300 */
301u64 ktime_divns(const ktime_t kt, s64 div)
302{
303 u64 dclc;
304 int sft = 0;
305
306 dclc = ktime_to_ns(kt);
307 /* Make sure the divisor is less than 2^32: */
308 while (div >> 32) {
309 sft++;
310 div >>= 1;
311 }
312 dclc >>= sft;
313 do_div(dclc, (unsigned long) div);
314
315 return dclc;
316}
317#endif /* BITS_PER_LONG >= 64 */
318
319/*
320 * Add two ktime values and do a safety check for overflow:
321 */
322ktime_t ktime_add_safe(const ktime_t lhs, const ktime_t rhs)
323{
324 ktime_t res = ktime_add(lhs, rhs);
325
326 /*
327 * We use KTIME_SEC_MAX here, the maximum timeout which we can
328 * return to user space in a timespec:
329 */
330 if (res.tv64 < 0 || res.tv64 < lhs.tv64 || res.tv64 < rhs.tv64)
331 res = ktime_set(KTIME_SEC_MAX, 0);
332
333 return res;
334}
335
336#ifdef CONFIG_DEBUG_OBJECTS_TIMERS
337
338static struct debug_obj_descr hrtimer_debug_descr;
339
340/*
341 * fixup_init is called when:
342 * - an active object is initialized
343 */
344static int hrtimer_fixup_init(void *addr, enum debug_obj_state state)
345{
346 struct hrtimer *timer = addr;
347
348 switch (state) {
349 case ODEBUG_STATE_ACTIVE:
350 hrtimer_cancel(timer);
351 debug_object_init(timer, &hrtimer_debug_descr);
352 return 1;
353 default:
354 return 0;
355 }
356}
357
358/*
359 * fixup_activate is called when:
360 * - an active object is activated
361 * - an unknown object is activated (might be a statically initialized object)
362 */
363static int hrtimer_fixup_activate(void *addr, enum debug_obj_state state)
364{
365 switch (state) {
366
367 case ODEBUG_STATE_NOTAVAILABLE:
368 WARN_ON_ONCE(1);
369 return 0;
370
371 case ODEBUG_STATE_ACTIVE:
372 WARN_ON(1);
373
374 default:
375 return 0;
376 }
377}
378
379/*
380 * fixup_free is called when:
381 * - an active object is freed
382 */
383static int hrtimer_fixup_free(void *addr, enum debug_obj_state state)
384{
385 struct hrtimer *timer = addr;
386
387 switch (state) {
388 case ODEBUG_STATE_ACTIVE:
389 hrtimer_cancel(timer);
390 debug_object_free(timer, &hrtimer_debug_descr);
391 return 1;
392 default:
393 return 0;
394 }
395}
396
397static struct debug_obj_descr hrtimer_debug_descr = {
398 .name = "hrtimer",
399 .fixup_init = hrtimer_fixup_init,
400 .fixup_activate = hrtimer_fixup_activate,
401 .fixup_free = hrtimer_fixup_free,
402};
403
404static inline void debug_hrtimer_init(struct hrtimer *timer)
405{
406 debug_object_init(timer, &hrtimer_debug_descr);
407}
408
409static inline void debug_hrtimer_activate(struct hrtimer *timer)
410{
411 debug_object_activate(timer, &hrtimer_debug_descr);
412}
413
414static inline void debug_hrtimer_deactivate(struct hrtimer *timer)
415{
416 debug_object_deactivate(timer, &hrtimer_debug_descr);
417}
418
419static inline void debug_hrtimer_free(struct hrtimer *timer)
420{
421 debug_object_free(timer, &hrtimer_debug_descr);
422}
423
424static void __hrtimer_init(struct hrtimer *timer, clockid_t clock_id,
425 enum hrtimer_mode mode);
426
427void hrtimer_init_on_stack(struct hrtimer *timer, clockid_t clock_id,
428 enum hrtimer_mode mode)
429{
430 debug_object_init_on_stack(timer, &hrtimer_debug_descr);
431 __hrtimer_init(timer, clock_id, mode);
432}
433
434void destroy_hrtimer_on_stack(struct hrtimer *timer)
435{
436 debug_object_free(timer, &hrtimer_debug_descr);
437}
438
439#else
440static inline void debug_hrtimer_init(struct hrtimer *timer) { }
441static inline void debug_hrtimer_activate(struct hrtimer *timer) { }
442static inline void debug_hrtimer_deactivate(struct hrtimer *timer) { }
443#endif
444
445/*
446 * Check, whether the timer is on the callback pending list
447 */
448static inline int hrtimer_cb_pending(const struct hrtimer *timer)
449{
450 return timer->state & HRTIMER_STATE_PENDING;
451}
452
453/*
454 * Remove a timer from the callback pending list
455 */
456static inline void hrtimer_remove_cb_pending(struct hrtimer *timer)
457{
458 list_del_init(&timer->cb_entry);
459}
460
461/* High resolution timer related functions */
462#ifdef CONFIG_HIGH_RES_TIMERS
463
464/*
465 * High resolution timer enabled ?
466 */
467static int hrtimer_hres_enabled __read_mostly = 1;
468
469/*
470 * Enable / Disable high resolution mode
471 */
472static int __init setup_hrtimer_hres(char *str)
473{
474 if (!strcmp(str, "off"))
475 hrtimer_hres_enabled = 0;
476 else if (!strcmp(str, "on"))
477 hrtimer_hres_enabled = 1;
478 else
479 return 0;
480 return 1;
481}
482
483__setup("highres=", setup_hrtimer_hres);
484
485/*
486 * hrtimer_high_res_enabled - query, if the highres mode is enabled
487 */
488static inline int hrtimer_is_hres_enabled(void)
489{
490 return hrtimer_hres_enabled;
491}
492
493/*
494 * Is the high resolution mode active ?
495 */
496static inline int hrtimer_hres_active(void)
497{
498 return __get_cpu_var(hrtimer_bases).hres_active;
499}
500
501/*
502 * Reprogram the event source with checking both queues for the
503 * next event
504 * Called with interrupts disabled and base->lock held
505 */
506static void hrtimer_force_reprogram(struct hrtimer_cpu_base *cpu_base)
507{
508 int i;
509 struct hrtimer_clock_base *base = cpu_base->clock_base;
510 ktime_t expires;
511
512 cpu_base->expires_next.tv64 = KTIME_MAX;
513
514 for (i = 0; i < HRTIMER_MAX_CLOCK_BASES; i++, base++) {
515 struct hrtimer *timer;
516
517 if (!base->first)
518 continue;
519 timer = rb_entry(base->first, struct hrtimer, node);
520 expires = ktime_sub(hrtimer_get_expires(timer), base->offset);
521 if (expires.tv64 < cpu_base->expires_next.tv64)
522 cpu_base->expires_next = expires;
523 }
524
525 if (cpu_base->expires_next.tv64 != KTIME_MAX)
526 tick_program_event(cpu_base->expires_next, 1);
527}
528
529/*
530 * Shared reprogramming for clock_realtime and clock_monotonic
531 *
532 * When a timer is enqueued and expires earlier than the already enqueued
533 * timers, we have to check, whether it expires earlier than the timer for
534 * which the clock event device was armed.
535 *
536 * Called with interrupts disabled and base->cpu_base.lock held
537 */
538static int hrtimer_reprogram(struct hrtimer *timer,
539 struct hrtimer_clock_base *base)
540{
541 ktime_t *expires_next = &__get_cpu_var(hrtimer_bases).expires_next;
542 ktime_t expires = ktime_sub(hrtimer_get_expires(timer), base->offset);
543 int res;
544
545 WARN_ON_ONCE(hrtimer_get_expires_tv64(timer) < 0);
546
547 /*
548 * When the callback is running, we do not reprogram the clock event
549 * device. The timer callback is either running on a different CPU or
550 * the callback is executed in the hrtimer_interrupt context. The
551 * reprogramming is handled either by the softirq, which called the
552 * callback or at the end of the hrtimer_interrupt.
553 */
554 if (hrtimer_callback_running(timer))
555 return 0;
556
557 /*
558 * CLOCK_REALTIME timer might be requested with an absolute
559 * expiry time which is less than base->offset. Nothing wrong
560 * about that, just avoid to call into the tick code, which
561 * has now objections against negative expiry values.
562 */
563 if (expires.tv64 < 0)
564 return -ETIME;
565
566 if (expires.tv64 >= expires_next->tv64)
567 return 0;
568
569 /*
570 * Clockevents returns -ETIME, when the event was in the past.
571 */
572 res = tick_program_event(expires, 0);
573 if (!IS_ERR_VALUE(res))
574 *expires_next = expires;
575 return res;
576}
577
578
579/*
580 * Retrigger next event is called after clock was set
581 *
582 * Called with interrupts disabled via on_each_cpu()
583 */
584static void retrigger_next_event(void *arg)
585{
586 struct hrtimer_cpu_base *base;
587 struct timespec realtime_offset;
588 unsigned long seq;
589
590 if (!hrtimer_hres_active())
591 return;
592
593 do {
594 seq = read_seqbegin(&xtime_lock);
595 set_normalized_timespec(&realtime_offset,
596 -wall_to_monotonic.tv_sec,
597 -wall_to_monotonic.tv_nsec);
598 } while (read_seqretry(&xtime_lock, seq));
599
600 base = &__get_cpu_var(hrtimer_bases);
601
602 /* Adjust CLOCK_REALTIME offset */
603 spin_lock(&base->lock);
604 base->clock_base[CLOCK_REALTIME].offset =
605 timespec_to_ktime(realtime_offset);
606
607 hrtimer_force_reprogram(base);
608 spin_unlock(&base->lock);
609}
610
611/*
612 * Clock realtime was set
613 *
614 * Change the offset of the realtime clock vs. the monotonic
615 * clock.
616 *
617 * We might have to reprogram the high resolution timer interrupt. On
618 * SMP we call the architecture specific code to retrigger _all_ high
619 * resolution timer interrupts. On UP we just disable interrupts and
620 * call the high resolution interrupt code.
621 */
622void clock_was_set(void)
623{
624 /* Retrigger the CPU local events everywhere */
625 on_each_cpu(retrigger_next_event, NULL, 1);
626}
627
628/*
629 * During resume we might have to reprogram the high resolution timer
630 * interrupt (on the local CPU):
631 */
632void hres_timers_resume(void)
633{
634 /* Retrigger the CPU local events: */
635 retrigger_next_event(NULL);
636}
637
638/*
639 * Initialize the high resolution related parts of cpu_base
640 */
641static inline void hrtimer_init_hres(struct hrtimer_cpu_base *base)
642{
643 base->expires_next.tv64 = KTIME_MAX;
644 base->hres_active = 0;
645}
646
647/*
648 * Initialize the high resolution related parts of a hrtimer
649 */
650static inline void hrtimer_init_timer_hres(struct hrtimer *timer)
651{
652}
653
654/*
655 * When High resolution timers are active, try to reprogram. Note, that in case
656 * the state has HRTIMER_STATE_CALLBACK set, no reprogramming and no expiry
657 * check happens. The timer gets enqueued into the rbtree. The reprogramming
658 * and expiry check is done in the hrtimer_interrupt or in the softirq.
659 */
660static inline int hrtimer_enqueue_reprogram(struct hrtimer *timer,
661 struct hrtimer_clock_base *base)
662{
663 if (base->cpu_base->hres_active && hrtimer_reprogram(timer, base)) {
664
665 /* Timer is expired, act upon the callback mode */
666 switch(timer->cb_mode) {
667 case HRTIMER_CB_IRQSAFE_NO_RESTART:
668 debug_hrtimer_deactivate(timer);
669 /*
670 * We can call the callback from here. No restart
671 * happens, so no danger of recursion
672 */
673 BUG_ON(timer->function(timer) != HRTIMER_NORESTART);
674 return 1;
675 case HRTIMER_CB_IRQSAFE_PERCPU:
676 case HRTIMER_CB_IRQSAFE_UNLOCKED:
677 /*
678 * This is solely for the sched tick emulation with
679 * dynamic tick support to ensure that we do not
680 * restart the tick right on the edge and end up with
681 * the tick timer in the softirq ! The calling site
682 * takes care of this. Also used for hrtimer sleeper !
683 */
684 debug_hrtimer_deactivate(timer);
685 return 1;
686 case HRTIMER_CB_IRQSAFE:
687 case HRTIMER_CB_SOFTIRQ:
688 /*
689 * Move everything else into the softirq pending list !
690 */
691 list_add_tail(&timer->cb_entry,
692 &base->cpu_base->cb_pending);
693 timer->state = HRTIMER_STATE_PENDING;
694 return 1;
695 default:
696 BUG();
697 }
698 }
699 return 0;
700}
701
702/*
703 * Switch to high resolution mode
704 */
705static int hrtimer_switch_to_hres(void)
706{
707 int cpu = smp_processor_id();
708 struct hrtimer_cpu_base *base = &per_cpu(hrtimer_bases, cpu);
709 unsigned long flags;
710
711 if (base->hres_active)
712 return 1;
713
714 local_irq_save(flags);
715
716 if (tick_init_highres()) {
717 local_irq_restore(flags);
718 printk(KERN_WARNING "Could not switch to high resolution "
719 "mode on CPU %d\n", cpu);
720 return 0;
721 }
722 base->hres_active = 1;
723 base->clock_base[CLOCK_REALTIME].resolution = KTIME_HIGH_RES;
724 base->clock_base[CLOCK_MONOTONIC].resolution = KTIME_HIGH_RES;
725
726 tick_setup_sched_timer();
727
728 /* "Retrigger" the interrupt to get things going */
729 retrigger_next_event(NULL);
730 local_irq_restore(flags);
731 printk(KERN_DEBUG "Switched to high resolution mode on CPU %d\n",
732 smp_processor_id());
733 return 1;
734}
735
736static inline void hrtimer_raise_softirq(void)
737{
738 raise_softirq(HRTIMER_SOFTIRQ);
739}
740
741#else
742
743static inline int hrtimer_hres_active(void) { return 0; }
744static inline int hrtimer_is_hres_enabled(void) { return 0; }
745static inline int hrtimer_switch_to_hres(void) { return 0; }
746static inline void hrtimer_force_reprogram(struct hrtimer_cpu_base *base) { }
747static inline int hrtimer_enqueue_reprogram(struct hrtimer *timer,
748 struct hrtimer_clock_base *base)
749{
750 return 0;
751}
752static inline void hrtimer_init_hres(struct hrtimer_cpu_base *base) { }
753static inline void hrtimer_init_timer_hres(struct hrtimer *timer) { }
754static inline int hrtimer_reprogram(struct hrtimer *timer,
755 struct hrtimer_clock_base *base)
756{
757 return 0;
758}
759static inline void hrtimer_raise_softirq(void) { }
760
761#endif /* CONFIG_HIGH_RES_TIMERS */
762
763#ifdef CONFIG_TIMER_STATS
764void __timer_stats_hrtimer_set_start_info(struct hrtimer *timer, void *addr)
765{
766 if (timer->start_site)
767 return;
768
769 timer->start_site = addr;
770 memcpy(timer->start_comm, current->comm, TASK_COMM_LEN);
771 timer->start_pid = current->pid;
772}
773#endif
774
775/*
776 * Counterpart to lock_hrtimer_base above:
777 */
778static inline
779void unlock_hrtimer_base(const struct hrtimer *timer, unsigned long *flags)
780{
781 spin_unlock_irqrestore(&timer->base->cpu_base->lock, *flags);
782}
783
784/**
785 * hrtimer_forward - forward the timer expiry
786 * @timer: hrtimer to forward
787 * @now: forward past this time
788 * @interval: the interval to forward
789 *
790 * Forward the timer expiry so it will expire in the future.
791 * Returns the number of overruns.
792 */
793u64 hrtimer_forward(struct hrtimer *timer, ktime_t now, ktime_t interval)
794{
795 u64 orun = 1;
796 ktime_t delta;
797
798 delta = ktime_sub(now, hrtimer_get_expires(timer));
799
800 if (delta.tv64 < 0)
801 return 0;
802
803 if (interval.tv64 < timer->base->resolution.tv64)
804 interval.tv64 = timer->base->resolution.tv64;
805
806 if (unlikely(delta.tv64 >= interval.tv64)) {
807 s64 incr = ktime_to_ns(interval);
808
809 orun = ktime_divns(delta, incr);
810 hrtimer_add_expires_ns(timer, incr * orun);
811 if (hrtimer_get_expires_tv64(timer) > now.tv64)
812 return orun;
813 /*
814 * This (and the ktime_add() below) is the
815 * correction for exact:
816 */
817 orun++;
818 }
819 hrtimer_add_expires(timer, interval);
820
821 return orun;
822}
823EXPORT_SYMBOL_GPL(hrtimer_forward);
824
825/*
826 * enqueue_hrtimer - internal function to (re)start a timer
827 *
828 * The timer is inserted in expiry order. Insertion into the
829 * red black tree is O(log(n)). Must hold the base lock.
830 */
831static void enqueue_hrtimer(struct hrtimer *timer,
832 struct hrtimer_clock_base *base, int reprogram)
833{
834 struct rb_node **link = &base->active.rb_node;
835 struct rb_node *parent = NULL;
836 struct hrtimer *entry;
837 int leftmost = 1;
838
839 debug_hrtimer_activate(timer);
840
841 /*
842 * Find the right place in the rbtree:
843 */
844 while (*link) {
845 parent = *link;
846 entry = rb_entry(parent, struct hrtimer, node);
847 /*
848 * We dont care about collisions. Nodes with
849 * the same expiry time stay together.
850 */
851 if (hrtimer_get_expires_tv64(timer) <
852 hrtimer_get_expires_tv64(entry)) {
853 link = &(*link)->rb_left;
854 } else {
855 link = &(*link)->rb_right;
856 leftmost = 0;
857 }
858 }
859
860 /*
861 * Insert the timer to the rbtree and check whether it
862 * replaces the first pending timer
863 */
864 if (leftmost) {
865 /*
866 * Reprogram the clock event device. When the timer is already
867 * expired hrtimer_enqueue_reprogram has either called the
868 * callback or added it to the pending list and raised the
869 * softirq.
870 *
871 * This is a NOP for !HIGHRES
872 */
873 if (reprogram && hrtimer_enqueue_reprogram(timer, base))
874 return;
875
876 base->first = &timer->node;
877 }
878
879 rb_link_node(&timer->node, parent, link);
880 rb_insert_color(&timer->node, &base->active);
881 /*
882 * HRTIMER_STATE_ENQUEUED is or'ed to the current state to preserve the
883 * state of a possibly running callback.
884 */
885 timer->state |= HRTIMER_STATE_ENQUEUED;
886}
887
888/*
889 * __remove_hrtimer - internal function to remove a timer
890 *
891 * Caller must hold the base lock.
892 *
893 * High resolution timer mode reprograms the clock event device when the
894 * timer is the one which expires next. The caller can disable this by setting
895 * reprogram to zero. This is useful, when the context does a reprogramming
896 * anyway (e.g. timer interrupt)
897 */
898static void __remove_hrtimer(struct hrtimer *timer,
899 struct hrtimer_clock_base *base,
900 unsigned long newstate, int reprogram)
901{
902 /* High res. callback list. NOP for !HIGHRES */
903 if (hrtimer_cb_pending(timer))
904 hrtimer_remove_cb_pending(timer);
905 else {
906 /*
907 * Remove the timer from the rbtree and replace the
908 * first entry pointer if necessary.
909 */
910 if (base->first == &timer->node) {
911 base->first = rb_next(&timer->node);
912 /* Reprogram the clock event device. if enabled */
913 if (reprogram && hrtimer_hres_active())
914 hrtimer_force_reprogram(base->cpu_base);
915 }
916 rb_erase(&timer->node, &base->active);
917 }
918 timer->state = newstate;
919}
920
921/*
922 * remove hrtimer, called with base lock held
923 */
924static inline int
925remove_hrtimer(struct hrtimer *timer, struct hrtimer_clock_base *base)
926{
927 if (hrtimer_is_queued(timer)) {
928 int reprogram;
929
930 /*
931 * Remove the timer and force reprogramming when high
932 * resolution mode is active and the timer is on the current
933 * CPU. If we remove a timer on another CPU, reprogramming is
934 * skipped. The interrupt event on this CPU is fired and
935 * reprogramming happens in the interrupt handler. This is a
936 * rare case and less expensive than a smp call.
937 */
938 debug_hrtimer_deactivate(timer);
939 timer_stats_hrtimer_clear_start_info(timer);
940 reprogram = base->cpu_base == &__get_cpu_var(hrtimer_bases);
941 __remove_hrtimer(timer, base, HRTIMER_STATE_INACTIVE,
942 reprogram);
943 return 1;
944 }
945 return 0;
946}
947
948/**
949 * hrtimer_start_range_ns - (re)start an hrtimer on the current CPU
950 * @timer: the timer to be added
951 * @tim: expiry time
952 * @delta_ns: "slack" range for the timer
953 * @mode: expiry mode: absolute (HRTIMER_ABS) or relative (HRTIMER_REL)
954 *
955 * Returns:
956 * 0 on success
957 * 1 when the timer was active
958 */
959int
960hrtimer_start_range_ns(struct hrtimer *timer, ktime_t tim, unsigned long delta_ns,
961 const enum hrtimer_mode mode)
962{
963 struct hrtimer_clock_base *base, *new_base;
964 unsigned long flags;
965 int ret, raise;
966
967 base = lock_hrtimer_base(timer, &flags);
968
969 /* Remove an active timer from the queue: */
970 ret = remove_hrtimer(timer, base);
971
972 /* Switch the timer base, if necessary: */
973 new_base = switch_hrtimer_base(timer, base);
974
975 if (mode == HRTIMER_MODE_REL) {
976 tim = ktime_add_safe(tim, new_base->get_time());
977 /*
978 * CONFIG_TIME_LOW_RES is a temporary way for architectures
979 * to signal that they simply return xtime in
980 * do_gettimeoffset(). In this case we want to round up by
981 * resolution when starting a relative timer, to avoid short
982 * timeouts. This will go away with the GTOD framework.
983 */
984#ifdef CONFIG_TIME_LOW_RES
985 tim = ktime_add_safe(tim, base->resolution);
986#endif
987 }
988
989 hrtimer_set_expires_range_ns(timer, tim, delta_ns);
990
991 timer_stats_hrtimer_set_start_info(timer);
992
993 /*
994 * Only allow reprogramming if the new base is on this CPU.
995 * (it might still be on another CPU if the timer was pending)
996 */
997 enqueue_hrtimer(timer, new_base,
998 new_base->cpu_base == &__get_cpu_var(hrtimer_bases));
999
1000 /*
1001 * The timer may be expired and moved to the cb_pending
1002 * list. We can not raise the softirq with base lock held due
1003 * to a possible deadlock with runqueue lock.
1004 */
1005 raise = timer->state == HRTIMER_STATE_PENDING;
1006
1007 /*
1008 * We use preempt_disable to prevent this task from migrating after
1009 * setting up the softirq and raising it. Otherwise, if me migrate
1010 * we will raise the softirq on the wrong CPU.
1011 */
1012 preempt_disable();
1013
1014 unlock_hrtimer_base(timer, &flags);
1015
1016 if (raise)
1017 hrtimer_raise_softirq();
1018 preempt_enable();
1019
1020 return ret;
1021}
1022EXPORT_SYMBOL_GPL(hrtimer_start_range_ns);
1023
1024/**
1025 * hrtimer_start - (re)start an hrtimer on the current CPU
1026 * @timer: the timer to be added
1027 * @tim: expiry time
1028 * @mode: expiry mode: absolute (HRTIMER_ABS) or relative (HRTIMER_REL)
1029 *
1030 * Returns:
1031 * 0 on success
1032 * 1 when the timer was active
1033 */
1034int
1035hrtimer_start(struct hrtimer *timer, ktime_t tim, const enum hrtimer_mode mode)
1036{
1037 return hrtimer_start_range_ns(timer, tim, 0, mode);
1038}
1039EXPORT_SYMBOL_GPL(hrtimer_start);
1040
1041
1042/**
1043 * hrtimer_try_to_cancel - try to deactivate a timer
1044 * @timer: hrtimer to stop
1045 *
1046 * Returns:
1047 * 0 when the timer was not active
1048 * 1 when the timer was active
1049 * -1 when the timer is currently excuting the callback function and
1050 * cannot be stopped
1051 */
1052int hrtimer_try_to_cancel(struct hrtimer *timer)
1053{
1054 struct hrtimer_clock_base *base;
1055 unsigned long flags;
1056 int ret = -1;
1057
1058 base = lock_hrtimer_base(timer, &flags);
1059
1060 if (!hrtimer_callback_running(timer))
1061 ret = remove_hrtimer(timer, base);
1062
1063 unlock_hrtimer_base(timer, &flags);
1064
1065 return ret;
1066
1067}
1068EXPORT_SYMBOL_GPL(hrtimer_try_to_cancel);
1069
1070/**
1071 * hrtimer_cancel - cancel a timer and wait for the handler to finish.
1072 * @timer: the timer to be cancelled
1073 *
1074 * Returns:
1075 * 0 when the timer was not active
1076 * 1 when the timer was active
1077 */
1078int hrtimer_cancel(struct hrtimer *timer)
1079{
1080 for (;;) {
1081 int ret = hrtimer_try_to_cancel(timer);
1082
1083 if (ret >= 0)
1084 return ret;
1085 cpu_relax();
1086 }
1087}
1088EXPORT_SYMBOL_GPL(hrtimer_cancel);
1089
1090/**
1091 * hrtimer_get_remaining - get remaining time for the timer
1092 * @timer: the timer to read
1093 */
1094ktime_t hrtimer_get_remaining(const struct hrtimer *timer)
1095{
1096 struct hrtimer_clock_base *base;
1097 unsigned long flags;
1098 ktime_t rem;
1099
1100 base = lock_hrtimer_base(timer, &flags);
1101 rem = hrtimer_expires_remaining(timer);
1102 unlock_hrtimer_base(timer, &flags);
1103
1104 return rem;
1105}
1106EXPORT_SYMBOL_GPL(hrtimer_get_remaining);
1107
1108#ifdef CONFIG_NO_HZ
1109/**
1110 * hrtimer_get_next_event - get the time until next expiry event
1111 *
1112 * Returns the delta to the next expiry event or KTIME_MAX if no timer
1113 * is pending.
1114 */
1115ktime_t hrtimer_get_next_event(void)
1116{
1117 struct hrtimer_cpu_base *cpu_base = &__get_cpu_var(hrtimer_bases);
1118 struct hrtimer_clock_base *base = cpu_base->clock_base;
1119 ktime_t delta, mindelta = { .tv64 = KTIME_MAX };
1120 unsigned long flags;
1121 int i;
1122
1123 spin_lock_irqsave(&cpu_base->lock, flags);
1124
1125 if (!hrtimer_hres_active()) {
1126 for (i = 0; i < HRTIMER_MAX_CLOCK_BASES; i++, base++) {
1127 struct hrtimer *timer;
1128
1129 if (!base->first)
1130 continue;
1131
1132 timer = rb_entry(base->first, struct hrtimer, node);
1133 delta.tv64 = hrtimer_get_expires_tv64(timer);
1134 delta = ktime_sub(delta, base->get_time());
1135 if (delta.tv64 < mindelta.tv64)
1136 mindelta.tv64 = delta.tv64;
1137 }
1138 }
1139
1140 spin_unlock_irqrestore(&cpu_base->lock, flags);
1141
1142 if (mindelta.tv64 < 0)
1143 mindelta.tv64 = 0;
1144 return mindelta;
1145}
1146#endif
1147
1148static void __hrtimer_init(struct hrtimer *timer, clockid_t clock_id,
1149 enum hrtimer_mode mode)
1150{
1151 struct hrtimer_cpu_base *cpu_base;
1152
1153 memset(timer, 0, sizeof(struct hrtimer));
1154
1155 cpu_base = &__raw_get_cpu_var(hrtimer_bases);
1156
1157 if (clock_id == CLOCK_REALTIME && mode != HRTIMER_MODE_ABS)
1158 clock_id = CLOCK_MONOTONIC;
1159
1160 timer->base = &cpu_base->clock_base[clock_id];
1161 INIT_LIST_HEAD(&timer->cb_entry);
1162 hrtimer_init_timer_hres(timer);
1163
1164#ifdef CONFIG_TIMER_STATS
1165 timer->start_site = NULL;
1166 timer->start_pid = -1;
1167 memset(timer->start_comm, 0, TASK_COMM_LEN);
1168#endif
1169}
1170
1171/**
1172 * hrtimer_init - initialize a timer to the given clock
1173 * @timer: the timer to be initialized
1174 * @clock_id: the clock to be used
1175 * @mode: timer mode abs/rel
1176 */
1177void hrtimer_init(struct hrtimer *timer, clockid_t clock_id,
1178 enum hrtimer_mode mode)
1179{
1180 debug_hrtimer_init(timer);
1181 __hrtimer_init(timer, clock_id, mode);
1182}
1183EXPORT_SYMBOL_GPL(hrtimer_init);
1184
1185/**
1186 * hrtimer_get_res - get the timer resolution for a clock
1187 * @which_clock: which clock to query
1188 * @tp: pointer to timespec variable to store the resolution
1189 *
1190 * Store the resolution of the clock selected by @which_clock in the
1191 * variable pointed to by @tp.
1192 */
1193int hrtimer_get_res(const clockid_t which_clock, struct timespec *tp)
1194{
1195 struct hrtimer_cpu_base *cpu_base;
1196
1197 cpu_base = &__raw_get_cpu_var(hrtimer_bases);
1198 *tp = ktime_to_timespec(cpu_base->clock_base[which_clock].resolution);
1199
1200 return 0;
1201}
1202EXPORT_SYMBOL_GPL(hrtimer_get_res);
1203
1204static void run_hrtimer_pending(struct hrtimer_cpu_base *cpu_base)
1205{
1206 spin_lock_irq(&cpu_base->lock);
1207
1208 while (!list_empty(&cpu_base->cb_pending)) {
1209 enum hrtimer_restart (*fn)(struct hrtimer *);
1210 struct hrtimer *timer;
1211 int restart;
1212
1213 timer = list_entry(cpu_base->cb_pending.next,
1214 struct hrtimer, cb_entry);
1215
1216 debug_hrtimer_deactivate(timer);
1217 timer_stats_account_hrtimer(timer);
1218
1219 fn = timer->function;
1220 __remove_hrtimer(timer, timer->base, HRTIMER_STATE_CALLBACK, 0);
1221 spin_unlock_irq(&cpu_base->lock);
1222
1223 restart = fn(timer);
1224
1225 spin_lock_irq(&cpu_base->lock);
1226
1227 timer->state &= ~HRTIMER_STATE_CALLBACK;
1228 if (restart == HRTIMER_RESTART) {
1229 BUG_ON(hrtimer_active(timer));
1230 /*
1231 * Enqueue the timer, allow reprogramming of the event
1232 * device
1233 */
1234 enqueue_hrtimer(timer, timer->base, 1);
1235 } else if (hrtimer_active(timer)) {
1236 /*
1237 * If the timer was rearmed on another CPU, reprogram
1238 * the event device.
1239 */
1240 struct hrtimer_clock_base *base = timer->base;
1241
1242 if (base->first == &timer->node &&
1243 hrtimer_reprogram(timer, base)) {
1244 /*
1245 * Timer is expired. Thus move it from tree to
1246 * pending list again.
1247 */
1248 __remove_hrtimer(timer, base,
1249 HRTIMER_STATE_PENDING, 0);
1250 list_add_tail(&timer->cb_entry,
1251 &base->cpu_base->cb_pending);
1252 }
1253 }
1254 }
1255 spin_unlock_irq(&cpu_base->lock);
1256}
1257
1258static void __run_hrtimer(struct hrtimer *timer)
1259{
1260 struct hrtimer_clock_base *base = timer->base;
1261 struct hrtimer_cpu_base *cpu_base = base->cpu_base;
1262 enum hrtimer_restart (*fn)(struct hrtimer *);
1263 int restart;
1264
1265 debug_hrtimer_deactivate(timer);
1266 __remove_hrtimer(timer, base, HRTIMER_STATE_CALLBACK, 0);
1267 timer_stats_account_hrtimer(timer);
1268
1269 fn = timer->function;
1270 if (timer->cb_mode == HRTIMER_CB_IRQSAFE_PERCPU ||
1271 timer->cb_mode == HRTIMER_CB_IRQSAFE_UNLOCKED) {
1272 /*
1273 * Used for scheduler timers, avoid lock inversion with
1274 * rq->lock and tasklist_lock.
1275 *
1276 * These timers are required to deal with enqueue expiry
1277 * themselves and are not allowed to migrate.
1278 */
1279 spin_unlock(&cpu_base->lock);
1280 restart = fn(timer);
1281 spin_lock(&cpu_base->lock);
1282 } else
1283 restart = fn(timer);
1284
1285 /*
1286 * Note: We clear the CALLBACK bit after enqueue_hrtimer to avoid
1287 * reprogramming of the event hardware. This happens at the end of this
1288 * function anyway.
1289 */
1290 if (restart != HRTIMER_NORESTART) {
1291 BUG_ON(timer->state != HRTIMER_STATE_CALLBACK);
1292 enqueue_hrtimer(timer, base, 0);
1293 }
1294 timer->state &= ~HRTIMER_STATE_CALLBACK;
1295}
1296
1297#ifdef CONFIG_HIGH_RES_TIMERS
1298
1299/*
1300 * High resolution timer interrupt
1301 * Called with interrupts disabled
1302 */
1303void hrtimer_interrupt(struct clock_event_device *dev)
1304{
1305 struct hrtimer_cpu_base *cpu_base = &__get_cpu_var(hrtimer_bases);
1306 struct hrtimer_clock_base *base;
1307 ktime_t expires_next, now;
1308 int i, raise = 0;
1309
1310 BUG_ON(!cpu_base->hres_active);
1311 cpu_base->nr_events++;
1312 dev->next_event.tv64 = KTIME_MAX;
1313
1314 retry:
1315 now = ktime_get();
1316
1317 expires_next.tv64 = KTIME_MAX;
1318
1319 base = cpu_base->clock_base;
1320
1321 for (i = 0; i < HRTIMER_MAX_CLOCK_BASES; i++) {
1322 ktime_t basenow;
1323 struct rb_node *node;
1324
1325 spin_lock(&cpu_base->lock);
1326
1327 basenow = ktime_add(now, base->offset);
1328
1329 while ((node = base->first)) {
1330 struct hrtimer *timer;
1331
1332 timer = rb_entry(node, struct hrtimer, node);
1333
1334 /*
1335 * The immediate goal for using the softexpires is
1336 * minimizing wakeups, not running timers at the
1337 * earliest interrupt after their soft expiration.
1338 * This allows us to avoid using a Priority Search
1339 * Tree, which can answer a stabbing querry for
1340 * overlapping intervals and instead use the simple
1341 * BST we already have.
1342 * We don't add extra wakeups by delaying timers that
1343 * are right-of a not yet expired timer, because that
1344 * timer will have to trigger a wakeup anyway.
1345 */
1346
1347 if (basenow.tv64 < hrtimer_get_softexpires_tv64(timer)) {
1348 ktime_t expires;
1349
1350 expires = ktime_sub(hrtimer_get_expires(timer),
1351 base->offset);
1352 if (expires.tv64 < expires_next.tv64)
1353 expires_next = expires;
1354 break;
1355 }
1356
1357 /* Move softirq callbacks to the pending list */
1358 if (timer->cb_mode == HRTIMER_CB_SOFTIRQ) {
1359 __remove_hrtimer(timer, base,
1360 HRTIMER_STATE_PENDING, 0);
1361 list_add_tail(&timer->cb_entry,
1362 &base->cpu_base->cb_pending);
1363 raise = 1;
1364 continue;
1365 }
1366
1367 __run_hrtimer(timer);
1368 }
1369 spin_unlock(&cpu_base->lock);
1370 base++;
1371 }
1372
1373 cpu_base->expires_next = expires_next;
1374
1375 /* Reprogramming necessary ? */
1376 if (expires_next.tv64 != KTIME_MAX) {
1377 if (tick_program_event(expires_next, 0))
1378 goto retry;
1379 }
1380
1381 /* Raise softirq ? */
1382 if (raise)
1383 raise_softirq(HRTIMER_SOFTIRQ);
1384}
1385
1386/**
1387 * hrtimer_peek_ahead_timers -- run soft-expired timers now
1388 *
1389 * hrtimer_peek_ahead_timers will peek at the timer queue of
1390 * the current cpu and check if there are any timers for which
1391 * the soft expires time has passed. If any such timers exist,
1392 * they are run immediately and then removed from the timer queue.
1393 *
1394 */
1395void hrtimer_peek_ahead_timers(void)
1396{
1397 struct tick_device *td;
1398 unsigned long flags;
1399
1400 if (!hrtimer_hres_active())
1401 return;
1402
1403 local_irq_save(flags);
1404 td = &__get_cpu_var(tick_cpu_device);
1405 if (td && td->evtdev)
1406 hrtimer_interrupt(td->evtdev);
1407 local_irq_restore(flags);
1408}
1409
1410static void run_hrtimer_softirq(struct softirq_action *h)
1411{
1412 run_hrtimer_pending(&__get_cpu_var(hrtimer_bases));
1413}
1414
1415#endif /* CONFIG_HIGH_RES_TIMERS */
1416
1417/*
1418 * Called from timer softirq every jiffy, expire hrtimers:
1419 *
1420 * For HRT its the fall back code to run the softirq in the timer
1421 * softirq context in case the hrtimer initialization failed or has
1422 * not been done yet.
1423 */
1424void hrtimer_run_pending(void)
1425{
1426 struct hrtimer_cpu_base *cpu_base = &__get_cpu_var(hrtimer_bases);
1427
1428 if (hrtimer_hres_active())
1429 return;
1430
1431 /*
1432 * This _is_ ugly: We have to check in the softirq context,
1433 * whether we can switch to highres and / or nohz mode. The
1434 * clocksource switch happens in the timer interrupt with
1435 * xtime_lock held. Notification from there only sets the
1436 * check bit in the tick_oneshot code, otherwise we might
1437 * deadlock vs. xtime_lock.
1438 */
1439 if (tick_check_oneshot_change(!hrtimer_is_hres_enabled()))
1440 hrtimer_switch_to_hres();
1441
1442 run_hrtimer_pending(cpu_base);
1443}
1444
1445/*
1446 * Called from hardirq context every jiffy
1447 */
1448void hrtimer_run_queues(void)
1449{
1450 struct rb_node *node;
1451 struct hrtimer_cpu_base *cpu_base = &__get_cpu_var(hrtimer_bases);
1452 struct hrtimer_clock_base *base;
1453 int index, gettime = 1;
1454
1455 if (hrtimer_hres_active())
1456 return;
1457
1458 for (index = 0; index < HRTIMER_MAX_CLOCK_BASES; index++) {
1459 base = &cpu_base->clock_base[index];
1460
1461 if (!base->first)
1462 continue;
1463
1464 if (gettime) {
1465 hrtimer_get_softirq_time(cpu_base);
1466 gettime = 0;
1467 }
1468
1469 spin_lock(&cpu_base->lock);
1470
1471 while ((node = base->first)) {
1472 struct hrtimer *timer;
1473
1474 timer = rb_entry(node, struct hrtimer, node);
1475 if (base->softirq_time.tv64 <=
1476 hrtimer_get_expires_tv64(timer))
1477 break;
1478
1479 if (timer->cb_mode == HRTIMER_CB_SOFTIRQ) {
1480 __remove_hrtimer(timer, base,
1481 HRTIMER_STATE_PENDING, 0);
1482 list_add_tail(&timer->cb_entry,
1483 &base->cpu_base->cb_pending);
1484 continue;
1485 }
1486
1487 __run_hrtimer(timer);
1488 }
1489 spin_unlock(&cpu_base->lock);
1490 }
1491}
1492
1493/*
1494 * Sleep related functions:
1495 */
1496static enum hrtimer_restart hrtimer_wakeup(struct hrtimer *timer)
1497{
1498 struct hrtimer_sleeper *t =
1499 container_of(timer, struct hrtimer_sleeper, timer);
1500 struct task_struct *task = t->task;
1501
1502 t->task = NULL;
1503 if (task)
1504 wake_up_process(task);
1505
1506 return HRTIMER_NORESTART;
1507}
1508
1509void hrtimer_init_sleeper(struct hrtimer_sleeper *sl, struct task_struct *task)
1510{
1511 sl->timer.function = hrtimer_wakeup;
1512 sl->task = task;
1513#ifdef CONFIG_HIGH_RES_TIMERS
1514 sl->timer.cb_mode = HRTIMER_CB_IRQSAFE_UNLOCKED;
1515#endif
1516}
1517
1518static int __sched do_nanosleep(struct hrtimer_sleeper *t, enum hrtimer_mode mode)
1519{
1520 hrtimer_init_sleeper(t, current);
1521
1522 do {
1523 set_current_state(TASK_INTERRUPTIBLE);
1524 hrtimer_start_expires(&t->timer, mode);
1525 if (!hrtimer_active(&t->timer))
1526 t->task = NULL;
1527
1528 if (likely(t->task))
1529 schedule();
1530
1531 hrtimer_cancel(&t->timer);
1532 mode = HRTIMER_MODE_ABS;
1533
1534 } while (t->task && !signal_pending(current));
1535
1536 __set_current_state(TASK_RUNNING);
1537
1538 return t->task == NULL;
1539}
1540
1541static int update_rmtp(struct hrtimer *timer, struct timespec __user *rmtp)
1542{
1543 struct timespec rmt;
1544 ktime_t rem;
1545
1546 rem = hrtimer_expires_remaining(timer);
1547 if (rem.tv64 <= 0)
1548 return 0;
1549 rmt = ktime_to_timespec(rem);
1550
1551 if (copy_to_user(rmtp, &rmt, sizeof(*rmtp)))
1552 return -EFAULT;
1553
1554 return 1;
1555}
1556
1557long __sched hrtimer_nanosleep_restart(struct restart_block *restart)
1558{
1559 struct hrtimer_sleeper t;
1560 struct timespec __user *rmtp;
1561 int ret = 0;
1562
1563 hrtimer_init_on_stack(&t.timer, restart->nanosleep.index,
1564 HRTIMER_MODE_ABS);
1565 hrtimer_set_expires_tv64(&t.timer, restart->nanosleep.expires);
1566
1567 if (do_nanosleep(&t, HRTIMER_MODE_ABS))
1568 goto out;
1569
1570 rmtp = restart->nanosleep.rmtp;
1571 if (rmtp) {
1572 ret = update_rmtp(&t.timer, rmtp);
1573 if (ret <= 0)
1574 goto out;
1575 }
1576
1577 /* The other values in restart are already filled in */
1578 ret = -ERESTART_RESTARTBLOCK;
1579out:
1580 destroy_hrtimer_on_stack(&t.timer);
1581 return ret;
1582}
1583
1584long hrtimer_nanosleep(struct timespec *rqtp, struct timespec __user *rmtp,
1585 const enum hrtimer_mode mode, const clockid_t clockid)
1586{
1587 struct restart_block *restart;
1588 struct hrtimer_sleeper t;
1589 int ret = 0;
1590 unsigned long slack;
1591
1592 slack = current->timer_slack_ns;
1593 if (rt_task(current))
1594 slack = 0;
1595
1596 hrtimer_init_on_stack(&t.timer, clockid, mode);
1597 hrtimer_set_expires_range_ns(&t.timer, timespec_to_ktime(*rqtp), slack);
1598 if (do_nanosleep(&t, mode))
1599 goto out;
1600
1601 /* Absolute timers do not update the rmtp value and restart: */
1602 if (mode == HRTIMER_MODE_ABS) {
1603 ret = -ERESTARTNOHAND;
1604 goto out;
1605 }
1606
1607 if (rmtp) {
1608 ret = update_rmtp(&t.timer, rmtp);
1609 if (ret <= 0)
1610 goto out;
1611 }
1612
1613 restart = &current_thread_info()->restart_block;
1614 restart->fn = hrtimer_nanosleep_restart;
1615 restart->nanosleep.index = t.timer.base->index;
1616 restart->nanosleep.rmtp = rmtp;
1617 restart->nanosleep.expires = hrtimer_get_expires_tv64(&t.timer);
1618
1619 ret = -ERESTART_RESTARTBLOCK;
1620out:
1621 destroy_hrtimer_on_stack(&t.timer);
1622 return ret;
1623}
1624
1625asmlinkage long
1626sys_nanosleep(struct timespec __user *rqtp, struct timespec __user *rmtp)
1627{
1628 struct timespec tu;
1629
1630 if (copy_from_user(&tu, rqtp, sizeof(tu)))
1631 return -EFAULT;
1632
1633 if (!timespec_valid(&tu))
1634 return -EINVAL;
1635
1636 return hrtimer_nanosleep(&tu, rmtp, HRTIMER_MODE_REL, CLOCK_MONOTONIC);
1637}
1638
1639/*
1640 * Functions related to boot-time initialization:
1641 */
1642static void __cpuinit init_hrtimers_cpu(int cpu)
1643{
1644 struct hrtimer_cpu_base *cpu_base = &per_cpu(hrtimer_bases, cpu);
1645 int i;
1646
1647 spin_lock_init(&cpu_base->lock);
1648
1649 for (i = 0; i < HRTIMER_MAX_CLOCK_BASES; i++)
1650 cpu_base->clock_base[i].cpu_base = cpu_base;
1651
1652 INIT_LIST_HEAD(&cpu_base->cb_pending);
1653 hrtimer_init_hres(cpu_base);
1654}
1655
1656#ifdef CONFIG_HOTPLUG_CPU
1657
1658static int migrate_hrtimer_list(struct hrtimer_clock_base *old_base,
1659 struct hrtimer_clock_base *new_base, int dcpu)
1660{
1661 struct hrtimer *timer;
1662 struct rb_node *node;
1663 int raise = 0;
1664
1665 while ((node = rb_first(&old_base->active))) {
1666 timer = rb_entry(node, struct hrtimer, node);
1667 BUG_ON(hrtimer_callback_running(timer));
1668 debug_hrtimer_deactivate(timer);
1669
1670 /*
1671 * Should not happen. Per CPU timers should be
1672 * canceled _before_ the migration code is called
1673 */
1674 if (timer->cb_mode == HRTIMER_CB_IRQSAFE_PERCPU) {
1675 __remove_hrtimer(timer, old_base,
1676 HRTIMER_STATE_INACTIVE, 0);
1677 WARN(1, "hrtimer (%p %p)active but cpu %d dead\n",
1678 timer, timer->function, dcpu);
1679 continue;
1680 }
1681
1682 /*
1683 * Mark it as STATE_MIGRATE not INACTIVE otherwise the
1684 * timer could be seen as !active and just vanish away
1685 * under us on another CPU
1686 */
1687 __remove_hrtimer(timer, old_base, HRTIMER_STATE_MIGRATE, 0);
1688 timer->base = new_base;
1689 /*
1690 * Enqueue the timer. Allow reprogramming of the event device
1691 */
1692 enqueue_hrtimer(timer, new_base, 1);
1693
1694#ifdef CONFIG_HIGH_RES_TIMERS
1695 /*
1696 * Happens with high res enabled when the timer was
1697 * already expired and the callback mode is
1698 * HRTIMER_CB_IRQSAFE_UNLOCKED (hrtimer_sleeper). The
1699 * enqueue code does not move them to the soft irq
1700 * pending list for performance/latency reasons, but
1701 * in the migration state, we need to do that
1702 * otherwise we end up with a stale timer.
1703 */
1704 if (timer->state == HRTIMER_STATE_MIGRATE) {
1705 timer->state = HRTIMER_STATE_PENDING;
1706 list_add_tail(&timer->cb_entry,
1707 &new_base->cpu_base->cb_pending);
1708 raise = 1;
1709 }
1710#endif
1711 /* Clear the migration state bit */
1712 timer->state &= ~HRTIMER_STATE_MIGRATE;
1713 }
1714 return raise;
1715}
1716
1717#ifdef CONFIG_HIGH_RES_TIMERS
1718static int migrate_hrtimer_pending(struct hrtimer_cpu_base *old_base,
1719 struct hrtimer_cpu_base *new_base)
1720{
1721 struct hrtimer *timer;
1722 int raise = 0;
1723
1724 while (!list_empty(&old_base->cb_pending)) {
1725 timer = list_entry(old_base->cb_pending.next,
1726 struct hrtimer, cb_entry);
1727
1728 __remove_hrtimer(timer, timer->base, HRTIMER_STATE_PENDING, 0);
1729 timer->base = &new_base->clock_base[timer->base->index];
1730 list_add_tail(&timer->cb_entry, &new_base->cb_pending);
1731 raise = 1;
1732 }
1733 return raise;
1734}
1735#else
1736static int migrate_hrtimer_pending(struct hrtimer_cpu_base *old_base,
1737 struct hrtimer_cpu_base *new_base)
1738{
1739 return 0;
1740}
1741#endif
1742
1743static void migrate_hrtimers(int cpu)
1744{
1745 struct hrtimer_cpu_base *old_base, *new_base;
1746 int i, raise = 0;
1747
1748 BUG_ON(cpu_online(cpu));
1749 old_base = &per_cpu(hrtimer_bases, cpu);
1750 new_base = &get_cpu_var(hrtimer_bases);
1751
1752 tick_cancel_sched_timer(cpu);
1753 /*
1754 * The caller is globally serialized and nobody else
1755 * takes two locks at once, deadlock is not possible.
1756 */
1757 spin_lock_irq(&new_base->lock);
1758 spin_lock_nested(&old_base->lock, SINGLE_DEPTH_NESTING);
1759
1760 for (i = 0; i < HRTIMER_MAX_CLOCK_BASES; i++) {
1761 if (migrate_hrtimer_list(&old_base->clock_base[i],
1762 &new_base->clock_base[i], cpu))
1763 raise = 1;
1764 }
1765
1766 if (migrate_hrtimer_pending(old_base, new_base))
1767 raise = 1;
1768
1769 spin_unlock(&old_base->lock);
1770 spin_unlock_irq(&new_base->lock);
1771 put_cpu_var(hrtimer_bases);
1772
1773 if (raise)
1774 hrtimer_raise_softirq();
1775}
1776#endif /* CONFIG_HOTPLUG_CPU */
1777
1778static int __cpuinit hrtimer_cpu_notify(struct notifier_block *self,
1779 unsigned long action, void *hcpu)
1780{
1781 unsigned int cpu = (long)hcpu;
1782
1783 switch (action) {
1784
1785 case CPU_UP_PREPARE:
1786 case CPU_UP_PREPARE_FROZEN:
1787 init_hrtimers_cpu(cpu);
1788 break;
1789
1790#ifdef CONFIG_HOTPLUG_CPU
1791 case CPU_DEAD:
1792 case CPU_DEAD_FROZEN:
1793 clockevents_notify(CLOCK_EVT_NOTIFY_CPU_DEAD, &cpu);
1794 migrate_hrtimers(cpu);
1795 break;
1796#endif
1797
1798 default:
1799 break;
1800 }
1801
1802 return NOTIFY_OK;
1803}
1804
1805static struct notifier_block __cpuinitdata hrtimers_nb = {
1806 .notifier_call = hrtimer_cpu_notify,
1807};
1808
1809void __init hrtimers_init(void)
1810{
1811 hrtimer_cpu_notify(&hrtimers_nb, (unsigned long)CPU_UP_PREPARE,
1812 (void *)(long)smp_processor_id());
1813 register_cpu_notifier(&hrtimers_nb);
1814#ifdef CONFIG_HIGH_RES_TIMERS
1815 open_softirq(HRTIMER_SOFTIRQ, run_hrtimer_softirq);
1816#endif
1817}
1818
1819/**
1820 * schedule_hrtimeout_range - sleep until timeout
1821 * @expires: timeout value (ktime_t)
1822 * @delta: slack in expires timeout (ktime_t)
1823 * @mode: timer mode, HRTIMER_MODE_ABS or HRTIMER_MODE_REL
1824 *
1825 * Make the current task sleep until the given expiry time has
1826 * elapsed. The routine will return immediately unless
1827 * the current task state has been set (see set_current_state()).
1828 *
1829 * The @delta argument gives the kernel the freedom to schedule the
1830 * actual wakeup to a time that is both power and performance friendly.
1831 * The kernel give the normal best effort behavior for "@expires+@delta",
1832 * but may decide to fire the timer earlier, but no earlier than @expires.
1833 *
1834 * You can set the task state as follows -
1835 *
1836 * %TASK_UNINTERRUPTIBLE - at least @timeout time is guaranteed to
1837 * pass before the routine returns.
1838 *
1839 * %TASK_INTERRUPTIBLE - the routine may return early if a signal is
1840 * delivered to the current task.
1841 *
1842 * The current task state is guaranteed to be TASK_RUNNING when this
1843 * routine returns.
1844 *
1845 * Returns 0 when the timer has expired otherwise -EINTR
1846 */
1847int __sched schedule_hrtimeout_range(ktime_t *expires, unsigned long delta,
1848 const enum hrtimer_mode mode)
1849{
1850 struct hrtimer_sleeper t;
1851
1852 /*
1853 * Optimize when a zero timeout value is given. It does not
1854 * matter whether this is an absolute or a relative time.
1855 */
1856 if (expires && !expires->tv64) {
1857 __set_current_state(TASK_RUNNING);
1858 return 0;
1859 }
1860
1861 /*
1862 * A NULL parameter means "inifinte"
1863 */
1864 if (!expires) {
1865 schedule();
1866 __set_current_state(TASK_RUNNING);
1867 return -EINTR;
1868 }
1869
1870 hrtimer_init_on_stack(&t.timer, CLOCK_MONOTONIC, mode);
1871 hrtimer_set_expires_range_ns(&t.timer, *expires, delta);
1872
1873 hrtimer_init_sleeper(&t, current);
1874
1875 hrtimer_start_expires(&t.timer, mode);
1876 if (!hrtimer_active(&t.timer))
1877 t.task = NULL;
1878
1879 if (likely(t.task))
1880 schedule();
1881
1882 hrtimer_cancel(&t.timer);
1883 destroy_hrtimer_on_stack(&t.timer);
1884
1885 __set_current_state(TASK_RUNNING);
1886
1887 return !t.task ? 0 : -EINTR;
1888}
1889EXPORT_SYMBOL_GPL(schedule_hrtimeout_range);
1890
1891/**
1892 * schedule_hrtimeout - sleep until timeout
1893 * @expires: timeout value (ktime_t)
1894 * @mode: timer mode, HRTIMER_MODE_ABS or HRTIMER_MODE_REL
1895 *
1896 * Make the current task sleep until the given expiry time has
1897 * elapsed. The routine will return immediately unless
1898 * the current task state has been set (see set_current_state()).
1899 *
1900 * You can set the task state as follows -
1901 *
1902 * %TASK_UNINTERRUPTIBLE - at least @timeout time is guaranteed to
1903 * pass before the routine returns.
1904 *
1905 * %TASK_INTERRUPTIBLE - the routine may return early if a signal is
1906 * delivered to the current task.
1907 *
1908 * The current task state is guaranteed to be TASK_RUNNING when this
1909 * routine returns.
1910 *
1911 * Returns 0 when the timer has expired otherwise -EINTR
1912 */
1913int __sched schedule_hrtimeout(ktime_t *expires,
1914 const enum hrtimer_mode mode)
1915{
1916 return schedule_hrtimeout_range(expires, 0, mode);
1917}
1918EXPORT_SYMBOL_GPL(schedule_hrtimeout);