| 1 | |
| 2 | * kernel/sched.c
|
| 3 | *
|
| 4 | * Kernel scheduler and related syscalls
|
| 5 | *
|
| 6 | * Copyright (C) 1991-2002 Linus Torvalds
|
| 7 | *
|
| 8 | * 1996-12-23 Modified by Dave Grothe to fix bugs in semaphores and
|
| 9 | * make semaphores SMP safe
|
| 10 | * 1998-11-19 Implemented schedule_timeout() and related stuff
|
| 11 | * by Andrea Arcangeli
|
| 12 | * 2002-01-04 New ultra-scalable O(1) scheduler by Ingo Molnar:
|
| 13 | * hybrid priority-list and round-robin design with
|
| 14 | * an array-switch method of distributing timeslices
|
| 15 | * and per-CPU runqueues. Cleanups and useful suggestions
|
| 16 | * by Davide Libenzi, preemptible kernel bits by Robert Love.
|
| 17 | * 2003-09-03 Interactivity tuning by Con Kolivas.
|
| 18 | * 2004-04-02 Scheduler domains code by Nick Piggin
|
| 19 |
|
| 20 |
|
| 21 | #include <linux/mm.h>
|
| 22 | #include <linux/module.h>
|
| 23 | #include <linux/nmi.h>
|
| 24 | #include <linux/init.h>
|
| 25 | #include <asm/uaccess.h>
|
| 26 | #include <linux/highmem.h>
|
| 27 | #include <linux/smp_lock.h>
|
| 28 | #include <asm/mmu_context.h>
|
| 29 | #include <linux/interrupt.h>
|
| 30 | #include <linux/completion.h>
|
| 31 | #include <linux/kernel_stat.h>
|
| 32 | #include <linux/security.h>
|
| 33 | #include <linux/notifier.h>
|
| 34 | #include <linux/profile.h>
|
| 35 | #include <linux/suspend.h>
|
| 36 | #include <linux/blkdev.h>
|
| 37 | #include <linux/delay.h>
|
| 38 | #include <linux/smp.h>
|
| 39 | #include <linux/threads.h>
|
| 40 | #include <linux/timer.h>
|
| 41 | #include <linux/rcupdate.h>
|
| 42 | #include <linux/cpu.h>
|
| 43 | #include <linux/cpuset.h>
|
| 44 | #include <linux/percpu.h>
|
| 45 | #include <linux/kthread.h>
|
| 46 | #include <linux/seq_file.h>
|
| 47 | #include <linux/syscalls.h>
|
| 48 | #include <linux/times.h>
|
| 49 | #include <linux/acct.h>
|
| 50 | #include <asm/tlb.h>
|
| 51 |
|
| 52 | #include <asm/unistd.h>
|
| 53 |
|
| 54 | |
| 55 | * Convert user-nice values [ -20 ... 0 ... 19 ]
|
| 56 | * to static priority [ MAX_RT_PRIO..MAX_PRIO-1 ],
|
| 57 | * and back.
|
| 58 |
|
| 59 | #define NICE_TO_PRIO(nice) (MAX_RT_PRIO + (nice) + 20)
|
| 60 | #define PRIO_TO_NICE(prio) ((prio) - MAX_RT_PRIO - 20)
|
| 61 | #define TASK_NICE(p) PRIO_TO_NICE((p)->static_prio)
|
| 62 |
|
| 63 | |
| 64 | * 'User priority' is the nice value converted to something we
|
| 65 | * can work with better when scaling various scheduler parameters,
|
| 66 | * it's a [ 0 ... 39 ] range.
|
| 67 |
|
| 68 | #define USER_PRIO(p) ((p)-MAX_RT_PRIO)
|
| 69 | #define TASK_USER_PRIO(p) USER_PRIO((p)->static_prio)
|
| 70 | #define MAX_USER_PRIO (USER_PRIO(MAX_PRIO))
|
| 71 |
|
| 72 | |
| 73 | * Some helpers for converting nanosecond timing to jiffy resolution
|
| 74 |
|
| 75 | #define NS_TO_JIFFIES(TIME) ((TIME) / (1000000000 / HZ))
|
| 76 | #define JIFFIES_TO_NS(TIME) ((TIME) * (1000000000 / HZ))
|
| 77 |
|
| 78 | |
| 79 | * These are the 'tuning knobs' of the scheduler:
|
| 80 | *
|
| 81 | * Minimum timeslice is 5 msecs (or 1 jiffy, whichever is larger),
|
| 82 | * default timeslice is 100 msecs, maximum timeslice is 800 msecs.
|
| 83 | * Timeslices get refilled after they expire.
|
| 84 |
|
| 85 | #define MIN_TIMESLICE max(5 * HZ / 1000, 1)
|
| 86 | #define DEF_TIMESLICE (100 * HZ / 1000)
|
| 87 | #define ON_RUNQUEUE_WEIGHT 30
|
| 88 | #define CHILD_PENALTY 95
|
| 89 | #define PARENT_PENALTY 100
|
| 90 | #define EXIT_WEIGHT 3
|
| 91 | #define PRIO_BONUS_RATIO 25
|
| 92 | #define MAX_BONUS (MAX_USER_PRIO * PRIO_BONUS_RATIO / 100)
|
| 93 | #define INTERACTIVE_DELTA 2
|
| 94 | #define MAX_SLEEP_AVG (DEF_TIMESLICE * MAX_BONUS)
|
| 95 | #define STARVATION_LIMIT (MAX_SLEEP_AVG)
|
| 96 | #define NS_MAX_SLEEP_AVG (JIFFIES_TO_NS(MAX_SLEEP_AVG))
|
| 97 |
|
| 98 | |
| 99 | * If a task is 'interactive' then we reinsert it in the active
|
| 100 | * array after it has expired its current timeslice. (it will not
|
| 101 | * continue to run immediately, it will still roundrobin with
|
| 102 | * other interactive tasks.)
|
| 103 | *
|
| 104 | * This part scales the interactivity limit depending on niceness.
|
| 105 | *
|
| 106 | * We scale it linearly, offset by the INTERACTIVE_DELTA delta.
|
| 107 | * Here are a few examples of different nice levels:
|
| 108 | *
|
| 109 | * TASK_INTERACTIVE(-20): [1,1,1,1,1,1,1,1,1,0,0]
|
| 110 | * TASK_INTERACTIVE(-10): [1,1,1,1,1,1,1,0,0,0,0]
|
| 111 | * TASK_INTERACTIVE( 0): [1,1,1,1,0,0,0,0,0,0,0]
|
| 112 | * TASK_INTERACTIVE( 10): [1,1,0,0,0,0,0,0,0,0,0]
|
| 113 | * TASK_INTERACTIVE( 19): [0,0,0,0,0,0,0,0,0,0,0]
|
| 114 | *
|
| 115 | * (the X axis represents the possible -5 ... 0 ... +5 dynamic
|
| 116 | * priority range a task can explore, a value of '1' means the
|
| 117 | * task is rated interactive.)
|
| 118 | *
|
| 119 | * Ie. nice +19 tasks can never get 'interactive' enough to be
|
| 120 | * reinserted into the active array. And only heavily CPU-hog nice -20
|
| 121 | * tasks will be expired. Default nice 0 tasks are somewhere between,
|
| 122 | * it takes some effort for them to get interactive, but it's not
|
| 123 | * too hard.
|
| 124 |
|
| 125 |
|
| 126 | #define CURRENT_BONUS(p) \
|
| 127 | (NS_TO_JIFFIES((p)->sleep_avg) * MAX_BONUS / \
|
| 128 | MAX_SLEEP_AVG)
|
| 129 |
|
| 130 | #define GRANULARITY (10 * HZ / 1000 ? : 1)
|
| 131 |
|
| 132 | #ifdef CONFIG_SMP
|
| 133 | #define TIMESLICE_GRANULARITY(p) (GRANULARITY * \
|
| 134 | (1 << (((MAX_BONUS - CURRENT_BONUS(p)) ? : 1) - 1)) * \
|
| 135 | num_online_cpus())
|
| 136 | #else
|
| 137 | #define TIMESLICE_GRANULARITY(p) (GRANULARITY * \
|
| 138 | (1 << (((MAX_BONUS - CURRENT_BONUS(p)) ? : 1) - 1)))
|
| 139 | #endif
|
| 140 |
|
| 141 | #define SCALE(v1,v1_max,v2_max) \
|
| 142 | (v1) * (v2_max) / (v1_max)
|
| 143 |
|
| 144 | #define DELTA(p) \
|
| 145 | (SCALE(TASK_NICE(p), 40, MAX_BONUS) + INTERACTIVE_DELTA)
|
| 146 |
|
| 147 | #define TASK_INTERACTIVE(p) \
|
| 148 | ((p)->prio <= (p)->static_prio - DELTA(p))
|
| 149 |
|
| 150 | #define INTERACTIVE_SLEEP(p) \
|
| 151 | (JIFFIES_TO_NS(MAX_SLEEP_AVG * \
|
| 152 | (MAX_BONUS / 2 + DELTA((p)) + 1) / MAX_BONUS - 1))
|
| 153 |
|
| 154 | #define TASK_PREEMPTS_CURR(p, rq) \
|
| 155 | ((p)->prio < (rq)->curr->prio)
|
| 156 |
|
| 157 | |
| 158 | * task_timeslice() scales user-nice values [ -20 ... 0 ... 19 ]
|
| 159 | * to time slice values: [800ms ... 100ms ... 5ms]
|
| 160 | *
|
| 161 | * The higher a thread's priority, the bigger timeslices
|
| 162 | * it gets during one round of execution. But even the lowest
|
| 163 | * priority thread gets MIN_TIMESLICE worth of execution time.
|
| 164 |
|
| 165 |
|
| 166 | #define SCALE_PRIO(x, prio) \
|
| 167 | max(x * (MAX_PRIO - prio) / (MAX_USER_PRIO/2), MIN_TIMESLICE)
|
| 168 |
|
| 169 | static unsigned int task_timeslice(task_t *p)
|
| 170 | {
|
| 171 | if (p->static_prio < NICE_TO_PRIO(0))
|
| 172 | return SCALE_PRIO(DEF_TIMESLICE*4, p->static_prio);
|
| 173 | else
|
| 174 | return SCALE_PRIO(DEF_TIMESLICE, p->static_prio);
|
| 175 | }
|
| 176 | #define task_hot(p, now, sd) ((long long) ((now) - (p)->last_ran) \
|
| 177 | < (long long) (sd)->cache_hot_time)
|
| 178 |
|
| 179 | |
| 180 | * These are the runqueue data structures:
|
| 181 |
|
| 182 |
|
| 183 | #define BITMAP_SIZE ((((MAX_PRIO+1+7)/8)+sizeof(long)-1)/sizeof(long))
|
| 184 |
|
| 185 | typedef struct runqueue runqueue_t;
|
| 186 |
|
| 187 | struct prio_array {
|
| 188 | unsigned int nr_active;
|
| 189 | unsigned long bitmap[BITMAP_SIZE];
|
| 190 | struct list_head queue[MAX_PRIO];
|
| 191 | };
|
| 192 |
|
| 193 | |
| 194 | * This is the main, per-CPU runqueue data structure.
|
| 195 | *
|
| 196 | * Locking rule: those places that want to lock multiple runqueues
|
| 197 | * (such as the load balancing or the thread migration code), lock
|
| 198 | * acquire operations must be ordered by ascending &runqueue.
|
| 199 |
|
| 200 | struct runqueue {
|
| 201 | spinlock_t lock;
|
| 202 |
|
| 203 | |
| 204 | * nr_running and cpu_load should be in the same cacheline because
|
| 205 | * remote CPUs use both these fields when doing load calculation.
|
| 206 |
|
| 207 | unsigned long nr_running;
|
| 208 | #ifdef CONFIG_SMP
|
| 209 | unsigned long cpu_load[3];
|
| 210 | #endif
|
| 211 | unsigned long long nr_switches;
|
| 212 |
|
| 213 | |
| 214 | * This is part of a global counter where only the total sum
|
| 215 | * over all CPUs matters. A task can increase this counter on
|
| 216 | * one CPU and if it got migrated afterwards it may decrease
|
| 217 | * it on another CPU. Always updated under the runqueue lock:
|
| 218 |
|
| 219 | unsigned long nr_uninterruptible;
|
| 220 |
|
| 221 | unsigned long expired_timestamp;
|
| 222 | unsigned long long timestamp_last_tick;
|
| 223 | task_t *curr, *idle;
|
| 224 | struct mm_struct *prev_mm;
|
| 225 | prio_array_t *active, *expired, arrays[2];
|
| 226 | int best_expired_prio;
|
| 227 | atomic_t nr_iowait;
|
| 228 |
|
| 229 | #ifdef CONFIG_SMP
|
| 230 | struct sched_domain *sd;
|
| 231 |
|
| 232 |
|
| 233 | int active_balance;
|
| 234 | int push_cpu;
|
| 235 |
|
| 236 | task_t *migration_thread;
|
| 237 | struct list_head migration_queue;
|
| 238 | #endif
|
| 239 |
|
| 240 | #ifdef CONFIG_SCHEDSTATS
|
| 241 |
|
| 242 | struct sched_info rq_sched_info;
|
| 243 |
|
| 244 |
|
| 245 | unsigned long yld_exp_empty;
|
| 246 | unsigned long yld_act_empty;
|
| 247 | unsigned long yld_both_empty;
|
| 248 | unsigned long yld_cnt;
|
| 249 |
|
| 250 |
|
| 251 | unsigned long sched_switch;
|
| 252 | unsigned long sched_cnt;
|
| 253 | unsigned long sched_goidle;
|
| 254 |
|
| 255 |
|
| 256 | unsigned long ttwu_cnt;
|
| 257 | unsigned long ttwu_local;
|
| 258 | #endif
|
| 259 | };
|
| 260 |
|
| 261 | static DEFINE_PER_CPU(struct runqueue, runqueues);
|
| 262 |
|
| 263 | |
| 264 | * The domain tree (rq->sd) is protected by RCU's quiescent state transition.
|
| 265 | * See detach_destroy_domains: synchronize_sched for details.
|
| 266 | *
|
| 267 | * The domain tree of any CPU may only be accessed from within
|
| 268 | * preempt-disabled sections.
|
| 269 |
|
| 270 | #define for_each_domain(cpu, domain) \
|
| 271 | for (domain = rcu_dereference(cpu_rq(cpu)->sd); domain; domain = domain->parent)
|
| 272 |
|
| 273 | #define cpu_rq(cpu) (&per_cpu(runqueues, (cpu)))
|
| 274 | #define this_rq() (&__get_cpu_var(runqueues))
|
| 275 | #define task_rq(p) cpu_rq(task_cpu(p))
|
| 276 | #define cpu_curr(cpu) (cpu_rq(cpu)->curr)
|
| 277 |
|
| 278 | #ifndef prepare_arch_switch
|
| 279 | # define prepare_arch_switch(next) do { } while (0)
|
| 280 | #endif
|
| 281 | #ifndef finish_arch_switch
|
| 282 | # define finish_arch_switch(prev) do { } while (0)
|
| 283 | #endif
|
| 284 |
|
| 285 | #ifndef __ARCH_WANT_UNLOCKED_CTXSW
|
| 286 | static inline int task_running(runqueue_t *rq, task_t *p)
|
| 287 | {
|
| 288 | return rq->curr == p;
|
| 289 | }
|
| 290 |
|
| 291 | static inline void prepare_lock_switch(runqueue_t *rq, task_t *next)
|
| 292 | {
|
| 293 | }
|
| 294 |
|
| 295 | static inline void finish_lock_switch(runqueue_t *rq, task_t *prev)
|
| 296 | {
|
| 297 | spin_unlock_irq(&rq->lock);
|
| 298 | }
|
| 299 |
|
| 300 | #else
|
| 301 | static inline int task_running(runqueue_t *rq, task_t *p)
|
| 302 | {
|
| 303 | #ifdef CONFIG_SMP
|
| 304 | return p->oncpu;
|
| 305 | #else
|
| 306 | return rq->curr == p;
|
| 307 | #endif
|
| 308 | }
|
| 309 |
|
| 310 | static inline void prepare_lock_switch(runqueue_t *rq, task_t *next)
|
| 311 | {
|
| 312 | #ifdef CONFIG_SMP
|
| 313 | |
| 314 | * We can optimise this out completely for !SMP, because the
|
| 315 | * SMP rebalancing from interrupt is the only thing that cares
|
| 316 | * here.
|
| 317 |
|
| 318 | next->oncpu = 1;
|
| 319 | #endif
|
| 320 | #ifdef __ARCH_WANT_INTERRUPTS_ON_CTXSW
|
| 321 | spin_unlock_irq(&rq->lock);
|
| 322 | #else
|
| 323 | spin_unlock(&rq->lock);
|
| 324 | #endif
|
| 325 | }
|
| 326 |
|
| 327 | static inline void finish_lock_switch(runqueue_t *rq, task_t *prev)
|
| 328 | {
|
| 329 | #ifdef CONFIG_SMP
|
| 330 | |
| 331 | * After ->oncpu is cleared, the task can be moved to a different CPU.
|
| 332 | * We must ensure this doesn't happen until the switch is completely
|
| 333 | * finished.
|
| 334 |
|
| 335 | smp_wmb();
|
| 336 | prev->oncpu = 0;
|
| 337 | #endif
|
| 338 | #ifndef __ARCH_WANT_INTERRUPTS_ON_CTXSW
|
| 339 | local_irq_enable();
|
| 340 | #endif
|
| 341 | }
|
| 342 | #endif
|
| 343 |
|
| 344 | |
| 345 | * task_rq_lock - lock the runqueue a given task resides on and disable
|
| 346 | * interrupts. Note the ordering: we can safely lookup the task_rq without
|
| 347 | * explicitly disabling preemption.
|
| 348 |
|
| 349 | static inline runqueue_t *task_rq_lock(task_t *p, unsigned long *flags)
|
| 350 | __acquires(rq->lock)
|
| 351 | {
|
| 352 | struct runqueue *rq;
|
| 353 |
|
| 354 | repeat_lock_task:
|
| 355 | local_irq_save(*flags);
|
| 356 | rq = task_rq(p);
|
| 357 | spin_lock(&rq->lock);
|
| 358 | if (unlikely(rq != task_rq(p))) {
|
| 359 | spin_unlock_irqrestore(&rq->lock, *flags);
|
| 360 | goto repeat_lock_task;
|
| 361 | }
|
| 362 | return rq;
|
| 363 | }
|
| 364 |
|
| 365 | static inline void task_rq_unlock(runqueue_t *rq, unsigned long *flags)
|
| 366 | __releases(rq->lock)
|
| 367 | {
|
| 368 | spin_unlock_irqrestore(&rq->lock, *flags);
|
| 369 | }
|
| 370 |
|
| 371 | #ifdef CONFIG_SCHEDSTATS
|
| 372 | |
| 373 | * bump this up when changing the output format or the meaning of an existing
|
| 374 | * format, so that tools can adapt (or abort)
|
| 375 |
|
| 376 | #define SCHEDSTAT_VERSION 12
|
| 377 |
|
| 378 | static int show_schedstat(struct seq_file *seq, void *v)
|
| 379 | {
|
| 380 | int cpu;
|
| 381 |
|
| 382 | seq_printf(seq, "version %d\n", SCHEDSTAT_VERSION);
|
| 383 | seq_printf(seq, "timestamp %lu\n", jiffies);
|
| 384 | for_each_online_cpu(cpu) {
|
| 385 | runqueue_t *rq = cpu_rq(cpu);
|
| 386 | #ifdef CONFIG_SMP
|
| 387 | struct sched_domain *sd;
|
| 388 | int dcnt = 0;
|
| 389 | #endif
|
| 390 |
|
| 391 |
|
| 392 | seq_printf(seq,
|
| 393 | "cpu%d %lu %lu %lu %lu %lu %lu %lu %lu %lu %lu %lu %lu",
|
| 394 | cpu, rq->yld_both_empty,
|
| 395 | rq->yld_act_empty, rq->yld_exp_empty, rq->yld_cnt,
|
| 396 | rq->sched_switch, rq->sched_cnt, rq->sched_goidle,
|
| 397 | rq->ttwu_cnt, rq->ttwu_local,
|
| 398 | rq->rq_sched_info.cpu_time,
|
| 399 | rq->rq_sched_info.run_delay, rq->rq_sched_info.pcnt);
|
| 400 |
|
| 401 | seq_printf(seq, "\n");
|
| 402 |
|
| 403 | #ifdef CONFIG_SMP
|
| 404 |
|
| 405 | preempt_disable();
|
| 406 | for_each_domain(cpu, sd) {
|
| 407 | enum idle_type itype;
|
| 408 | char mask_str[NR_CPUS];
|
| 409 |
|
| 410 | cpumask_scnprintf(mask_str, NR_CPUS, sd->span);
|
| 411 | seq_printf(seq, "domain%d %s", dcnt++, mask_str);
|
| 412 | for (itype = SCHED_IDLE; itype < MAX_IDLE_TYPES;
|
| 413 | itype++) {
|
| 414 | seq_printf(seq, " %lu %lu %lu %lu %lu %lu %lu %lu",
|
| 415 | sd->lb_cnt[itype],
|
| 416 | sd->lb_balanced[itype],
|
| 417 | sd->lb_failed[itype],
|
| 418 | sd->lb_imbalance[itype],
|
| 419 | sd->lb_gained[itype],
|
| 420 | sd->lb_hot_gained[itype],
|
| 421 | sd->lb_nobusyq[itype],
|
| 422 | sd->lb_nobusyg[itype]);
|
| 423 | }
|
| 424 | seq_printf(seq, " %lu %lu %lu %lu %lu %lu %lu %lu %lu %lu %lu %lu\n",
|
| 425 | sd->alb_cnt, sd->alb_failed, sd->alb_pushed,
|
| 426 | sd->sbe_cnt, sd->sbe_balanced, sd->sbe_pushed,
|
| 427 | sd->sbf_cnt, sd->sbf_balanced, sd->sbf_pushed,
|
| 428 | sd->ttwu_wake_remote, sd->ttwu_move_affine, sd->ttwu_move_balance);
|
| 429 | }
|
| 430 | preempt_enable();
|
| 431 | #endif
|
| 432 | }
|
| 433 | return 0;
|
| 434 | }
|
| 435 |
|
| 436 | static int schedstat_open(struct inode *inode, struct file *file)
|
| 437 | {
|
| 438 | unsigned int size = PAGE_SIZE * (1 + num_online_cpus() / 32);
|
| 439 | char *buf = kmalloc(size, GFP_KERNEL);
|
| 440 | struct seq_file *m;
|
| 441 | int res;
|
| 442 |
|
| 443 | if (!buf)
|
| 444 | return -ENOMEM;
|
| 445 | res = single_open(file, show_schedstat, NULL);
|
| 446 | if (!res) {
|
| 447 | m = file->private_data;
|
| 448 | m->buf = buf;
|
| 449 | m->size = size;
|
| 450 | } else
|
| 451 | kfree(buf);
|
| 452 | return res;
|
| 453 | }
|
| 454 |
|
| 455 | struct file_operations proc_schedstat_operations = {
|
| 456 | .open = schedstat_open,
|
| 457 | .read = seq_read,
|
| 458 | .llseek = seq_lseek,
|
| 459 | .release = single_release,
|
| 460 | };
|
| 461 |
|
| 462 | # define schedstat_inc(rq, field) do { (rq)->field++; } while (0)
|
| 463 | # define schedstat_add(rq, field, amt) do { (rq)->field += (amt); } while (0)
|
| 464 | #else
|
| 465 | # define schedstat_inc(rq, field) do { } while (0)
|
| 466 | # define schedstat_add(rq, field, amt) do { } while (0)
|
| 467 | #endif
|
| 468 |
|
| 469 | |
| 470 | * rq_lock - lock a given runqueue and disable interrupts.
|
| 471 |
|
| 472 | static inline runqueue_t *this_rq_lock(void)
|
| 473 | __acquires(rq->lock)
|
| 474 | {
|
| 475 | runqueue_t *rq;
|
| 476 |
|
| 477 | local_irq_disable();
|
| 478 | rq = this_rq();
|
| 479 | spin_lock(&rq->lock);
|
| 480 |
|
| 481 | return rq;
|
| 482 | }
|
| 483 |
|
| 484 | #ifdef CONFIG_SCHEDSTATS
|
| 485 | |
| 486 | * Called when a process is dequeued from the active array and given
|
| 487 | * the cpu. We should note that with the exception of interactive
|
| 488 | * tasks, the expired queue will become the active queue after the active
|
| 489 | * queue is empty, without explicitly dequeuing and requeuing tasks in the
|
| 490 | * expired queue. (Interactive tasks may be requeued directly to the
|
| 491 | * active queue, thus delaying tasks in the expired queue from running;
|
| 492 | * see scheduler_tick()).
|
| 493 | *
|
| 494 | * This function is only called from sched_info_arrive(), rather than
|
| 495 | * dequeue_task(). Even though a task may be queued and dequeued multiple
|
| 496 | * times as it is shuffled about, we're really interested in knowing how
|
| 497 | * long it was from the *first* time it was queued to the time that it
|
| 498 | * finally hit a cpu.
|
| 499 |
|
| 500 | static inline void sched_info_dequeued(task_t *t)
|
| 501 | {
|
| 502 | t->sched_info.last_queued = 0;
|
| 503 | }
|
| 504 |
|
| 505 | |
| 506 | * Called when a task finally hits the cpu. We can now calculate how
|
| 507 | * long it was waiting to run. We also note when it began so that we
|
| 508 | * can keep stats on how long its timeslice is.
|
| 509 |
|
| 510 | static inline void sched_info_arrive(task_t *t)
|
| 511 | {
|
| 512 | unsigned long now = jiffies, diff = 0;
|
| 513 | struct runqueue *rq = task_rq(t);
|
| 514 |
|
| 515 | if (t->sched_info.last_queued)
|
| 516 | diff = now - t->sched_info.last_queued;
|
| 517 | sched_info_dequeued(t);
|
| 518 | t->sched_info.run_delay += diff;
|
| 519 | t->sched_info.last_arrival = now;
|
| 520 | t->sched_info.pcnt++;
|
| 521 |
|
| 522 | if (!rq)
|
| 523 | return;
|
| 524 |
|
| 525 | rq->rq_sched_info.run_delay += diff;
|
| 526 | rq->rq_sched_info.pcnt++;
|
| 527 | }
|
| 528 |
|
| 529 | |
| 530 | * Called when a process is queued into either the active or expired
|
| 531 | * array. The time is noted and later used to determine how long we
|
| 532 | * had to wait for us to reach the cpu. Since the expired queue will
|
| 533 | * become the active queue after active queue is empty, without dequeuing
|
| 534 | * and requeuing any tasks, we are interested in queuing to either. It
|
| 535 | * is unusual but not impossible for tasks to be dequeued and immediately
|
| 536 | * requeued in the same or another array: this can happen in sched_yield(),
|
| 537 | * set_user_nice(), and even load_balance() as it moves tasks from runqueue
|
| 538 | * to runqueue.
|
| 539 | *
|
| 540 | * This function is only called from enqueue_task(), but also only updates
|
| 541 | * the timestamp if it is already not set. It's assumed that
|
| 542 | * sched_info_dequeued() will clear that stamp when appropriate.
|
| 543 |
|
| 544 | static inline void sched_info_queued(task_t *t)
|
| 545 | {
|
| 546 | if (!t->sched_info.last_queued)
|
| 547 | t->sched_info.last_queued = jiffies;
|
| 548 | }
|
| 549 |
|
| 550 | |
| 551 | * Called when a process ceases being the active-running process, either
|
| 552 | * voluntarily or involuntarily. Now we can calculate how long we ran.
|
| 553 |
|
| 554 | static inline void sched_info_depart(task_t *t)
|
| 555 | {
|
| 556 | struct runqueue *rq = task_rq(t);
|
| 557 | unsigned long diff = jiffies - t->sched_info.last_arrival;
|
| 558 |
|
| 559 | t->sched_info.cpu_time += diff;
|
| 560 |
|
| 561 | if (rq)
|
| 562 | rq->rq_sched_info.cpu_time += diff;
|
| 563 | }
|
| 564 |
|
| 565 | |
| 566 | * Called when tasks are switched involuntarily due, typically, to expiring
|
| 567 | * their time slice. (This may also be called when switching to or from
|
| 568 | * the idle task.) We are only called when prev != next.
|
| 569 |
|
| 570 | static inline void sched_info_switch(task_t *prev, task_t *next)
|
| 571 | {
|
| 572 | struct runqueue *rq = task_rq(prev);
|
| 573 |
|
| 574 | |
| 575 | * prev now departs the cpu. It's not interesting to record
|
| 576 | * stats about how efficient we were at scheduling the idle
|
| 577 | * process, however.
|
| 578 |
|
| 579 | if (prev != rq->idle)
|
| 580 | sched_info_depart(prev);
|
| 581 |
|
| 582 | if (next != rq->idle)
|
| 583 | sched_info_arrive(next);
|
| 584 | }
|
| 585 | #else
|
| 586 | #define sched_info_queued(t) do { } while (0)
|
| 587 | #define sched_info_switch(t, next) do { } while (0)
|
| 588 | #endif
|
| 589 |
|
| 590 | |
| 591 | * Adding/removing a task to/from a priority array:
|
| 592 |
|
| 593 | static void dequeue_task(struct task_struct *p, prio_array_t *array)
|
| 594 | {
|
| 595 | array->nr_active--;
|
| 596 | list_del(&p->run_list);
|
| 597 | if (list_empty(array->queue + p->prio))
|
| 598 | __clear_bit(p->prio, array->bitmap);
|
| 599 | }
|
| 600 |
|
| 601 | static void enqueue_task(struct task_struct *p, prio_array_t *array)
|
| 602 | {
|
| 603 | sched_info_queued(p);
|
| 604 | list_add_tail(&p->run_list, array->queue + p->prio);
|
| 605 | __set_bit(p->prio, array->bitmap);
|
| 606 | array->nr_active++;
|
| 607 | p->array = array;
|
| 608 | }
|
| 609 |
|
| 610 | |
| 611 | * Put task to the end of the run list without the overhead of dequeue
|
| 612 | * followed by enqueue.
|
| 613 |
|
| 614 | static void requeue_task(struct task_struct *p, prio_array_t *array)
|
| 615 | {
|
| 616 | list_move_tail(&p->run_list, array->queue + p->prio);
|
| 617 | }
|
| 618 |
|
| 619 | static inline void enqueue_task_head(struct task_struct *p, prio_array_t *array)
|
| 620 | {
|
| 621 | list_add(&p->run_list, array->queue + p->prio);
|
| 622 | __set_bit(p->prio, array->bitmap);
|
| 623 | array->nr_active++;
|
| 624 | p->array = array;
|
| 625 | }
|
| 626 |
|
| 627 | |
| 628 | * effective_prio - return the priority that is based on the static
|
| 629 | * priority but is modified by bonuses/penalties.
|
| 630 | *
|
| 631 | * We scale the actual sleep average [0 .... MAX_SLEEP_AVG]
|
| 632 | * into the -5 ... 0 ... +5 bonus/penalty range.
|
| 633 | *
|
| 634 | * We use 25% of the full 0...39 priority range so that:
|
| 635 | *
|
| 636 | * 1) nice +19 interactive tasks do not preempt nice 0 CPU hogs.
|
| 637 | * 2) nice -20 CPU hogs do not get preempted by nice 0 tasks.
|
| 638 | *
|
| 639 | * Both properties are important to certain workloads.
|
| 640 |
|
| 641 | static int effective_prio(task_t *p)
|
| 642 | {
|
| 643 | int bonus, prio;
|
| 644 |
|
| 645 | if (rt_task(p))
|
| 646 | return p->prio;
|
| 647 |
|
| 648 | bonus = CURRENT_BONUS(p) - MAX_BONUS / 2;
|
| 649 |
|
| 650 | prio = p->static_prio - bonus;
|
| 651 | if (prio < MAX_RT_PRIO)
|
| 652 | prio = MAX_RT_PRIO;
|
| 653 | if (prio > MAX_PRIO-1)
|
| 654 | prio = MAX_PRIO-1;
|
| 655 | return prio;
|
| 656 | }
|
| 657 |
|
| 658 | |
| 659 | * __activate_task - move a task to the runqueue.
|
| 660 |
|
| 661 | static inline void __activate_task(task_t *p, runqueue_t *rq)
|
| 662 | {
|
| 663 | enqueue_task(p, rq->active);
|
| 664 | rq->nr_running++;
|
| 665 | }
|
| 666 |
|
| 667 | |
| 668 | * __activate_idle_task - move idle task to the _front_ of runqueue.
|
| 669 |
|
| 670 | static inline void __activate_idle_task(task_t *p, runqueue_t *rq)
|
| 671 | {
|
| 672 | enqueue_task_head(p, rq->active);
|
| 673 | rq->nr_running++;
|
| 674 | }
|
| 675 |
|
| 676 | static int recalc_task_prio(task_t *p, unsigned long long now)
|
| 677 | {
|
| 678 |
|
| 679 | unsigned long long __sleep_time = now - p->timestamp;
|
| 680 | unsigned long sleep_time;
|
| 681 |
|
| 682 | if (__sleep_time > NS_MAX_SLEEP_AVG)
|
| 683 | sleep_time = NS_MAX_SLEEP_AVG;
|
| 684 | else
|
| 685 | sleep_time = (unsigned long)__sleep_time;
|
| 686 |
|
| 687 | if (likely(sleep_time > 0)) {
|
| 688 | |
| 689 | * User tasks that sleep a long time are categorised as
|
| 690 | * idle and will get just interactive status to stay active &
|
| 691 | * prevent them suddenly becoming cpu hogs and starving
|
| 692 | * other processes.
|
| 693 |
|
| 694 | if (p->mm && p->activated != -1 &&
|
| 695 | sleep_time > INTERACTIVE_SLEEP(p)) {
|
| 696 | p->sleep_avg = JIFFIES_TO_NS(MAX_SLEEP_AVG -
|
| 697 | DEF_TIMESLICE);
|
| 698 | } else {
|
| 699 | |
| 700 | * The lower the sleep avg a task has the more
|
| 701 | * rapidly it will rise with sleep time.
|
| 702 |
|
| 703 | sleep_time *= (MAX_BONUS - CURRENT_BONUS(p)) ? : 1;
|
| 704 |
|
| 705 | |
| 706 | * Tasks waking from uninterruptible sleep are
|
| 707 | * limited in their sleep_avg rise as they
|
| 708 | * are likely to be waiting on I/O
|
| 709 |
|
| 710 | if (p->activated == -1 && p->mm) {
|
| 711 | if (p->sleep_avg >= INTERACTIVE_SLEEP(p))
|
| 712 | sleep_time = 0;
|
| 713 | else if (p->sleep_avg + sleep_time >=
|
| 714 | INTERACTIVE_SLEEP(p)) {
|
| 715 | p->sleep_avg = INTERACTIVE_SLEEP(p);
|
| 716 | sleep_time = 0;
|
| 717 | }
|
| 718 | }
|
| 719 |
|
| 720 | |
| 721 | * This code gives a bonus to interactive tasks.
|
| 722 | *
|
| 723 | * The boost works by updating the 'average sleep time'
|
| 724 | * value here, based on ->timestamp. The more time a
|
| 725 | * task spends sleeping, the higher the average gets -
|
| 726 | * and the higher the priority boost gets as well.
|
| 727 |
|
| 728 | p->sleep_avg += sleep_time;
|
| 729 |
|
| 730 | if (p->sleep_avg > NS_MAX_SLEEP_AVG)
|
| 731 | p->sleep_avg = NS_MAX_SLEEP_AVG;
|
| 732 | }
|
| 733 | }
|
| 734 |
|
| 735 | return effective_prio(p);
|
| 736 | }
|
| 737 |
|
| 738 | |
| 739 | * activate_task - move a task to the runqueue and do priority recalculation
|
| 740 | *
|
| 741 | * Update all the scheduling statistics stuff. (sleep average
|
| 742 | * calculation, priority modifiers, etc.)
|
| 743 |
|
| 744 | static void activate_task(task_t *p, runqueue_t *rq, int local)
|
| 745 | {
|
| 746 | unsigned long long now;
|
| 747 |
|
| 748 | now = sched_clock();
|
| 749 | #ifdef CONFIG_SMP
|
| 750 | if (!local) {
|
| 751 |
|
| 752 | runqueue_t *this_rq = this_rq();
|
| 753 | now = (now - this_rq->timestamp_last_tick)
|
| 754 | + rq->timestamp_last_tick;
|
| 755 | }
|
| 756 | #endif
|
| 757 |
|
| 758 | p->prio = recalc_task_prio(p, now);
|
| 759 |
|
| 760 | |
| 761 | * This checks to make sure it's not an uninterruptible task
|
| 762 | * that is now waking up.
|
| 763 |
|
| 764 | if (!p->activated) {
|
| 765 | |
| 766 | * Tasks which were woken up by interrupts (ie. hw events)
|
| 767 | * are most likely of interactive nature. So we give them
|
| 768 | * the credit of extending their sleep time to the period
|
| 769 | * of time they spend on the runqueue, waiting for execution
|
| 770 | * on a CPU, first time around:
|
| 771 |
|
| 772 | if (in_interrupt())
|
| 773 | p->activated = 2;
|
| 774 | else {
|
| 775 | |
| 776 | * Normal first-time wakeups get a credit too for
|
| 777 | * on-runqueue time, but it will be weighted down:
|
| 778 |
|
| 779 | p->activated = 1;
|
| 780 | }
|
| 781 | }
|
| 782 | p->timestamp = now;
|
| 783 |
|
| 784 | __activate_task(p, rq);
|
| 785 | }
|
| 786 |
|
| 787 | |
| 788 | * deactivate_task - remove a task from the runqueue.
|
| 789 |
|
| 790 | static void deactivate_task(struct task_struct *p, runqueue_t *rq)
|
| 791 | {
|
| 792 | rq->nr_running--;
|
| 793 | dequeue_task(p, p->array);
|
| 794 | p->array = NULL;
|
| 795 | }
|
| 796 |
|
| 797 | |
| 798 | * resched_task - mark a task 'to be rescheduled now'.
|
| 799 | *
|
| 800 | * On UP this means the setting of the need_resched flag, on SMP it
|
| 801 | * might also involve a cross-CPU call to trigger the scheduler on
|
| 802 | * the target CPU.
|
| 803 |
|
| 804 | #ifdef CONFIG_SMP
|
| 805 | static void resched_task(task_t *p)
|
| 806 | {
|
| 807 | int need_resched, nrpolling;
|
| 808 |
|
| 809 | assert_spin_locked(&task_rq(p)->lock);
|
| 810 |
|
| 811 |
|
| 812 | nrpolling = test_tsk_thread_flag(p,TIF_POLLING_NRFLAG);
|
| 813 | need_resched = test_and_set_tsk_thread_flag(p,TIF_NEED_RESCHED);
|
| 814 | nrpolling |= test_tsk_thread_flag(p,TIF_POLLING_NRFLAG);
|
| 815 |
|
| 816 | if (!need_resched && !nrpolling && (task_cpu(p) != smp_processor_id()))
|
| 817 | smp_send_reschedule(task_cpu(p));
|
| 818 | }
|
| 819 | #else
|
| 820 | static inline void resched_task(task_t *p)
|
| 821 | {
|
| 822 | set_tsk_need_resched(p);
|
| 823 | }
|
| 824 | #endif
|
| 825 |
|
| 826 | |
| 827 | * task_curr - is this task currently executing on a CPU?
|
| 828 | * @p: the task in question.
|
| 829 |
|
| 830 | inline int task_curr(const task_t *p)
|
| 831 | {
|
| 832 | return cpu_curr(task_cpu(p)) == p;
|
| 833 | }
|
| 834 |
|
| 835 | #ifdef CONFIG_SMP
|
| 836 | typedef struct {
|
| 837 | struct list_head list;
|
| 838 |
|
| 839 | task_t *task;
|
| 840 | int dest_cpu;
|
| 841 |
|
| 842 | struct completion done;
|
| 843 | } migration_req_t;
|
| 844 |
|
| 845 | |
| 846 | * The task's runqueue lock must be held.
|
| 847 | * Returns true if you have to wait for migration thread.
|
| 848 |
|
| 849 | static int migrate_task(task_t *p, int dest_cpu, migration_req_t *req)
|
| 850 | {
|
| 851 | runqueue_t *rq = task_rq(p);
|
| 852 |
|
| 853 | |
| 854 | * If the task is not on a runqueue (and not running), then
|
| 855 | * it is sufficient to simply update the task's cpu field.
|
| 856 |
|
| 857 | if (!p->array && !task_running(rq, p)) {
|
| 858 | set_task_cpu(p, dest_cpu);
|
| 859 | return 0;
|
| 860 | }
|
| 861 |
|
| 862 | init_completion(&req->done);
|
| 863 | req->task = p;
|
| 864 | req->dest_cpu = dest_cpu;
|
| 865 | list_add(&req->list, &rq->migration_queue);
|
| 866 | return 1;
|
| 867 | }
|
| 868 |
|
| 869 | |
| 870 | * wait_task_inactive - wait for a thread to unschedule.
|
| 871 | *
|
| 872 | * The caller must ensure that the task *will* unschedule sometime soon,
|
| 873 | * else this function might spin for a *long* time. This function can't
|
| 874 | * be called with interrupts off, or it may introduce deadlock with
|
| 875 | * smp_call_function() if an IPI is sent by the same process we are
|
| 876 | * waiting to become inactive.
|
| 877 |
|
| 878 | void wait_task_inactive(task_t * p)
|
| 879 | {
|
| 880 | unsigned long flags;
|
| 881 | runqueue_t *rq;
|
| 882 | int preempted;
|
| 883 |
|
| 884 | repeat:
|
| 885 | rq = task_rq_lock(p, &flags);
|
| 886 |
|
| 887 | if (unlikely(p->array || task_running(rq, p))) {
|
| 888 |
|
| 889 | preempted = !task_running(rq, p);
|
| 890 | task_rq_unlock(rq, &flags);
|
| 891 | cpu_relax();
|
| 892 | if (preempted)
|
| 893 | yield();
|
| 894 | goto repeat;
|
| 895 | }
|
| 896 | task_rq_unlock(rq, &flags);
|
| 897 | }
|
| 898 |
|
| 899 | |
| 900 | * kick_process - kick a running thread to enter/exit the kernel
|
| 901 | * @p: the to-be-kicked thread
|
| 902 | *
|
| 903 | * Cause a process which is running on another CPU to enter
|
| 904 | * kernel-mode, without any delay. (to get signals handled.)
|
| 905 | *
|
| 906 | * NOTE: this function doesnt have to take the runqueue lock,
|
| 907 | * because all it wants to ensure is that the remote task enters
|
| 908 | * the kernel. If the IPI races and the task has been migrated
|
| 909 | * to another CPU then no harm is done and the purpose has been
|
| 910 | * achieved as well.
|
| 911 |
|
| 912 | void kick_process(task_t *p)
|
| 913 | {
|
| 914 | int cpu;
|
| 915 |
|
| 916 | preempt_disable();
|
| 917 | cpu = task_cpu(p);
|
| 918 | if ((cpu != smp_processor_id()) && task_curr(p))
|
| 919 | smp_send_reschedule(cpu);
|
| 920 | preempt_enable();
|
| 921 | }
|
| 922 |
|
| 923 | |
| 924 | * Return a low guess at the load of a migration-source cpu.
|
| 925 | *
|
| 926 | * We want to under-estimate the load of migration sources, to
|
| 927 | * balance conservatively.
|
| 928 |
|
| 929 | static inline unsigned long source_load(int cpu, int type)
|
| 930 | {
|
| 931 | runqueue_t *rq = cpu_rq(cpu);
|
| 932 | unsigned long load_now = rq->nr_running * SCHED_LOAD_SCALE;
|
| 933 | if (type == 0)
|
| 934 | return load_now;
|
| 935 |
|
| 936 | return min(rq->cpu_load[type-1], load_now);
|
| 937 | }
|
| 938 |
|
| 939 | |
| 940 | * Return a high guess at the load of a migration-target cpu
|
| 941 |
|
| 942 | static inline unsigned long target_load(int cpu, int type)
|
| 943 | {
|
| 944 | runqueue_t *rq = cpu_rq(cpu);
|
| 945 | unsigned long load_now = rq->nr_running * SCHED_LOAD_SCALE;
|
| 946 | if (type == 0)
|
| 947 | return load_now;
|
| 948 |
|
| 949 | return max(rq->cpu_load[type-1], load_now);
|
| 950 | }
|
| 951 |
|
| 952 | |
| 953 | * find_idlest_group finds and returns the least busy CPU group within the
|
| 954 | * domain.
|
| 955 |
|
| 956 | static struct sched_group *
|
| 957 | find_idlest_group(struct sched_domain *sd, struct task_struct *p, int this_cpu)
|
| 958 | {
|
| 959 | struct sched_group *idlest = NULL, *this = NULL, *group = sd->groups;
|
| 960 | unsigned long min_load = ULONG_MAX, this_load = 0;
|
| 961 | int load_idx = sd->forkexec_idx;
|
| 962 | int imbalance = 100 + (sd->imbalance_pct-100)/2;
|
| 963 |
|
| 964 | do {
|
| 965 | unsigned long load, avg_load;
|
| 966 | int local_group;
|
| 967 | int i;
|
| 968 |
|
| 969 | local_group = cpu_isset(this_cpu, group->cpumask);
|
| 970 |
|
| 971 |
|
| 972 |
|
| 973 | avg_load = 0;
|
| 974 |
|
| 975 | for_each_cpu_mask(i, group->cpumask) {
|
| 976 |
|
| 977 | if (local_group)
|
| 978 | load = source_load(i, load_idx);
|
| 979 | else
|
| 980 | load = target_load(i, load_idx);
|
| 981 |
|
| 982 | avg_load += load;
|
| 983 | }
|
| 984 |
|
| 985 |
|
| 986 | avg_load = (avg_load * SCHED_LOAD_SCALE) / group->cpu_power;
|
| 987 |
|
| 988 | if (local_group) {
|
| 989 | this_load = avg_load;
|
| 990 | this = group;
|
| 991 | } else if (avg_load < min_load) {
|
| 992 | min_load = avg_load;
|
| 993 | idlest = group;
|
| 994 | }
|
| 995 | group = group->next;
|
| 996 | } while (group != sd->groups);
|
| 997 |
|
| 998 | if (!idlest || 100*this_load < imbalance*min_load)
|
| 999 | return NULL;
|
| 1000 | return idlest;
|
| 1001 | }
|
| 1002 |
|
| 1003 | |
| 1004 | * find_idlest_queue - find the idlest runqueue among the cpus in group.
|
| 1005 |
|
| 1006 | static int find_idlest_cpu(struct sched_group *group, int this_cpu)
|
| 1007 | {
|
| 1008 | unsigned long load, min_load = ULONG_MAX;
|
| 1009 | int idlest = -1;
|
| 1010 | int i;
|
| 1011 |
|
| 1012 | for_each_cpu_mask(i, group->cpumask) {
|
| 1013 | load = source_load(i, 0);
|
| 1014 |
|
| 1015 | if (load < min_load || (load == min_load && i == this_cpu)) {
|
| 1016 | min_load = load;
|
| 1017 | idlest = i;
|
| 1018 | }
|
| 1019 | }
|
| 1020 |
|
| 1021 | return idlest;
|
| 1022 | }
|
| 1023 |
|
| 1024 | |
| 1025 | * sched_balance_self: balance the current task (running on cpu) in domains
|
| 1026 | * that have the 'flag' flag set. In practice, this is SD_BALANCE_FORK and
|
| 1027 | * SD_BALANCE_EXEC.
|
| 1028 | *
|
| 1029 | * Balance, ie. select the least loaded group.
|
| 1030 | *
|
| 1031 | * Returns the target CPU number, or the same CPU if no balancing is needed.
|
| 1032 | *
|
| 1033 | * preempt must be disabled.
|
| 1034 |
|
| 1035 | static int sched_balance_self(int cpu, int flag)
|
| 1036 | {
|
| 1037 | struct task_struct *t = current;
|
| 1038 | struct sched_domain *tmp, *sd = NULL;
|
| 1039 |
|
| 1040 | for_each_domain(cpu, tmp)
|
| 1041 | if (tmp->flags & flag)
|
| 1042 | sd = tmp;
|
| 1043 |
|
| 1044 | while (sd) {
|
| 1045 | cpumask_t span;
|
| 1046 | struct sched_group *group;
|
| 1047 | int new_cpu;
|
| 1048 | int weight;
|
| 1049 |
|
| 1050 | span = sd->span;
|
| 1051 | group = find_idlest_group(sd, t, cpu);
|
| 1052 | if (!group)
|
| 1053 | goto nextlevel;
|
| 1054 |
|
| 1055 | new_cpu = find_idlest_cpu(group, cpu);
|
| 1056 | if (new_cpu == -1 || new_cpu == cpu)
|
| 1057 | goto nextlevel;
|
| 1058 |
|
| 1059 |
|
| 1060 | cpu = new_cpu;
|
| 1061 | nextlevel:
|
| 1062 | sd = NULL;
|
| 1063 | weight = cpus_weight(span);
|
| 1064 | for_each_domain(cpu, tmp) {
|
| 1065 | if (weight <= cpus_weight(tmp->span))
|
| 1066 | break;
|
| 1067 | if (tmp->flags & flag)
|
| 1068 | sd = tmp;
|
| 1069 | }
|
| 1070 |
|
| 1071 | }
|
| 1072 |
|
| 1073 | return cpu;
|
| 1074 | }
|
| 1075 |
|
| 1076 | #endif
|
| 1077 |
|
| 1078 | |
| 1079 | * wake_idle() will wake a task on an idle cpu if task->cpu is
|
| 1080 | * not idle and an idle cpu is available. The span of cpus to
|
| 1081 | * search starts with cpus closest then further out as needed,
|
| 1082 | * so we always favor a closer, idle cpu.
|
| 1083 | *
|
| 1084 | * Returns the CPU we should wake onto.
|
| 1085 |
|
| 1086 | #if defined(ARCH_HAS_SCHED_WAKE_IDLE)
|
| 1087 | static int wake_idle(int cpu, task_t *p)
|
| 1088 | {
|
| 1089 | cpumask_t tmp;
|
| 1090 | struct sched_domain *sd;
|
| 1091 | int i;
|
| 1092 |
|
| 1093 | if (idle_cpu(cpu))
|
| 1094 | return cpu;
|
| 1095 |
|
| 1096 | for_each_domain(cpu, sd) {
|
| 1097 | if (sd->flags & SD_WAKE_IDLE) {
|
| 1098 | cpus_and(tmp, sd->span, p->cpus_allowed);
|
| 1099 | for_each_cpu_mask(i, tmp) {
|
| 1100 | if (idle_cpu(i))
|
| 1101 | return i;
|
| 1102 | }
|
| 1103 | }
|
| 1104 | else
|
| 1105 | break;
|
| 1106 | }
|
| 1107 | return cpu;
|
| 1108 | }
|
| 1109 | #else
|
| 1110 | static inline int wake_idle(int cpu, task_t *p)
|
| 1111 | {
|
| 1112 | return cpu;
|
| 1113 | }
|
| 1114 | #endif
|
| 1115 |
|
| 1116 | |
| 1117 | * try_to_wake_up - wake up a thread
|
| 1118 | * @p: the to-be-woken-up thread
|
| 1119 | * @state: the mask of task states that can be woken
|
| 1120 | * @sync: do a synchronous wakeup?
|
| 1121 | *
|
| 1122 | * Put it on the run-queue if it's not already there. The "current"
|
| 1123 | * thread is always on the run-queue (except when the actual
|
| 1124 | * re-schedule is in progress), and as such you're allowed to do
|
| 1125 | * the simpler "current->state = TASK_RUNNING" to mark yourself
|
| 1126 | * runnable without the overhead of this.
|
| 1127 | *
|
| 1128 | * returns failure only if the task is already active.
|
| 1129 |
|
| 1130 | static int try_to_wake_up(task_t * p, unsigned int state, int sync)
|
| 1131 | {
|
| 1132 | int cpu, this_cpu, success = 0;
|
| 1133 | unsigned long flags;
|
| 1134 | long old_state;
|
| 1135 | runqueue_t *rq;
|
| 1136 | #ifdef CONFIG_SMP
|
| 1137 | unsigned long load, this_load;
|
| 1138 | struct sched_domain *sd, *this_sd = NULL;
|
| 1139 | int new_cpu;
|
| 1140 | #endif
|
| 1141 |
|
| 1142 | rq = task_rq_lock(p, &flags);
|
| 1143 | old_state = p->state;
|
| 1144 | if (!(old_state & state))
|
| 1145 | goto out;
|
| 1146 |
|
| 1147 | if (p->array)
|
| 1148 | goto out_running;
|
| 1149 |
|
| 1150 | cpu = task_cpu(p);
|
| 1151 | this_cpu = smp_processor_id();
|
| 1152 |
|
| 1153 | #ifdef CONFIG_SMP
|
| 1154 | if (unlikely(task_running(rq, p)))
|
| 1155 | goto out_activate;
|
| 1156 |
|
| 1157 | new_cpu = cpu;
|
| 1158 |
|
| 1159 | schedstat_inc(rq, ttwu_cnt);
|
| 1160 | if (cpu == this_cpu) {
|
| 1161 | schedstat_inc(rq, ttwu_local);
|
| 1162 | goto out_set_cpu;
|
| 1163 | }
|
| 1164 |
|
| 1165 | for_each_domain(this_cpu, sd) {
|
| 1166 | if (cpu_isset(cpu, sd->span)) {
|
| 1167 | schedstat_inc(sd, ttwu_wake_remote);
|
| 1168 | this_sd = sd;
|
| 1169 | break;
|
| 1170 | }
|
| 1171 | }
|
| 1172 |
|
| 1173 | if (unlikely(!cpu_isset(this_cpu, p->cpus_allowed)))
|
| 1174 | goto out_set_cpu;
|
| 1175 |
|
| 1176 | |
| 1177 | * Check for affine wakeup and passive balancing possibilities.
|
| 1178 |
|
| 1179 | if (this_sd) {
|
| 1180 | int idx = this_sd->wake_idx;
|
| 1181 | unsigned int imbalance;
|
| 1182 |
|
| 1183 | imbalance = 100 + (this_sd->imbalance_pct - 100) / 2;
|
| 1184 |
|
| 1185 | load = source_load(cpu, idx);
|
| 1186 | this_load = target_load(this_cpu, idx);
|
| 1187 |
|
| 1188 | new_cpu = this_cpu;
|
| 1189 |
|
| 1190 | if (this_sd->flags & SD_WAKE_AFFINE) {
|
| 1191 | unsigned long tl = this_load;
|
| 1192 | |
| 1193 | * If sync wakeup then subtract the (maximum possible)
|
| 1194 | * effect of the currently running task from the load
|
| 1195 | * of the current CPU:
|
| 1196 |
|
| 1197 | if (sync)
|
| 1198 | tl -= SCHED_LOAD_SCALE;
|
| 1199 |
|
| 1200 | if ((tl <= load &&
|
| 1201 | tl + target_load(cpu, idx) <= SCHED_LOAD_SCALE) ||
|
| 1202 | 100*(tl + SCHED_LOAD_SCALE) <= imbalance*load) {
|
| 1203 | |
| 1204 | * This domain has SD_WAKE_AFFINE and
|
| 1205 | * p is cache cold in this domain, and
|
| 1206 | * there is no bad imbalance.
|
| 1207 |
|
| 1208 | schedstat_inc(this_sd, ttwu_move_affine);
|
| 1209 | goto out_set_cpu;
|
| 1210 | }
|
| 1211 | }
|
| 1212 |
|
| 1213 | |
| 1214 | * Start passive balancing when half the imbalance_pct
|
| 1215 | * limit is reached.
|
| 1216 |
|
| 1217 | if (this_sd->flags & SD_WAKE_BALANCE) {
|
| 1218 | if (imbalance*this_load <= 100*load) {
|
| 1219 | schedstat_inc(this_sd, ttwu_move_balance);
|
| 1220 | goto out_set_cpu;
|
| 1221 | }
|
| 1222 | }
|
| 1223 | }
|
| 1224 |
|
| 1225 | new_cpu = cpu;
|
| 1226 | out_set_cpu:
|
| 1227 | new_cpu = wake_idle(new_cpu, p);
|
| 1228 | if (new_cpu != cpu) {
|
| 1229 | set_task_cpu(p, new_cpu);
|
| 1230 | task_rq_unlock(rq, &flags);
|
| 1231 |
|
| 1232 | rq = task_rq_lock(p, &flags);
|
| 1233 | old_state = p->state;
|
| 1234 | if (!(old_state & state))
|
| 1235 | goto out;
|
| 1236 | if (p->array)
|
| 1237 | goto out_running;
|
| 1238 |
|
| 1239 | this_cpu = smp_processor_id();
|
| 1240 | cpu = task_cpu(p);
|
| 1241 | }
|
| 1242 |
|
| 1243 | out_activate:
|
| 1244 | #endif
|
| 1245 | if (old_state == TASK_UNINTERRUPTIBLE) {
|
| 1246 | rq->nr_uninterruptible--;
|
| 1247 | |
| 1248 | * Tasks on involuntary sleep don't earn
|
| 1249 | * sleep_avg beyond just interactive state.
|
| 1250 |
|
| 1251 | p->activated = -1;
|
| 1252 | }
|
| 1253 |
|
| 1254 | |
| 1255 | * Sync wakeups (i.e. those types of wakeups where the waker
|
| 1256 | * has indicated that it will leave the CPU in short order)
|
| 1257 | * don't trigger a preemption, if the woken up task will run on
|
| 1258 | * this cpu. (in this case the 'I will reschedule' promise of
|
| 1259 | * the waker guarantees that the freshly woken up task is going
|
| 1260 | * to be considered on this CPU.)
|
| 1261 |
|
| 1262 | activate_task(p, rq, cpu == this_cpu);
|
| 1263 | if (!sync || cpu != this_cpu) {
|
| 1264 | if (TASK_PREEMPTS_CURR(p, rq))
|
| 1265 | resched_task(rq->curr);
|
| 1266 | }
|
| 1267 | success = 1;
|
| 1268 |
|
| 1269 | out_running:
|
| 1270 | p->state = TASK_RUNNING;
|
| 1271 | out:
|
| 1272 | task_rq_unlock(rq, &flags);
|
| 1273 |
|
| 1274 | return success;
|
| 1275 | }
|
| 1276 |
|
| 1277 | int fastcall wake_up_process(task_t * p)
|
| 1278 | {
|
| 1279 | return try_to_wake_up(p, TASK_STOPPED | TASK_TRACED |
|
| 1280 | TASK_INTERRUPTIBLE | TASK_UNINTERRUPTIBLE, 0);
|
| 1281 | }
|
| 1282 |
|
| 1283 | EXPORT_SYMBOL(wake_up_process);
|
| 1284 |
|
| 1285 | int fastcall wake_up_state(task_t *p, unsigned int state)
|
| 1286 | {
|
| 1287 | return try_to_wake_up(p, state, 0);
|
| 1288 | }
|
| 1289 |
|
| 1290 | |
| 1291 | * Perform scheduler related setup for a newly forked process p.
|
| 1292 | * p is forked by current.
|
| 1293 |
|
| 1294 | void fastcall sched_fork(task_t *p, int clone_flags)
|
| 1295 | {
|
| 1296 | int cpu = get_cpu();
|
| 1297 |
|
| 1298 | #ifdef CONFIG_SMP
|
| 1299 | cpu = sched_balance_self(cpu, SD_BALANCE_FORK);
|
| 1300 | #endif
|
| 1301 | set_task_cpu(p, cpu);
|
| 1302 |
|
| 1303 | |
| 1304 | * We mark the process as running here, but have not actually
|
| 1305 | * inserted it onto the runqueue yet. This guarantees that
|
| 1306 | * nobody will actually run it, and a signal or other external
|
| 1307 | * event cannot wake it up and insert it on the runqueue either.
|
| 1308 |
|
| 1309 | p->state = TASK_RUNNING;
|
| 1310 | INIT_LIST_HEAD(&p->run_list);
|
| 1311 | p->array = NULL;
|
| 1312 | #ifdef CONFIG_SCHEDSTATS
|
| 1313 | memset(&p->sched_info, 0, sizeof(p->sched_info));
|
| 1314 | #endif
|
| 1315 | #if defined(CONFIG_SMP) && defined(__ARCH_WANT_UNLOCKED_CTXSW)
|
| 1316 | p->oncpu = 0;
|
| 1317 | #endif
|
| 1318 | #ifdef CONFIG_PREEMPT
|
| 1319 |
|
| 1320 | p->thread_info->preempt_count = 1;
|
| 1321 | #endif
|
| 1322 | |
| 1323 | * Share the timeslice between parent and child, thus the
|
| 1324 | * total amount of pending timeslices in the system doesn't change,
|
| 1325 | * resulting in more scheduling fairness.
|
| 1326 |
|
| 1327 | local_irq_disable();
|
| 1328 | p->time_slice = (current->time_slice + 1) >> 1;
|
| 1329 | |
| 1330 | * The remainder of the first timeslice might be recovered by
|
| 1331 | * the parent if the child exits early enough.
|
| 1332 |
|
| 1333 | p->first_time_slice = 1;
|
| 1334 | current->time_slice >>= 1;
|
| 1335 | p->timestamp = sched_clock();
|
| 1336 | if (unlikely(!current->time_slice)) {
|
| 1337 | |
| 1338 | * This case is rare, it happens when the parent has only
|
| 1339 | * a single jiffy left from its timeslice. Taking the
|
| 1340 | * runqueue lock is not a problem.
|
| 1341 |
|
| 1342 | current->time_slice = 1;
|
| 1343 | scheduler_tick();
|
| 1344 | }
|
| 1345 | local_irq_enable();
|
| 1346 | put_cpu();
|
| 1347 | }
|
| 1348 |
|
| 1349 | |
| 1350 | * wake_up_new_task - wake up a newly created task for the first time.
|
| 1351 | *
|
| 1352 | * This function will do some initial scheduler statistics housekeeping
|
| 1353 | * that must be done for every newly created context, then puts the task
|
| 1354 | * on the runqueue and wakes it.
|
| 1355 |
|
| 1356 | void fastcall wake_up_new_task(task_t * p, unsigned long clone_flags)
|
| 1357 | {
|
| 1358 | unsigned long flags;
|
| 1359 | int this_cpu, cpu;
|
| 1360 | runqueue_t *rq, *this_rq;
|
| 1361 |
|
| 1362 | rq = task_rq_lock(p, &flags);
|
| 1363 | BUG_ON(p->state != TASK_RUNNING);
|
| 1364 | this_cpu = smp_processor_id();
|
| 1365 | cpu = task_cpu(p);
|
| 1366 |
|
| 1367 | |
| 1368 | * We decrease the sleep average of forking parents
|
| 1369 | * and children as well, to keep max-interactive tasks
|
| 1370 | * from forking tasks that are max-interactive. The parent
|
| 1371 | * (current) is done further down, under its lock.
|
| 1372 |
|
| 1373 | p->sleep_avg = JIFFIES_TO_NS(CURRENT_BONUS(p) *
|
| 1374 | CHILD_PENALTY / 100 * MAX_SLEEP_AVG / MAX_BONUS);
|
| 1375 |
|
| 1376 | p->prio = effective_prio(p);
|
| 1377 |
|
| 1378 | if (likely(cpu == this_cpu)) {
|
| 1379 | if (!(clone_flags & CLONE_VM)) {
|
| 1380 | |
| 1381 | * The VM isn't cloned, so we're in a good position to
|
| 1382 | * do child-runs-first in anticipation of an exec. This
|
| 1383 | * usually avoids a lot of COW overhead.
|
| 1384 |
|
| 1385 | if (unlikely(!current->array))
|
| 1386 | __activate_task(p, rq);
|
| 1387 | else {
|
| 1388 | p->prio = current->prio;
|
| 1389 | list_add_tail(&p->run_list, ¤t->run_list);
|
| 1390 | p->array = current->array;
|
| 1391 | p->array->nr_active++;
|
| 1392 | rq->nr_running++;
|
| 1393 | }
|
| 1394 | set_need_resched();
|
| 1395 | } else
|
| 1396 |
|
| 1397 | __activate_task(p, rq);
|
| 1398 | |
| 1399 | * We skip the following code due to cpu == this_cpu
|
| 1400 | *
|
| 1401 | * task_rq_unlock(rq, &flags);
|
| 1402 | * this_rq = task_rq_lock(current, &flags);
|
| 1403 |
|
| 1404 | this_rq = rq;
|
| 1405 | } else {
|
| 1406 | this_rq = cpu_rq(this_cpu);
|
| 1407 |
|
| 1408 | |
| 1409 | * Not the local CPU - must adjust timestamp. This should
|
| 1410 | * get optimised away in the !CONFIG_SMP case.
|
| 1411 |
|
| 1412 | p->timestamp = (p->timestamp - this_rq->timestamp_last_tick)
|
| 1413 | + rq->timestamp_last_tick;
|
| 1414 | __activate_task(p, rq);
|
| 1415 | if (TASK_PREEMPTS_CURR(p, rq))
|
| 1416 | resched_task(rq->curr);
|
| 1417 |
|
| 1418 | |
| 1419 | * Parent and child are on different CPUs, now get the
|
| 1420 | * parent runqueue to update the parent's ->sleep_avg:
|
| 1421 |
|
| 1422 | task_rq_unlock(rq, &flags);
|
| 1423 | this_rq = task_rq_lock(current, &flags);
|
| 1424 | }
|
| 1425 | current->sleep_avg = JIFFIES_TO_NS(CURRENT_BONUS(current) *
|
| 1426 | PARENT_PENALTY / 100 * MAX_SLEEP_AVG / MAX_BONUS);
|
| 1427 | task_rq_unlock(this_rq, &flags);
|
| 1428 | }
|
| 1429 |
|
| 1430 | |
| 1431 | * Potentially available exiting-child timeslices are
|
| 1432 | * retrieved here - this way the parent does not get
|
| 1433 | * penalized for creating too many threads.
|
| 1434 | *
|
| 1435 | * (this cannot be used to 'generate' timeslices
|
| 1436 | * artificially, because any timeslice recovered here
|
| 1437 | * was given away by the parent in the first place.)
|
| 1438 |
|
| 1439 | void fastcall sched_exit(task_t * p)
|
| 1440 | {
|
| 1441 | unsigned long flags;
|
| 1442 | runqueue_t *rq;
|
| 1443 |
|
| 1444 | |
| 1445 | * If the child was a (relative-) CPU hog then decrease
|
| 1446 | * the sleep_avg of the parent as well.
|
| 1447 |
|
| 1448 | rq = task_rq_lock(p->parent, &flags);
|
| 1449 | if (p->first_time_slice) {
|
| 1450 | p->parent->time_slice += p->time_slice;
|
| 1451 | if (unlikely(p->parent->time_slice > task_timeslice(p)))
|
| 1452 | p->parent->time_slice = task_timeslice(p);
|
| 1453 | }
|
| 1454 | if (p->sleep_avg < p->parent->sleep_avg)
|
| 1455 | p->parent->sleep_avg = p->parent->sleep_avg /
|
| 1456 | (EXIT_WEIGHT + 1) * EXIT_WEIGHT + p->sleep_avg /
|
| 1457 | (EXIT_WEIGHT + 1);
|
| 1458 | task_rq_unlock(rq, &flags);
|
| 1459 | }
|
| 1460 |
|
| 1461 | |
| 1462 | * prepare_task_switch - prepare to switch tasks
|
| 1463 | * @rq: the runqueue preparing to switch
|
| 1464 | * @next: the task we are going to switch to.
|
| 1465 | *
|
| 1466 | * This is called with the rq lock held and interrupts off. It must
|
| 1467 | * be paired with a subsequent finish_task_switch after the context
|
| 1468 | * switch.
|
| 1469 | *
|
| 1470 | * prepare_task_switch sets up locking and calls architecture specific
|
| 1471 | * hooks.
|
| 1472 |
|
| 1473 | static inline void prepare_task_switch(runqueue_t *rq, task_t *next)
|
| 1474 | {
|
| 1475 | prepare_lock_switch(rq, next);
|
| 1476 | prepare_arch_switch(next);
|
| 1477 | }
|
| 1478 |
|
| 1479 | |
| 1480 | * finish_task_switch - clean up after a task-switch
|
| 1481 | * @prev: the thread we just switched away from.
|
| 1482 | *
|
| 1483 | * finish_task_switch must be called after the context switch, paired
|
| 1484 | * with a prepare_task_switch call before the context switch.
|
| 1485 | * finish_task_switch will reconcile locking set up by prepare_task_switch,
|
| 1486 | * and do any other architecture-specific cleanup actions.
|
| 1487 | *
|
| 1488 | * Note that we may have delayed dropping an mm in context_switch(). If
|
| 1489 | * so, we finish that here outside of the runqueue lock. (Doing it
|
| 1490 | * with the lock held can cause deadlocks; see schedule() for
|
| 1491 | * details.)
|
| 1492 |
|
| 1493 | static inline void finish_task_switch(runqueue_t *rq, task_t *prev)
|
| 1494 | __releases(rq->lock)
|
| 1495 | {
|
| 1496 | struct mm_struct *mm = rq->prev_mm;
|
| 1497 | unsigned long prev_task_flags;
|
| 1498 |
|
| 1499 | rq->prev_mm = NULL;
|
| 1500 |
|
| 1501 | |
| 1502 | * A task struct has one reference for the use as "current".
|
| 1503 | * If a task dies, then it sets EXIT_ZOMBIE in tsk->exit_state and
|
| 1504 | * calls schedule one last time. The schedule call will never return,
|
| 1505 | * and the scheduled task must drop that reference.
|
| 1506 | * The test for EXIT_ZOMBIE must occur while the runqueue locks are
|
| 1507 | * still held, otherwise prev could be scheduled on another cpu, die
|
| 1508 | * there before we look at prev->state, and then the reference would
|
| 1509 | * be dropped twice.
|
| 1510 | * Manfred Spraul <[email protected]>
|
| 1511 |
|
| 1512 | prev_task_flags = prev->flags;
|
| 1513 | finish_arch_switch(prev);
|
| 1514 | finish_lock_switch(rq, prev);
|
| 1515 | if (mm)
|
| 1516 | mmdrop(mm);
|
| 1517 | if (unlikely(prev_task_flags & PF_DEAD))
|
| 1518 | put_task_struct(prev);
|
| 1519 | }
|
| 1520 |
|
| 1521 | |
| 1522 | * schedule_tail - first thing a freshly forked thread must call.
|
| 1523 | * @prev: the thread we just switched away from.
|
| 1524 |
|
| 1525 | asmlinkage void schedule_tail(task_t *prev)
|
| 1526 | __releases(rq->lock)
|
| 1527 | {
|
| 1528 | runqueue_t *rq = this_rq();
|
| 1529 | finish_task_switch(rq, prev);
|
| 1530 | #ifdef __ARCH_WANT_UNLOCKED_CTXSW
|
| 1531 |
|
| 1532 | preempt_enable();
|
| 1533 | #endif
|
| 1534 | if (current->set_child_tid)
|
| 1535 | put_user(current->pid, current->set_child_tid);
|
| 1536 | }
|
| 1537 |
|
| 1538 | |
| 1539 | * context_switch - switch to the new MM and the new
|
| 1540 | * thread's register state.
|
| 1541 |
|
| 1542 | static inline
|
| 1543 | task_t * context_switch(runqueue_t *rq, task_t *prev, task_t *next)
|
| 1544 | {
|
| 1545 | struct mm_struct *mm = next->mm;
|
| 1546 | struct mm_struct *oldmm = prev->active_mm;
|
| 1547 |
|
| 1548 | if (unlikely(!mm)) {
|
| 1549 | next->active_mm = oldmm;
|
| 1550 | atomic_inc(&oldmm->mm_count);
|
| 1551 | enter_lazy_tlb(oldmm, next);
|
| 1552 | } else
|
| 1553 | switch_mm(oldmm, mm, next);
|
| 1554 |
|
| 1555 | if (unlikely(!prev->mm)) {
|
| 1556 | prev->active_mm = NULL;
|
| 1557 | WARN_ON(rq->prev_mm);
|
| 1558 | rq->prev_mm = oldmm;
|
| 1559 | }
|
| 1560 |
|
| 1561 |
|
| 1562 | switch_to(prev, next, prev);
|
| 1563 |
|
| 1564 | return prev;
|
| 1565 | }
|
| 1566 |
|
| 1567 | |
| 1568 | * nr_running, nr_uninterruptible and nr_context_switches:
|
| 1569 | *
|
| 1570 | * externally visible scheduler statistics: current number of runnable
|
| 1571 | * threads, current number of uninterruptible-sleeping threads, total
|
| 1572 | * number of context switches performed since bootup.
|
| 1573 |
|
| 1574 | unsigned long nr_running(void)
|
| 1575 | {
|
| 1576 | unsigned long i, sum = 0;
|
| 1577 |
|
| 1578 | for_each_online_cpu(i)
|
| 1579 | sum += cpu_rq(i)->nr_running;
|
| 1580 |
|
| 1581 | return sum;
|
| 1582 | }
|
| 1583 |
|
| 1584 | unsigned long nr_uninterruptible(void)
|
| 1585 | {
|
| 1586 | unsigned long i, sum = 0;
|
| 1587 |
|
| 1588 | for_each_cpu(i)
|
| 1589 | sum += cpu_rq(i)->nr_uninterruptible;
|
| 1590 |
|
| 1591 | |
| 1592 | * Since we read the counters lockless, it might be slightly
|
| 1593 | * inaccurate. Do not allow it to go below zero though:
|
| 1594 |
|
| 1595 | if (unlikely((long)sum < 0))
|
| 1596 | sum = 0;
|
| 1597 |
|
| 1598 | return sum;
|
| 1599 | }
|
| 1600 |
|
| 1601 | unsigned long long nr_context_switches(void)
|
| 1602 | {
|
| 1603 | unsigned long long i, sum = 0;
|
| 1604 |
|
| 1605 | for_each_cpu(i)
|
| 1606 | sum += cpu_rq(i)->nr_switches;
|
| 1607 |
|
| 1608 | return sum;
|
| 1609 | }
|
| 1610 |
|
| 1611 | unsigned long nr_iowait(void)
|
| 1612 | {
|
| 1613 | unsigned long i, sum = 0;
|
| 1614 |
|
| 1615 | for_each_cpu(i)
|
| 1616 | sum += atomic_read(&cpu_rq(i)->nr_iowait);
|
| 1617 |
|
| 1618 | return sum;
|
| 1619 | }
|
| 1620 |
|
| 1621 | #ifdef CONFIG_SMP
|
| 1622 |
|
| 1623 | |
| 1624 | * double_rq_lock - safely lock two runqueues
|
| 1625 | *
|
| 1626 | * Note this does not disable interrupts like task_rq_lock,
|
| 1627 | * you need to do so manually before calling.
|
| 1628 |
|
| 1629 | static void double_rq_lock(runqueue_t *rq1, runqueue_t *rq2)
|
| 1630 | __acquires(rq1->lock)
|
| 1631 | __acquires(rq2->lock)
|
| 1632 | {
|
| 1633 | if (rq1 == rq2) {
|
| 1634 | spin_lock(&rq1->lock);
|
| 1635 | __acquire(rq2->lock);
|
| 1636 | } else {
|
| 1637 | if (rq1 < rq2) {
|
| 1638 | spin_lock(&rq1->lock);
|
| 1639 | spin_lock(&rq2->lock);
|
| 1640 | } else {
|
| 1641 | spin_lock(&rq2->lock);
|
| 1642 | spin_lock(&rq1->lock);
|
| 1643 | }
|
| 1644 | }
|
| 1645 | }
|
| 1646 |
|
| 1647 | |
| 1648 | * double_rq_unlock - safely unlock two runqueues
|
| 1649 | *
|
| 1650 | * Note this does not restore interrupts like task_rq_unlock,
|
| 1651 | * you need to do so manually after calling.
|
| 1652 |
|
| 1653 | static void double_rq_unlock(runqueue_t *rq1, runqueue_t *rq2)
|
| 1654 | __releases(rq1->lock)
|
| 1655 | __releases(rq2->lock)
|
| 1656 | {
|
| 1657 | spin_unlock(&rq1->lock);
|
| 1658 | if (rq1 != rq2)
|
| 1659 | spin_unlock(&rq2->lock);
|
| 1660 | else
|
| 1661 | __release(rq2->lock);
|
| 1662 | }
|
| 1663 |
|
| 1664 | |
| 1665 | * double_lock_balance - lock the busiest runqueue, this_rq is locked already.
|
| 1666 |
|
| 1667 | static void double_lock_balance(runqueue_t *this_rq, runqueue_t *busiest)
|
| 1668 | __releases(this_rq->lock)
|
| 1669 | __acquires(busiest->lock)
|
| 1670 | __acquires(this_rq->lock)
|
| 1671 | {
|
| 1672 | if (unlikely(!spin_trylock(&busiest->lock))) {
|
| 1673 | if (busiest < this_rq) {
|
| 1674 | spin_unlock(&this_rq->lock);
|
| 1675 | spin_lock(&busiest->lock);
|
| 1676 | spin_lock(&this_rq->lock);
|
| 1677 | } else
|
| 1678 | spin_lock(&busiest->lock);
|
| 1679 | }
|
| 1680 | }
|
| 1681 |
|
| 1682 | |
| 1683 | * If dest_cpu is allowed for this process, migrate the task to it.
|
| 1684 | * This is accomplished by forcing the cpu_allowed mask to only
|
| 1685 | * allow dest_cpu, which will force the cpu onto dest_cpu. Then
|
| 1686 | * the cpu_allowed mask is restored.
|
| 1687 |
|
| 1688 | static void sched_migrate_task(task_t *p, int dest_cpu)
|
| 1689 | {
|
| 1690 | migration_req_t req;
|
| 1691 | runqueue_t *rq;
|
| 1692 | unsigned long flags;
|
| 1693 |
|
| 1694 | rq = task_rq_lock(p, &flags);
|
| 1695 | if (!cpu_isset(dest_cpu, p->cpus_allowed)
|
| 1696 | || unlikely(cpu_is_offline(dest_cpu)))
|
| 1697 | goto out;
|
| 1698 |
|
| 1699 |
|
| 1700 | if (migrate_task(p, dest_cpu, &req)) {
|
| 1701 |
|
| 1702 | struct task_struct *mt = rq->migration_thread;
|
| 1703 | get_task_struct(mt);
|
| 1704 | task_rq_unlock(rq, &flags);
|
| 1705 | wake_up_process(mt);
|
| 1706 | put_task_struct(mt);
|
| 1707 | wait_for_completion(&req.done);
|
| 1708 | return;
|
| 1709 | }
|
| 1710 | out:
|
| 1711 | task_rq_unlock(rq, &flags);
|
| 1712 | }
|
| 1713 |
|
| 1714 | |
| 1715 | * sched_exec - execve() is a valuable balancing opportunity, because at
|
| 1716 | * this point the task has the smallest effective memory and cache footprint.
|
| 1717 |
|
| 1718 | void sched_exec(void)
|
| 1719 | {
|
| 1720 | int new_cpu, this_cpu = get_cpu();
|
| 1721 | new_cpu = sched_balance_self(this_cpu, SD_BALANCE_EXEC);
|
| 1722 | put_cpu();
|
| 1723 | if (new_cpu != this_cpu)
|
| 1724 | sched_migrate_task(current, new_cpu);
|
| 1725 | }
|
| 1726 |
|
| 1727 | |
| 1728 | * pull_task - move a task from a remote runqueue to the local runqueue.
|
| 1729 | * Both runqueues must be locked.
|
| 1730 |
|
| 1731 | static inline
|
| 1732 | void pull_task(runqueue_t *src_rq, prio_array_t *src_array, task_t *p,
|
| 1733 | runqueue_t *this_rq, prio_array_t *this_array, int this_cpu)
|
| 1734 | {
|
| 1735 | dequeue_task(p, src_array);
|
| 1736 | src_rq->nr_running--;
|
| 1737 | set_task_cpu(p, this_cpu);
|
| 1738 | this_rq->nr_running++;
|
| 1739 | enqueue_task(p, this_array);
|
| 1740 | p->timestamp = (p->timestamp - src_rq->timestamp_last_tick)
|
| 1741 | + this_rq->timestamp_last_tick;
|
| 1742 | |
| 1743 | * Note that idle threads have a prio of MAX_PRIO, for this test
|
| 1744 | * to be always true for them.
|
| 1745 |
|
| 1746 | if (TASK_PREEMPTS_CURR(p, this_rq))
|
| 1747 | resched_task(this_rq->curr);
|
| 1748 | }
|
| 1749 |
|
| 1750 | |
| 1751 | * can_migrate_task - may task p from runqueue rq be migrated to this_cpu?
|
| 1752 |
|
| 1753 | static inline
|
| 1754 | int can_migrate_task(task_t *p, runqueue_t *rq, int this_cpu,
|
| 1755 | struct sched_domain *sd, enum idle_type idle, int *all_pinned)
|
| 1756 | {
|
| 1757 | |
| 1758 | * We do not migrate tasks that are:
|
| 1759 | * 1) running (obviously), or
|
| 1760 | * 2) cannot be migrated to this CPU due to cpus_allowed, or
|
| 1761 | * 3) are cache-hot on their current CPU.
|
| 1762 |
|
| 1763 | if (!cpu_isset(this_cpu, p->cpus_allowed))
|
| 1764 | return 0;
|
| 1765 | *all_pinned = 0;
|
| 1766 |
|
| 1767 | if (task_running(rq, p))
|
| 1768 | return 0;
|
| 1769 |
|
| 1770 | |
| 1771 | * Aggressive migration if:
|
| 1772 | * 1) task is cache cold, or
|
| 1773 | * 2) too many balance attempts have failed.
|
| 1774 |
|
| 1775 |
|
| 1776 | if (sd->nr_balance_failed > sd->cache_nice_tries)
|
| 1777 | return 1;
|
| 1778 |
|
| 1779 | if (task_hot(p, rq->timestamp_last_tick, sd))
|
| 1780 | return 0;
|
| 1781 | return 1;
|
| 1782 | }
|
| 1783 |
|
| 1784 | |
| 1785 | * move_tasks tries to move up to max_nr_move tasks from busiest to this_rq,
|
| 1786 | * as part of a balancing operation within "domain". Returns the number of
|
| 1787 | * tasks moved.
|
| 1788 | *
|
| 1789 | * Called with both runqueues locked.
|
| 1790 |
|
| 1791 | static int move_tasks(runqueue_t *this_rq, int this_cpu, runqueue_t *busiest,
|
| 1792 | unsigned long max_nr_move, struct sched_domain *sd,
|
| 1793 | enum idle_type idle, int *all_pinned)
|
| 1794 | {
|
| 1795 | prio_array_t *array, *dst_array;
|
| 1796 | struct list_head *head, *curr;
|
| 1797 | int idx, pulled = 0, pinned = 0;
|
| 1798 | task_t *tmp;
|
| 1799 |
|
| 1800 | if (max_nr_move == 0)
|
| 1801 | goto out;
|
| 1802 |
|
| 1803 | pinned = 1;
|
| 1804 |
|
| 1805 | |
| 1806 | * We first consider expired tasks. Those will likely not be
|
| 1807 | * executed in the near future, and they are most likely to
|
| 1808 | * be cache-cold, thus switching CPUs has the least effect
|
| 1809 | * on them.
|
| 1810 |
|
| 1811 | if (busiest->expired->nr_active) {
|
| 1812 | array = busiest->expired;
|
| 1813 | dst_array = this_rq->expired;
|
| 1814 | } else {
|
| 1815 | array = busiest->active;
|
| 1816 | dst_array = this_rq->active;
|
| 1817 | }
|
| 1818 |
|
| 1819 | new_array:
|
| 1820 |
|
| 1821 | idx = 0;
|
| 1822 | skip_bitmap:
|
| 1823 | if (!idx)
|
| 1824 | idx = sched_find_first_bit(array->bitmap);
|
| 1825 | else
|
| 1826 | idx = find_next_bit(array->bitmap, MAX_PRIO, idx);
|
| 1827 | if (idx >= MAX_PRIO) {
|
| 1828 | if (array == busiest->expired && busiest->active->nr_active) {
|
| 1829 | array = busiest->active;
|
| 1830 | dst_array = this_rq->active;
|
| 1831 | goto new_array;
|
| 1832 | }
|
| 1833 | goto out;
|
| 1834 | }
|
| 1835 |
|
| 1836 | head = array->queue + idx;
|
| 1837 | curr = head->prev;
|
| 1838 | skip_queue:
|
| 1839 | tmp = list_entry(curr, task_t, run_list);
|
| 1840 |
|
| 1841 | curr = curr->prev;
|
| 1842 |
|
| 1843 | if (!can_migrate_task(tmp, busiest, this_cpu, sd, idle, &pinned)) {
|
| 1844 | if (curr != head)
|
| 1845 | goto skip_queue;
|
| 1846 | idx++;
|
| 1847 | goto skip_bitmap;
|
| 1848 | }
|
| 1849 |
|
| 1850 | #ifdef CONFIG_SCHEDSTATS
|
| 1851 | if (task_hot(tmp, busiest->timestamp_last_tick, sd))
|
| 1852 | schedstat_inc(sd, lb_hot_gained[idle]);
|
| 1853 | #endif
|
| 1854 |
|
| 1855 | pull_task(busiest, array, tmp, this_rq, dst_array, this_cpu);
|
| 1856 | pulled++;
|
| 1857 |
|
| 1858 |
|
| 1859 | if (pulled < max_nr_move) {
|
| 1860 | if (curr != head)
|
| 1861 | goto skip_queue;
|
| 1862 | idx++;
|
| 1863 | goto skip_bitmap;
|
| 1864 | }
|
| 1865 | out:
|
| 1866 | |
| 1867 | * Right now, this is the only place pull_task() is called,
|
| 1868 | * so we can safely collect pull_task() stats here rather than
|
| 1869 | * inside pull_task().
|
| 1870 |
|
| 1871 | schedstat_add(sd, lb_gained[idle], pulled);
|
| 1872 |
|
| 1873 | if (all_pinned)
|
| 1874 | *all_pinned = pinned;
|
| 1875 | return pulled;
|
| 1876 | }
|
| 1877 |
|
| 1878 | |
| 1879 | * find_busiest_group finds and returns the busiest CPU group within the
|
| 1880 | * domain. It calculates and returns the number of tasks which should be
|
| 1881 | * moved to restore balance via the imbalance parameter.
|
| 1882 |
|
| 1883 | static struct sched_group *
|
| 1884 | find_busiest_group(struct sched_domain *sd, int this_cpu,
|
| 1885 | unsigned long *imbalance, enum idle_type idle)
|
| 1886 | {
|
| 1887 | struct sched_group *busiest = NULL, *this = NULL, *group = sd->groups;
|
| 1888 | unsigned long max_load, avg_load, total_load, this_load, total_pwr;
|
| 1889 | int load_idx;
|
| 1890 |
|
| 1891 | max_load = this_load = total_load = total_pwr = 0;
|
| 1892 | if (idle == NOT_IDLE)
|
| 1893 | load_idx = sd->busy_idx;
|
| 1894 | else if (idle == NEWLY_IDLE)
|
| 1895 | load_idx = sd->newidle_idx;
|
| 1896 | else
|
| 1897 | load_idx = sd->idle_idx;
|
| 1898 |
|
| 1899 | do {
|
| 1900 | unsigned long load;
|
| 1901 | int local_group;
|
| 1902 | int i;
|
| 1903 |
|
| 1904 | local_group = cpu_isset(this_cpu, group->cpumask);
|
| 1905 |
|
| 1906 |
|
| 1907 | avg_load = 0;
|
| 1908 |
|
| 1909 | for_each_cpu_mask(i, group->cpumask) {
|
| 1910 |
|
| 1911 | if (local_group)
|
| 1912 | load = target_load(i, load_idx);
|
| 1913 | else
|
| 1914 | load = source_load(i, load_idx);
|
| 1915 |
|
| 1916 | avg_load += load;
|
| 1917 | }
|
| 1918 |
|
| 1919 | total_load += avg_load;
|
| 1920 | total_pwr += group->cpu_power;
|
| 1921 |
|
| 1922 |
|
| 1923 | avg_load = (avg_load * SCHED_LOAD_SCALE) / group->cpu_power;
|
| 1924 |
|
| 1925 | if (local_group) {
|
| 1926 | this_load = avg_load;
|
| 1927 | this = group;
|
| 1928 | } else if (avg_load > max_load) {
|
| 1929 | max_load = avg_load;
|
| 1930 | busiest = group;
|
| 1931 | }
|
| 1932 | group = group->next;
|
| 1933 | } while (group != sd->groups);
|
| 1934 |
|
| 1935 | if (!busiest || this_load >= max_load)
|
| 1936 | goto out_balanced;
|
| 1937 |
|
| 1938 | avg_load = (SCHED_LOAD_SCALE * total_load) / total_pwr;
|
| 1939 |
|
| 1940 | if (this_load >= avg_load ||
|
| 1941 | 100*max_load <= sd->imbalance_pct*this_load)
|
| 1942 | goto out_balanced;
|
| 1943 |
|
| 1944 | |
| 1945 | * We're trying to get all the cpus to the average_load, so we don't
|
| 1946 | * want to push ourselves above the average load, nor do we wish to
|
| 1947 | * reduce the max loaded cpu below the average load, as either of these
|
| 1948 | * actions would just result in more rebalancing later, and ping-pong
|
| 1949 | * tasks around. Thus we look for the minimum possible imbalance.
|
| 1950 | * Negative imbalances (*we* are more loaded than anyone else) will
|
| 1951 | * be counted as no imbalance for these purposes -- we can't fix that
|
| 1952 | * by pulling tasks to us. Be careful of negative numbers as they'll
|
| 1953 | * appear as very large values with unsigned longs.
|
| 1954 |
|
| 1955 |
|
| 1956 | *imbalance = min((max_load - avg_load) * busiest->cpu_power,
|
| 1957 | (avg_load - this_load) * this->cpu_power)
|
| 1958 | / SCHED_LOAD_SCALE;
|
| 1959 |
|
| 1960 | if (*imbalance < SCHED_LOAD_SCALE) {
|
| 1961 | unsigned long pwr_now = 0, pwr_move = 0;
|
| 1962 | unsigned long tmp;
|
| 1963 |
|
| 1964 | if (max_load - this_load >= SCHED_LOAD_SCALE*2) {
|
| 1965 | *imbalance = 1;
|
| 1966 | return busiest;
|
| 1967 | }
|
| 1968 |
|
| 1969 | |
| 1970 | * OK, we don't have enough imbalance to justify moving tasks,
|
| 1971 | * however we may be able to increase total CPU power used by
|
| 1972 | * moving them.
|
| 1973 |
|
| 1974 |
|
| 1975 | pwr_now += busiest->cpu_power*min(SCHED_LOAD_SCALE, max_load);
|
| 1976 | pwr_now += this->cpu_power*min(SCHED_LOAD_SCALE, this_load);
|
| 1977 | pwr_now /= SCHED_LOAD_SCALE;
|
| 1978 |
|
| 1979 |
|
| 1980 | tmp = SCHED_LOAD_SCALE*SCHED_LOAD_SCALE/busiest->cpu_power;
|
| 1981 | if (max_load > tmp)
|
| 1982 | pwr_move += busiest->cpu_power*min(SCHED_LOAD_SCALE,
|
| 1983 | max_load - tmp);
|
| 1984 |
|
| 1985 |
|
| 1986 | if (max_load*busiest->cpu_power <
|
| 1987 | SCHED_LOAD_SCALE*SCHED_LOAD_SCALE)
|
| 1988 | tmp = max_load*busiest->cpu_power/this->cpu_power;
|
| 1989 | else
|
| 1990 | tmp = SCHED_LOAD_SCALE*SCHED_LOAD_SCALE/this->cpu_power;
|
| 1991 | pwr_move += this->cpu_power*min(SCHED_LOAD_SCALE, this_load + tmp);
|
| 1992 | pwr_move /= SCHED_LOAD_SCALE;
|
| 1993 |
|
| 1994 |
|
| 1995 | if (pwr_move <= pwr_now)
|
| 1996 | goto out_balanced;
|
| 1997 |
|
| 1998 | *imbalance = 1;
|
| 1999 | return busiest;
|
| 2000 | }
|
| 2001 |
|
| 2002 |
|
| 2003 | *imbalance = *imbalance / SCHED_LOAD_SCALE;
|
| 2004 | return busiest;
|
| 2005 |
|
| 2006 | out_balanced:
|
| 2007 |
|
| 2008 | *imbalance = 0;
|
| 2009 | return NULL;
|
| 2010 | }
|
| 2011 |
|
| 2012 | |
| 2013 | * find_busiest_queue - find the busiest runqueue among the cpus in group.
|
| 2014 |
|
| 2015 | static runqueue_t *find_busiest_queue(struct sched_group *group)
|
| 2016 | {
|
| 2017 | unsigned long load, max_load = 0;
|
| 2018 | runqueue_t *busiest = NULL;
|
| 2019 | int i;
|
| 2020 |
|
| 2021 | for_each_cpu_mask(i, group->cpumask) {
|
| 2022 | load = source_load(i, 0);
|
| 2023 |
|
| 2024 | if (load > max_load) {
|
| 2025 | max_load = load;
|
| 2026 | busiest = cpu_rq(i);
|
| 2027 | }
|
| 2028 | }
|
| 2029 |
|
| 2030 | return busiest;
|
| 2031 | }
|
| 2032 |
|
| 2033 | |
| 2034 | * Max backoff if we encounter pinned tasks. Pretty arbitrary value, but
|
| 2035 | * so long as it is large enough.
|
| 2036 |
|
| 2037 | #define MAX_PINNED_INTERVAL 512
|
| 2038 |
|
| 2039 | |
| 2040 | * Check this_cpu to ensure it is balanced within domain. Attempt to move
|
| 2041 | * tasks if there is an imbalance.
|
| 2042 | *
|
| 2043 | * Called with this_rq unlocked.
|
| 2044 |
|
| 2045 | static int load_balance(int this_cpu, runqueue_t *this_rq,
|
| 2046 | struct sched_domain *sd, enum idle_type idle)
|
| 2047 | {
|
| 2048 | struct sched_group *group;
|
| 2049 | runqueue_t *busiest;
|
| 2050 | unsigned long imbalance;
|
| 2051 | int nr_moved, all_pinned = 0;
|
| 2052 | int active_balance = 0;
|
| 2053 |
|
| 2054 | spin_lock(&this_rq->lock);
|
| 2055 | schedstat_inc(sd, lb_cnt[idle]);
|
| 2056 |
|
| 2057 | group = find_busiest_group(sd, this_cpu, &imbalance, idle);
|
| 2058 | if (!group) {
|
| 2059 | schedstat_inc(sd, lb_nobusyg[idle]);
|
| 2060 | goto out_balanced;
|
| 2061 | }
|
| 2062 |
|
| 2063 | busiest = find_busiest_queue(group);
|
| 2064 | if (!busiest) {
|
| 2065 | schedstat_inc(sd, lb_nobusyq[idle]);
|
| 2066 | goto out_balanced;
|
| 2067 | }
|
| 2068 |
|
| 2069 | BUG_ON(busiest == this_rq);
|
| 2070 |
|
| 2071 | schedstat_add(sd, lb_imbalance[idle], imbalance);
|
| 2072 |
|
| 2073 | nr_moved = 0;
|
| 2074 | if (busiest->nr_running > 1) {
|
| 2075 | |
| 2076 | * Attempt to move tasks. If find_busiest_group has found
|
| 2077 | * an imbalance but busiest->nr_running <= 1, the group is
|
| 2078 | * still unbalanced. nr_moved simply stays zero, so it is
|
| 2079 | * correctly treated as an imbalance.
|
| 2080 |
|
| 2081 | double_lock_balance(this_rq, busiest);
|
| 2082 | nr_moved = move_tasks(this_rq, this_cpu, busiest,
|
| 2083 | imbalance, sd, idle,
|
| 2084 | &all_pinned);
|
| 2085 | spin_unlock(&busiest->lock);
|
| 2086 |
|
| 2087 |
|
| 2088 | if (unlikely(all_pinned))
|
| 2089 | goto out_balanced;
|
| 2090 | }
|
| 2091 |
|
| 2092 | spin_unlock(&this_rq->lock);
|
| 2093 |
|
| 2094 | if (!nr_moved) {
|
| 2095 | schedstat_inc(sd, lb_failed[idle]);
|
| 2096 | sd->nr_balance_failed++;
|
| 2097 |
|
| 2098 | if (unlikely(sd->nr_balance_failed > sd->cache_nice_tries+2)) {
|
| 2099 |
|
| 2100 | spin_lock(&busiest->lock);
|
| 2101 | if (!busiest->active_balance) {
|
| 2102 | busiest->active_balance = 1;
|
| 2103 | busiest->push_cpu = this_cpu;
|
| 2104 | active_balance = 1;
|
| 2105 | }
|
| 2106 | spin_unlock(&busiest->lock);
|
| 2107 | if (active_balance)
|
| 2108 | wake_up_process(busiest->migration_thread);
|
| 2109 |
|
| 2110 | |
| 2111 | * We've kicked active balancing, reset the failure
|
| 2112 | * counter.
|
| 2113 |
|
| 2114 | sd->nr_balance_failed = sd->cache_nice_tries+1;
|
| 2115 | }
|
| 2116 | } else
|
| 2117 | sd->nr_balance_failed = 0;
|
| 2118 |
|
| 2119 | if (likely(!active_balance)) {
|
| 2120 |
|
| 2121 | sd->balance_interval = sd->min_interval;
|
| 2122 | } else {
|
| 2123 | |
| 2124 | * If we've begun active balancing, start to back off. This
|
| 2125 | * case may not be covered by the all_pinned logic if there
|
| 2126 | * is only 1 task on the busy runqueue (because we don't call
|
| 2127 | * move_tasks).
|
| 2128 |
|
| 2129 | if (sd->balance_interval < sd->max_interval)
|
| 2130 | sd->balance_interval *= 2;
|
| 2131 | }
|
| 2132 |
|
| 2133 | return nr_moved;
|
| 2134 |
|
| 2135 | out_balanced:
|
| 2136 | spin_unlock(&this_rq->lock);
|
| 2137 |
|
| 2138 | schedstat_inc(sd, lb_balanced[idle]);
|
| 2139 |
|
| 2140 | sd->nr_balance_failed = 0;
|
| 2141 |
|
| 2142 | if ((all_pinned && sd->balance_interval < MAX_PINNED_INTERVAL) ||
|
| 2143 | (sd->balance_interval < sd->max_interval))
|
| 2144 | sd->balance_interval *= 2;
|
| 2145 |
|
| 2146 | return 0;
|
| 2147 | }
|
| 2148 |
|
| 2149 | |
| 2150 | * Check this_cpu to ensure it is balanced within domain. Attempt to move
|
| 2151 | * tasks if there is an imbalance.
|
| 2152 | *
|
| 2153 | * Called from schedule when this_rq is about to become idle (NEWLY_IDLE).
|
| 2154 | * this_rq is locked.
|
| 2155 |
|
| 2156 | static int load_balance_newidle(int this_cpu, runqueue_t *this_rq,
|
| 2157 | struct sched_domain *sd)
|
| 2158 | {
|
| 2159 | struct sched_group *group;
|
| 2160 | runqueue_t *busiest = NULL;
|
| 2161 | unsigned long imbalance;
|
| 2162 | int nr_moved = 0;
|
| 2163 |
|
| 2164 | schedstat_inc(sd, lb_cnt[NEWLY_IDLE]);
|
| 2165 | group = find_busiest_group(sd, this_cpu, &imbalance, NEWLY_IDLE);
|
| 2166 | if (!group) {
|
| 2167 | schedstat_inc(sd, lb_nobusyg[NEWLY_IDLE]);
|
| 2168 | goto out_balanced;
|
| 2169 | }
|
| 2170 |
|
| 2171 | busiest = find_busiest_queue(group);
|
| 2172 | if (!busiest) {
|
| 2173 | schedstat_inc(sd, lb_nobusyq[NEWLY_IDLE]);
|
| 2174 | goto out_balanced;
|
| 2175 | }
|
| 2176 |
|
| 2177 | BUG_ON(busiest == this_rq);
|
| 2178 |
|
| 2179 |
|
| 2180 | double_lock_balance(this_rq, busiest);
|
| 2181 |
|
| 2182 | schedstat_add(sd, lb_imbalance[NEWLY_IDLE], imbalance);
|
| 2183 | nr_moved = move_tasks(this_rq, this_cpu, busiest,
|
| 2184 | imbalance, sd, NEWLY_IDLE, NULL);
|
| 2185 | if (!nr_moved)
|
| 2186 | schedstat_inc(sd, lb_failed[NEWLY_IDLE]);
|
| 2187 | else
|
| 2188 | sd->nr_balance_failed = 0;
|
| 2189 |
|
| 2190 | spin_unlock(&busiest->lock);
|
| 2191 | return nr_moved;
|
| 2192 |
|
| 2193 | out_balanced:
|
| 2194 | schedstat_inc(sd, lb_balanced[NEWLY_IDLE]);
|
| 2195 | sd->nr_balance_failed = 0;
|
| 2196 | return 0;
|
| 2197 | }
|
| 2198 |
|
| 2199 | |
| 2200 | * idle_balance is called by schedule() if this_cpu is about to become
|
| 2201 | * idle. Attempts to pull tasks from other CPUs.
|
| 2202 |
|
| 2203 | static inline void idle_balance(int this_cpu, runqueue_t *this_rq)
|
| 2204 | {
|
| 2205 | struct sched_domain *sd;
|
| 2206 |
|
| 2207 | for_each_domain(this_cpu, sd) {
|
| 2208 | if (sd->flags & SD_BALANCE_NEWIDLE) {
|
| 2209 | if (load_balance_newidle(this_cpu, this_rq, sd)) {
|
| 2210 |
|
| 2211 | break;
|
| 2212 | }
|
| 2213 | }
|
| 2214 | }
|
| 2215 | }
|
| 2216 |
|
| 2217 | |
| 2218 | * active_load_balance is run by migration threads. It pushes running tasks
|
| 2219 | * off the busiest CPU onto idle CPUs. It requires at least 1 task to be
|
| 2220 | * running on each physical CPU where possible, and avoids physical /
|
| 2221 | * logical imbalances.
|
| 2222 | *
|
| 2223 | * Called with busiest_rq locked.
|
| 2224 |
|
| 2225 | static void active_load_balance(runqueue_t *busiest_rq, int busiest_cpu)
|
| 2226 | {
|
| 2227 | struct sched_domain *sd;
|
| 2228 | runqueue_t *target_rq;
|
| 2229 | int target_cpu = busiest_rq->push_cpu;
|
| 2230 |
|
| 2231 | if (busiest_rq->nr_running <= 1)
|
| 2232 |
|
| 2233 | return;
|
| 2234 |
|
| 2235 | target_rq = cpu_rq(target_cpu);
|
| 2236 |
|
| 2237 | |
| 2238 | * This condition is "impossible", if it occurs
|
| 2239 | * we need to fix it. Originally reported by
|
| 2240 | * Bjorn Helgaas on a 128-cpu setup.
|
| 2241 |
|
| 2242 | BUG_ON(busiest_rq == target_rq);
|
| 2243 |
|
| 2244 |
|
| 2245 | double_lock_balance(busiest_rq, target_rq);
|
| 2246 |
|
| 2247 |
|
| 2248 | for_each_domain(target_cpu, sd)
|
| 2249 | if ((sd->flags & SD_LOAD_BALANCE) &&
|
| 2250 | cpu_isset(busiest_cpu, sd->span))
|
| 2251 | break;
|
| 2252 |
|
| 2253 | if (unlikely(sd == NULL))
|
| 2254 | goto out;
|
| 2255 |
|
| 2256 | schedstat_inc(sd, alb_cnt);
|
| 2257 |
|
| 2258 | if (move_tasks(target_rq, target_cpu, busiest_rq, 1, sd, SCHED_IDLE, NULL))
|
| 2259 | schedstat_inc(sd, alb_pushed);
|
| 2260 | else
|
| 2261 | schedstat_inc(sd, alb_failed);
|
| 2262 | out:
|
| 2263 | spin_unlock(&target_rq->lock);
|
| 2264 | }
|
| 2265 |
|
| 2266 | |
| 2267 | * rebalance_tick will get called every timer tick, on every CPU.
|
| 2268 | *
|
| 2269 | * It checks each scheduling domain to see if it is due to be balanced,
|
| 2270 | * and initiates a balancing operation if so.
|
| 2271 | *
|
| 2272 | * Balancing parameters are set up in arch_init_sched_domains.
|
| 2273 |
|
| 2274 |
|
| 2275 |
|
| 2276 | #define CPU_OFFSET(cpu) (HZ * cpu / NR_CPUS)
|
| 2277 |
|
| 2278 | static void rebalance_tick(int this_cpu, runqueue_t *this_rq,
|
| 2279 | enum idle_type idle)
|
| 2280 | {
|
| 2281 | unsigned long old_load, this_load;
|
| 2282 | unsigned long j = jiffies + CPU_OFFSET(this_cpu);
|
| 2283 | struct sched_domain *sd;
|
| 2284 | int i;
|
| 2285 |
|
| 2286 | this_load = this_rq->nr_running * SCHED_LOAD_SCALE;
|
| 2287 |
|
| 2288 | for (i = 0; i < 3; i++) {
|
| 2289 | unsigned long new_load = this_load;
|
| 2290 | int scale = 1 << i;
|
| 2291 | old_load = this_rq->cpu_load[i];
|
| 2292 | |
| 2293 | * Round up the averaging division if load is increasing. This
|
| 2294 | * prevents us from getting stuck on 9 if the load is 10, for
|
| 2295 | * example.
|
| 2296 |
|
| 2297 | if (new_load > old_load)
|
| 2298 | new_load += scale-1;
|
| 2299 | this_rq->cpu_load[i] = (old_load*(scale-1) + new_load) / scale;
|
| 2300 | }
|
| 2301 |
|
| 2302 | for_each_domain(this_cpu, sd) {
|
| 2303 | unsigned long interval;
|
| 2304 |
|
| 2305 | if (!(sd->flags & SD_LOAD_BALANCE))
|
| 2306 | continue;
|
| 2307 |
|
| 2308 | interval = sd->balance_interval;
|
| 2309 | if (idle != SCHED_IDLE)
|
| 2310 | interval *= sd->busy_factor;
|
| 2311 |
|
| 2312 |
|
| 2313 | interval = msecs_to_jiffies(interval);
|
| 2314 | if (unlikely(!interval))
|
| 2315 | interval = 1;
|
| 2316 |
|
| 2317 | if (j - sd->last_balance >= interval) {
|
| 2318 | if (load_balance(this_cpu, this_rq, sd, idle)) {
|
| 2319 |
|
| 2320 | idle = NOT_IDLE;
|
| 2321 | }
|
| 2322 | sd->last_balance += interval;
|
| 2323 | }
|
| 2324 | }
|
| 2325 | }
|
| 2326 | #else
|
| 2327 | |
| 2328 | * on UP we do not need to balance between CPUs:
|
| 2329 |
|
| 2330 | static inline void rebalance_tick(int cpu, runqueue_t *rq, enum idle_type idle)
|
| 2331 | {
|
| 2332 | }
|
| 2333 | static inline void idle_balance(int cpu, runqueue_t *rq)
|
| 2334 | {
|
| 2335 | }
|
| 2336 | #endif
|
| 2337 |
|
| 2338 | static inline int wake_priority_sleeper(runqueue_t *rq)
|
| 2339 | {
|
| 2340 | int ret = 0;
|
| 2341 | #ifdef CONFIG_SCHED_SMT
|
| 2342 | spin_lock(&rq->lock);
|
| 2343 | |
| 2344 | * If an SMT sibling task has been put to sleep for priority
|
| 2345 | * reasons reschedule the idle task to see if it can now run.
|
| 2346 |
|
| 2347 | if (rq->nr_running) {
|
| 2348 | resched_task(rq->idle);
|
| 2349 | ret = 1;
|
| 2350 | }
|
| 2351 | spin_unlock(&rq->lock);
|
| 2352 | #endif
|
| 2353 | return ret;
|
| 2354 | }
|
| 2355 |
|
| 2356 | DEFINE_PER_CPU(struct kernel_stat, kstat);
|
| 2357 |
|
| 2358 | EXPORT_PER_CPU_SYMBOL(kstat);
|
| 2359 |
|
| 2360 | |
| 2361 | * This is called on clock ticks and on context switches.
|
| 2362 | * Bank in p->sched_time the ns elapsed since the last tick or switch.
|
| 2363 |
|
| 2364 | static inline void update_cpu_clock(task_t *p, runqueue_t *rq,
|
| 2365 | unsigned long long now)
|
| 2366 | {
|
| 2367 | unsigned long long last = max(p->timestamp, rq->timestamp_last_tick);
|
| 2368 | p->sched_time += now - last;
|
| 2369 | }
|
| 2370 |
|
| 2371 | |
| 2372 | * Return current->sched_time plus any more ns on the sched_clock
|
| 2373 | * that have not yet been banked.
|
| 2374 |
|
| 2375 | unsigned long long current_sched_time(const task_t *tsk)
|
| 2376 | {
|
| 2377 | unsigned long long ns;
|
| 2378 | unsigned long flags;
|
| 2379 | local_irq_save(flags);
|
| 2380 | ns = max(tsk->timestamp, task_rq(tsk)->timestamp_last_tick);
|
| 2381 | ns = tsk->sched_time + (sched_clock() - ns);
|
| 2382 | local_irq_restore(flags);
|
| 2383 | return ns;
|
| 2384 | }
|
| 2385 |
|
| 2386 | |
| 2387 | * We place interactive tasks back into the active array, if possible.
|
| 2388 | *
|
| 2389 | * To guarantee that this does not starve expired tasks we ignore the
|
| 2390 | * interactivity of a task if the first expired task had to wait more
|
| 2391 | * than a 'reasonable' amount of time. This deadline timeout is
|
| 2392 | * load-dependent, as the frequency of array switched decreases with
|
| 2393 | * increasing number of running tasks. We also ignore the interactivity
|
| 2394 | * if a better static_prio task has expired:
|
| 2395 |
|
| 2396 | #define EXPIRED_STARVING(rq) \
|
| 2397 | ((STARVATION_LIMIT && ((rq)->expired_timestamp && \
|
| 2398 | (jiffies - (rq)->expired_timestamp >= \
|
| 2399 | STARVATION_LIMIT * ((rq)->nr_running) + 1))) || \
|
| 2400 | ((rq)->curr->static_prio > (rq)->best_expired_prio))
|
| 2401 |
|
| 2402 | |
| 2403 | * Account user cpu time to a process.
|
| 2404 | * @p: the process that the cpu time gets accounted to
|
| 2405 | * @hardirq_offset: the offset to subtract from hardirq_count()
|
| 2406 | * @cputime: the cpu time spent in user space since the last update
|
| 2407 |
|
| 2408 | void account_user_time(struct task_struct *p, cputime_t cputime)
|
| 2409 | {
|
| 2410 | struct cpu_usage_stat *cpustat = &kstat_this_cpu.cpustat;
|
| 2411 | cputime64_t tmp;
|
| 2412 |
|
| 2413 | p->utime = cputime_add(p->utime, cputime);
|
| 2414 |
|
| 2415 |
|
| 2416 | tmp = cputime_to_cputime64(cputime);
|
| 2417 | if (TASK_NICE(p) > 0)
|
| 2418 | cpustat->nice = cputime64_add(cpustat->nice, tmp);
|
| 2419 | else
|
| 2420 | cpustat->user = cputime64_add(cpustat->user, tmp);
|
| 2421 | }
|
| 2422 |
|
| 2423 | |
| 2424 | * Account system cpu time to a process.
|
| 2425 | * @p: the process that the cpu time gets accounted to
|
| 2426 | * @hardirq_offset: the offset to subtract from hardirq_count()
|
| 2427 | * @cputime: the cpu time spent in kernel space since the last update
|
| 2428 |
|
| 2429 | void account_system_time(struct task_struct *p, int hardirq_offset,
|
| 2430 | cputime_t cputime)
|
| 2431 | {
|
| 2432 | struct cpu_usage_stat *cpustat = &kstat_this_cpu.cpustat;
|
| 2433 | runqueue_t *rq = this_rq();
|
| 2434 | cputime64_t tmp;
|
| 2435 |
|
| 2436 | p->stime = cputime_add(p->stime, cputime);
|
| 2437 |
|
| 2438 |
|
| 2439 | tmp = cputime_to_cputime64(cputime);
|
| 2440 | if (hardirq_count() - hardirq_offset)
|
| 2441 | cpustat->irq = cputime64_add(cpustat->irq, tmp);
|
| 2442 | else if (softirq_count())
|
| 2443 | cpustat->softirq = cputime64_add(cpustat->softirq, tmp);
|
| 2444 | else if (p != rq->idle)
|
| 2445 | cpustat->system = cputime64_add(cpustat->system, tmp);
|
| 2446 | else if (atomic_read(&rq->nr_iowait) > 0)
|
| 2447 | cpustat->iowait = cputime64_add(cpustat->iowait, tmp);
|
| 2448 | else
|
| 2449 | cpustat->idle = cputime64_add(cpustat->idle, tmp);
|
| 2450 |
|
| 2451 | acct_update_integrals(p);
|
| 2452 |
|
| 2453 | update_mem_hiwater(p);
|
| 2454 | }
|
| 2455 |
|
| 2456 | |
| 2457 | * Account for involuntary wait time.
|
| 2458 | * @p: the process from which the cpu time has been stolen
|
| 2459 | * @steal: the cpu time spent in involuntary wait
|
| 2460 |
|
| 2461 | void account_steal_time(struct task_struct *p, cputime_t steal)
|
| 2462 | {
|
| 2463 | struct cpu_usage_stat *cpustat = &kstat_this_cpu.cpustat;
|
| 2464 | cputime64_t tmp = cputime_to_cputime64(steal);
|
| 2465 | runqueue_t *rq = this_rq();
|
| 2466 |
|
| 2467 | if (p == rq->idle) {
|
| 2468 | p->stime = cputime_add(p->stime, steal);
|
| 2469 | if (atomic_read(&rq->nr_iowait) > 0)
|
| 2470 | cpustat->iowait = cputime64_add(cpustat->iowait, tmp);
|
| 2471 | else
|
| 2472 | cpustat->idle = cputime64_add(cpustat->idle, tmp);
|
| 2473 | } else
|
| 2474 | cpustat->steal = cputime64_add(cpustat->steal, tmp);
|
| 2475 | }
|
| 2476 |
|
| 2477 | |
| 2478 | * This function gets called by the timer code, with HZ frequency.
|
| 2479 | * We call it with interrupts disabled.
|
| 2480 | *
|
| 2481 | * It also gets called by the fork code, when changing the parent's
|
| 2482 | * timeslices.
|
| 2483 |
|
| 2484 | void scheduler_tick(void)
|
| 2485 | {
|
| 2486 | int cpu = smp_processor_id();
|
| 2487 | runqueue_t *rq = this_rq();
|
| 2488 | task_t *p = current;
|
| 2489 | unsigned long long now = sched_clock();
|
| 2490 |
|
| 2491 | update_cpu_clock(p, rq, now);
|
| 2492 |
|
| 2493 | rq->timestamp_last_tick = now;
|
| 2494 |
|
| 2495 | if (p == rq->idle) {
|
| 2496 | if (wake_priority_sleeper(rq))
|
| 2497 | goto out;
|
| 2498 | rebalance_tick(cpu, rq, SCHED_IDLE);
|
| 2499 | return;
|
| 2500 | }
|
| 2501 |
|
| 2502 |
|
| 2503 | if (p->array != rq->active) {
|
| 2504 | set_tsk_need_resched(p);
|
| 2505 | goto out;
|
| 2506 | }
|
| 2507 | spin_lock(&rq->lock);
|
| 2508 | |
| 2509 | * The task was running during this tick - update the
|
| 2510 | * time slice counter. Note: we do not update a thread's
|
| 2511 | * priority until it either goes to sleep or uses up its
|
| 2512 | * timeslice. This makes it possible for interactive tasks
|
| 2513 | * to use up their timeslices at their highest priority levels.
|
| 2514 |
|
| 2515 | if (rt_task(p)) {
|
| 2516 | |
| 2517 | * RR tasks need a special form of timeslice management.
|
| 2518 | * FIFO tasks have no timeslices.
|
| 2519 |
|
| 2520 | if ((p->policy == SCHED_RR) && !--p->time_slice) {
|
| 2521 | p->time_slice = task_timeslice(p);
|
| 2522 | p->first_time_slice = 0;
|
| 2523 | set_tsk_need_resched(p);
|
| 2524 |
|
| 2525 |
|
| 2526 | requeue_task(p, rq->active);
|
| 2527 | }
|
| 2528 | goto out_unlock;
|
| 2529 | }
|
| 2530 | if (!--p->time_slice) {
|
| 2531 | dequeue_task(p, rq->active);
|
| 2532 | set_tsk_need_resched(p);
|
| 2533 | p->prio = effective_prio(p);
|
| 2534 | p->time_slice = task_timeslice(p);
|
| 2535 | p->first_time_slice = 0;
|
| 2536 |
|
| 2537 | if (!rq->expired_timestamp)
|
| 2538 | rq->expired_timestamp = jiffies;
|
| 2539 | if (!TASK_INTERACTIVE(p) || EXPIRED_STARVING(rq)) {
|
| 2540 | enqueue_task(p, rq->expired);
|
| 2541 | if (p->static_prio < rq->best_expired_prio)
|
| 2542 | rq->best_expired_prio = p->static_prio;
|
| 2543 | } else
|
| 2544 | enqueue_task(p, rq->active);
|
| 2545 | } else {
|
| 2546 | |
| 2547 | * Prevent a too long timeslice allowing a task to monopolize
|
| 2548 | * the CPU. We do this by splitting up the timeslice into
|
| 2549 | * smaller pieces.
|
| 2550 | *
|
| 2551 | * Note: this does not mean the task's timeslices expire or
|
| 2552 | * get lost in any way, they just might be preempted by
|
| 2553 | * another task of equal priority. (one with higher
|
| 2554 | * priority would have preempted this task already.) We
|
| 2555 | * requeue this task to the end of the list on this priority
|
| 2556 | * level, which is in essence a round-robin of tasks with
|
| 2557 | * equal priority.
|
| 2558 | *
|
| 2559 | * This only applies to tasks in the interactive
|
| 2560 | * delta range with at least TIMESLICE_GRANULARITY to requeue.
|
| 2561 |
|
| 2562 | if (TASK_INTERACTIVE(p) && !((task_timeslice(p) -
|
| 2563 | p->time_slice) % TIMESLICE_GRANULARITY(p)) &&
|
| 2564 | (p->time_slice >= TIMESLICE_GRANULARITY(p)) &&
|
| 2565 | (p->array == rq->active)) {
|
| 2566 |
|
| 2567 | requeue_task(p, rq->active);
|
| 2568 | set_tsk_need_resched(p);
|
| 2569 | }
|
| 2570 | }
|
| 2571 | out_unlock:
|
| 2572 | spin_unlock(&rq->lock);
|
| 2573 | out:
|
| 2574 | rebalance_tick(cpu, rq, NOT_IDLE);
|
| 2575 | }
|
| 2576 |
|
| 2577 | #ifdef CONFIG_SCHED_SMT
|
| 2578 | static inline void wake_sleeping_dependent(int this_cpu, runqueue_t *this_rq)
|
| 2579 | {
|
| 2580 | struct sched_domain *tmp, *sd = NULL;
|
| 2581 | cpumask_t sibling_map;
|
| 2582 | int i;
|
| 2583 |
|
| 2584 | for_each_domain(this_cpu, tmp)
|
| 2585 | if (tmp->flags & SD_SHARE_CPUPOWER)
|
| 2586 | sd = tmp;
|
| 2587 |
|
| 2588 | if (!sd)
|
| 2589 | return;
|
| 2590 |
|
| 2591 | |
| 2592 | * Unlock the current runqueue because we have to lock in
|
| 2593 | * CPU order to avoid deadlocks. Caller knows that we might
|
| 2594 | * unlock. We keep IRQs disabled.
|
| 2595 |
|
| 2596 | spin_unlock(&this_rq->lock);
|
| 2597 |
|
| 2598 | sibling_map = sd->span;
|
| 2599 |
|
| 2600 | for_each_cpu_mask(i, sibling_map)
|
| 2601 | spin_lock(&cpu_rq(i)->lock);
|
| 2602 | |
| 2603 | * We clear this CPU from the mask. This both simplifies the
|
| 2604 | * inner loop and keps this_rq locked when we exit:
|
| 2605 |
|
| 2606 | cpu_clear(this_cpu, sibling_map);
|
| 2607 |
|
| 2608 | for_each_cpu_mask(i, sibling_map) {
|
| 2609 | runqueue_t *smt_rq = cpu_rq(i);
|
| 2610 |
|
| 2611 | |
| 2612 | * If an SMT sibling task is sleeping due to priority
|
| 2613 | * reasons wake it up now.
|
| 2614 |
|
| 2615 | if (smt_rq->curr == smt_rq->idle && smt_rq->nr_running)
|
| 2616 | resched_task(smt_rq->idle);
|
| 2617 | }
|
| 2618 |
|
| 2619 | for_each_cpu_mask(i, sibling_map)
|
| 2620 | spin_unlock(&cpu_rq(i)->lock);
|
| 2621 | |
| 2622 | * We exit with this_cpu's rq still held and IRQs
|
| 2623 | * still disabled:
|
| 2624 |
|
| 2625 | }
|
| 2626 |
|
| 2627 | static inline int dependent_sleeper(int this_cpu, runqueue_t *this_rq)
|
| 2628 | {
|
| 2629 | struct sched_domain *tmp, *sd = NULL;
|
| 2630 | cpumask_t sibling_map;
|
| 2631 | prio_array_t *array;
|
| 2632 | int ret = 0, i;
|
| 2633 | task_t *p;
|
| 2634 |
|
| 2635 | for_each_domain(this_cpu, tmp)
|
| 2636 | if (tmp->flags & SD_SHARE_CPUPOWER)
|
| 2637 | sd = tmp;
|
| 2638 |
|
| 2639 | if (!sd)
|
| 2640 | return 0;
|
| 2641 |
|
| 2642 | |
| 2643 | * The same locking rules and details apply as for
|
| 2644 | * wake_sleeping_dependent():
|
| 2645 |
|
| 2646 | spin_unlock(&this_rq->lock);
|
| 2647 | sibling_map = sd->span;
|
| 2648 | for_each_cpu_mask(i, sibling_map)
|
| 2649 | spin_lock(&cpu_rq(i)->lock);
|
| 2650 | cpu_clear(this_cpu, sibling_map);
|
| 2651 |
|
| 2652 | |
| 2653 | * Establish next task to be run - it might have gone away because
|
| 2654 | * we released the runqueue lock above:
|
| 2655 |
|
| 2656 | if (!this_rq->nr_running)
|
| 2657 | goto out_unlock;
|
| 2658 | array = this_rq->active;
|
| 2659 | if (!array->nr_active)
|
| 2660 | array = this_rq->expired;
|
| 2661 | BUG_ON(!array->nr_active);
|
| 2662 |
|
| 2663 | p = list_entry(array->queue[sched_find_first_bit(array->bitmap)].next,
|
| 2664 | task_t, run_list);
|
| 2665 |
|
| 2666 | for_each_cpu_mask(i, sibling_map) {
|
| 2667 | runqueue_t *smt_rq = cpu_rq(i);
|
| 2668 | task_t *smt_curr = smt_rq->curr;
|
| 2669 |
|
| 2670 | |
| 2671 | * If a user task with lower static priority than the
|
| 2672 | * running task on the SMT sibling is trying to schedule,
|
| 2673 | * delay it till there is proportionately less timeslice
|
| 2674 | * left of the sibling task to prevent a lower priority
|
| 2675 | * task from using an unfair proportion of the
|
| 2676 | * physical cpu's resources. -ck
|
| 2677 |
|
| 2678 | if (((smt_curr->time_slice * (100 - sd->per_cpu_gain) / 100) >
|
| 2679 | task_timeslice(p) || rt_task(smt_curr)) &&
|
| 2680 | p->mm && smt_curr->mm && !rt_task(p))
|
| 2681 | ret = 1;
|
| 2682 |
|
| 2683 | |
| 2684 | * Reschedule a lower priority task on the SMT sibling,
|
| 2685 | * or wake it up if it has been put to sleep for priority
|
| 2686 | * reasons.
|
| 2687 |
|
| 2688 | if ((((p->time_slice * (100 - sd->per_cpu_gain) / 100) >
|
| 2689 | task_timeslice(smt_curr) || rt_task(p)) &&
|
| 2690 | smt_curr->mm && p->mm && !rt_task(smt_curr)) ||
|
| 2691 | (smt_curr == smt_rq->idle && smt_rq->nr_running))
|
| 2692 | resched_task(smt_curr);
|
| 2693 | }
|
| 2694 | out_unlock:
|
| 2695 | for_each_cpu_mask(i, sibling_map)
|
| 2696 | spin_unlock(&cpu_rq(i)->lock);
|
| 2697 | return ret;
|
| 2698 | }
|
| 2699 | #else
|
| 2700 | static inline void wake_sleeping_dependent(int this_cpu, runqueue_t *this_rq)
|
| 2701 | {
|
| 2702 | }
|
| 2703 |
|
| 2704 | static inline int dependent_sleeper(int this_cpu, runqueue_t *this_rq)
|
| 2705 | {
|
| 2706 | return 0;
|
| 2707 | }
|
| 2708 | #endif
|
| 2709 |
|
| 2710 | #if defined(CONFIG_PREEMPT) && defined(CONFIG_DEBUG_PREEMPT)
|
| 2711 |
|
| 2712 | void fastcall add_preempt_count(int val)
|
| 2713 | {
|
| 2714 | |
| 2715 | * Underflow?
|
| 2716 |
|
| 2717 | BUG_ON((preempt_count() < 0));
|
| 2718 | preempt_count() += val;
|
| 2719 | |
| 2720 | * Spinlock count overflowing soon?
|
| 2721 |
|
| 2722 | BUG_ON((preempt_count() & PREEMPT_MASK) >= PREEMPT_MASK-10);
|
| 2723 | }
|
| 2724 | EXPORT_SYMBOL(add_preempt_count);
|
| 2725 |
|
| 2726 | void fastcall sub_preempt_count(int val)
|
| 2727 | {
|
| 2728 | |
| 2729 | * Underflow?
|
| 2730 |
|
| 2731 | BUG_ON(val > preempt_count());
|
| 2732 | |
| 2733 | * Is the spinlock portion underflowing?
|
| 2734 |
|
| 2735 | BUG_ON((val < PREEMPT_MASK) && !(preempt_count() & PREEMPT_MASK));
|
| 2736 | preempt_count() -= val;
|
| 2737 | }
|
| 2738 | EXPORT_SYMBOL(sub_preempt_count);
|
| 2739 |
|
| 2740 | #endif
|
| 2741 |
|
| 2742 | |
| 2743 | * schedule() is the main scheduler function.
|
| 2744 |
|
| 2745 | asmlinkage void __sched schedule(void)
|
| 2746 | {
|
| 2747 | long *switch_count;
|
| 2748 | task_t *prev, *next;
|
| 2749 | runqueue_t *rq;
|
| 2750 | prio_array_t *array;
|
| 2751 | struct list_head *queue;
|
| 2752 | unsigned long long now;
|
| 2753 | unsigned long run_time;
|
| 2754 | int cpu, idx, new_prio;
|
| 2755 |
|
| 2756 | |
| 2757 | * Test if we are atomic. Since do_exit() needs to call into
|
| 2758 | * schedule() atomically, we ignore that path for now.
|
| 2759 | * Otherwise, whine if we are scheduling when we should not be.
|
| 2760 |
|
| 2761 | if (likely(!current->exit_state)) {
|
| 2762 | if (unlikely(in_atomic())) {
|
| 2763 | printk(KERN_ERR "scheduling while atomic: "
|
| 2764 | "%s/0x%08x/%d\n",
|
| 2765 | current->comm, preempt_count(), current->pid);
|
| 2766 | dump_stack();
|
| 2767 | }
|
| 2768 | }
|
| 2769 | profile_hit(SCHED_PROFILING, __builtin_return_address(0));
|
| 2770 |
|
| 2771 | need_resched:
|
| 2772 | preempt_disable();
|
| 2773 | prev = current;
|
| 2774 | release_kernel_lock(prev);
|
| 2775 | need_resched_nonpreemptible:
|
| 2776 | rq = this_rq();
|
| 2777 |
|
| 2778 | |
| 2779 | * The idle thread is not allowed to schedule!
|
| 2780 | * Remove this check after it has been exercised a bit.
|
| 2781 |
|
| 2782 | if (unlikely(prev == rq->idle) && prev->state != TASK_RUNNING) {
|
| 2783 | printk(KERN_ERR "bad: scheduling from the idle thread!\n");
|
| 2784 | dump_stack();
|
| 2785 | }
|
| 2786 |
|
| 2787 | schedstat_inc(rq, sched_cnt);
|
| 2788 | now = sched_clock();
|
| 2789 | if (likely((long long)(now - prev->timestamp) < NS_MAX_SLEEP_AVG)) {
|
| 2790 | run_time = now - prev->timestamp;
|
| 2791 | if (unlikely((long long)(now - prev->timestamp) < 0))
|
| 2792 | run_time = 0;
|
| 2793 | } else
|
| 2794 | run_time = NS_MAX_SLEEP_AVG;
|
| 2795 |
|
| 2796 | |
| 2797 | * Tasks charged proportionately less run_time at high sleep_avg to
|
| 2798 | * delay them losing their interactive status
|
| 2799 |
|
| 2800 | run_time /= (CURRENT_BONUS(prev) ? : 1);
|
| 2801 |
|
| 2802 | spin_lock_irq(&rq->lock);
|
| 2803 |
|
| 2804 | if (unlikely(prev->flags & PF_DEAD))
|
| 2805 | prev->state = EXIT_DEAD;
|
| 2806 |
|
| 2807 | switch_count = &prev->nivcsw;
|
| 2808 | if (prev->state && !(preempt_count() & PREEMPT_ACTIVE)) {
|
| 2809 | switch_count = &prev->nvcsw;
|
| 2810 | if (unlikely((prev->state & TASK_INTERRUPTIBLE) &&
|
| 2811 | unlikely(signal_pending(prev))))
|
| 2812 | prev->state = TASK_RUNNING;
|
| 2813 | else {
|
| 2814 | if (prev->state == TASK_UNINTERRUPTIBLE)
|
| 2815 | rq->nr_uninterruptible++;
|
| 2816 | deactivate_task(prev, rq);
|
| 2817 | }
|
| 2818 | }
|
| 2819 |
|
| 2820 | cpu = smp_processor_id();
|
| 2821 | if (unlikely(!rq->nr_running)) {
|
| 2822 | go_idle:
|
| 2823 | idle_balance(cpu, rq);
|
| 2824 | if (!rq->nr_running) {
|
| 2825 | next = rq->idle;
|
| 2826 | rq->expired_timestamp = 0;
|
| 2827 | wake_sleeping_dependent(cpu, rq);
|
| 2828 | |
| 2829 | * wake_sleeping_dependent() might have released
|
| 2830 | * the runqueue, so break out if we got new
|
| 2831 | * tasks meanwhile:
|
| 2832 |
|
| 2833 | if (!rq->nr_running)
|
| 2834 | goto switch_tasks;
|
| 2835 | }
|
| 2836 | } else {
|
| 2837 | if (dependent_sleeper(cpu, rq)) {
|
| 2838 | next = rq->idle;
|
| 2839 | goto switch_tasks;
|
| 2840 | }
|
| 2841 | |
| 2842 | * dependent_sleeper() releases and reacquires the runqueue
|
| 2843 | * lock, hence go into the idle loop if the rq went
|
| 2844 | * empty meanwhile:
|
| 2845 |
|
| 2846 | if (unlikely(!rq->nr_running))
|
| 2847 | goto go_idle;
|
| 2848 | }
|
| 2849 |
|
| 2850 | array = rq->active;
|
| 2851 | if (unlikely(!array->nr_active)) {
|
| 2852 | |
| 2853 | * Switch the active and expired arrays.
|
| 2854 |
|
| 2855 | schedstat_inc(rq, sched_switch);
|
| 2856 | rq->active = rq->expired;
|
| 2857 | rq->expired = array;
|
| 2858 | array = rq->active;
|
| 2859 | rq->expired_timestamp = 0;
|
| 2860 | rq->best_expired_prio = MAX_PRIO;
|
| 2861 | }
|
| 2862 |
|
| 2863 | idx = sched_find_first_bit(array->bitmap);
|
| 2864 | queue = array->queue + idx;
|
| 2865 | next = list_entry(queue->next, task_t, run_list);
|
| 2866 |
|
| 2867 | if (!rt_task(next) && next->activated > 0) {
|
| 2868 | unsigned long long delta = now - next->timestamp;
|
| 2869 | if (unlikely((long long)(now - next->timestamp) < 0))
|
| 2870 | delta = 0;
|
| 2871 |
|
| 2872 | if (next->activated == 1)
|
| 2873 | delta = delta * (ON_RUNQUEUE_WEIGHT * 128 / 100) / 128;
|
| 2874 |
|
| 2875 | array = next->array;
|
| 2876 | new_prio = recalc_task_prio(next, next->timestamp + delta);
|
| 2877 |
|
| 2878 | if (unlikely(next->prio != new_prio)) {
|
| 2879 | dequeue_task(next, array);
|
| 2880 | next->prio = new_prio;
|
| 2881 | enqueue_task(next, array);
|
| 2882 | } else
|
| 2883 | requeue_task(next, array);
|
| 2884 | }
|
| 2885 | next->activated = 0;
|
| 2886 | switch_tasks:
|
| 2887 | if (next == rq->idle)
|
| 2888 | schedstat_inc(rq, sched_goidle);
|
| 2889 | prefetch(next);
|
| 2890 | clear_tsk_need_resched(prev);
|
| 2891 | rcu_qsctr_inc(task_cpu(prev));
|
| 2892 |
|
| 2893 | update_cpu_clock(prev, rq, now);
|
| 2894 |
|
| 2895 | prev->sleep_avg -= run_time;
|
| 2896 | if ((long)prev->sleep_avg <= 0)
|
| 2897 | prev->sleep_avg = 0;
|
| 2898 | prev->timestamp = prev->last_ran = now;
|
| 2899 |
|
| 2900 | sched_info_switch(prev, next);
|
| 2901 | if (likely(prev != next)) {
|
| 2902 | next->timestamp = now;
|
| 2903 | rq->nr_switches++;
|
| 2904 | rq->curr = next;
|
| 2905 | ++*switch_count;
|
| 2906 |
|
| 2907 | prepare_task_switch(rq, next);
|
| 2908 | prev = context_switch(rq, prev, next);
|
| 2909 | barrier();
|
| 2910 | |
| 2911 | * this_rq must be evaluated again because prev may have moved
|
| 2912 | * CPUs since it called schedule(), thus the 'rq' on its stack
|
| 2913 | * frame will be invalid.
|
| 2914 |
|
| 2915 | finish_task_switch(this_rq(), prev);
|
| 2916 | } else
|
| 2917 | spin_unlock_irq(&rq->lock);
|
| 2918 |
|
| 2919 | prev = current;
|
| 2920 | if (unlikely(reacquire_kernel_lock(prev) < 0))
|
| 2921 | goto need_resched_nonpreemptible;
|
| 2922 | preempt_enable_no_resched();
|
| 2923 | if (unlikely(test_thread_flag(TIF_NEED_RESCHED)))
|
| 2924 | goto need_resched;
|
| 2925 | }
|
| 2926 |
|
| 2927 | EXPORT_SYMBOL(schedule);
|
| 2928 |
|
| 2929 | #ifdef CONFIG_PREEMPT
|
| 2930 | |
| 2931 | * this is is the entry point to schedule() from in-kernel preemption
|
| 2932 | * off of preempt_enable. Kernel preemptions off return from interrupt
|
| 2933 | * occur there and call schedule directly.
|
| 2934 |
|
| 2935 | asmlinkage void __sched preempt_schedule(void)
|
| 2936 | {
|
| 2937 | struct thread_info *ti = current_thread_info();
|
| 2938 | #ifdef CONFIG_PREEMPT_BKL
|
| 2939 | struct task_struct *task = current;
|
| 2940 | int saved_lock_depth;
|
| 2941 | #endif
|
| 2942 | |
| 2943 | * If there is a non-zero preempt_count or interrupts are disabled,
|
| 2944 | * we do not want to preempt the current task. Just return..
|
| 2945 |
|
| 2946 | if (unlikely(ti->preempt_count || irqs_disabled()))
|
| 2947 | return;
|
| 2948 |
|
| 2949 | need_resched:
|
| 2950 | add_preempt_count(PREEMPT_ACTIVE);
|
| 2951 | |
| 2952 | * We keep the big kernel semaphore locked, but we
|
| 2953 | * clear ->lock_depth so that schedule() doesnt
|
| 2954 | * auto-release the semaphore:
|
| 2955 |
|
| 2956 | #ifdef CONFIG_PREEMPT_BKL
|
| 2957 | saved_lock_depth = task->lock_depth;
|
| 2958 | task->lock_depth = -1;
|
| 2959 | #endif
|
| 2960 | schedule();
|
| 2961 | #ifdef CONFIG_PREEMPT_BKL
|
| 2962 | task->lock_depth = saved_lock_depth;
|
| 2963 | #endif
|
| 2964 | sub_preempt_count(PREEMPT_ACTIVE);
|
| 2965 |
|
| 2966 |
|
| 2967 | barrier();
|
| 2968 | if (unlikely(test_thread_flag(TIF_NEED_RESCHED)))
|
| 2969 | goto need_resched;
|
| 2970 | }
|
| 2971 |
|
| 2972 | EXPORT_SYMBOL(preempt_schedule);
|
| 2973 |
|
| 2974 | |
| 2975 | * this is is the entry point to schedule() from kernel preemption
|
| 2976 | * off of irq context.
|
| 2977 | * Note, that this is called and return with irqs disabled. This will
|
| 2978 | * protect us against recursive calling from irq.
|
| 2979 |
|
| 2980 | asmlinkage void __sched preempt_schedule_irq(void)
|
| 2981 | {
|
| 2982 | struct thread_info *ti = current_thread_info();
|
| 2983 | #ifdef CONFIG_PREEMPT_BKL
|
| 2984 | struct task_struct *task = current;
|
| 2985 | int saved_lock_depth;
|
| 2986 | #endif
|
| 2987 |
|
| 2988 | BUG_ON(ti->preempt_count || !irqs_disabled());
|
| 2989 |
|
| 2990 | need_resched:
|
| 2991 | add_preempt_count(PREEMPT_ACTIVE);
|
| 2992 | |
| 2993 | * We keep the big kernel semaphore locked, but we
|
| 2994 | * clear ->lock_depth so that schedule() doesnt
|
| 2995 | * auto-release the semaphore:
|
| 2996 |
|
| 2997 | #ifdef CONFIG_PREEMPT_BKL
|
| 2998 | saved_lock_depth = task->lock_depth;
|
| 2999 | task->lock_depth = -1;
|
| 3000 | #endif
|
| 3001 | local_irq_enable();
|
| 3002 | schedule();
|
| 3003 | local_irq_disable();
|
| 3004 | #ifdef CONFIG_PREEMPT_BKL
|
| 3005 | task->lock_depth = saved_lock_depth;
|
| 3006 | #endif
|
| 3007 | sub_preempt_count(PREEMPT_ACTIVE);
|
| 3008 |
|
| 3009 |
|
| 3010 | barrier();
|
| 3011 | if (unlikely(test_thread_flag(TIF_NEED_RESCHED)))
|
| 3012 | goto need_resched;
|
| 3013 | }
|
| 3014 |
|
| 3015 | #endif
|
| 3016 |
|
| 3017 | int default_wake_function(wait_queue_t *curr, unsigned mode, int sync, void *key)
|
| 3018 | {
|
| 3019 | task_t *p = curr->private;
|
| 3020 | return try_to_wake_up(p, mode, sync);
|
| 3021 | }
|
| 3022 |
|
| 3023 | EXPORT_SYMBOL(default_wake_function);
|
| 3024 |
|
| 3025 | |
| 3026 | * The core wakeup function. Non-exclusive wakeups (nr_exclusive == 0) just
|
| 3027 | * wake everything up. If it's an exclusive wakeup (nr_exclusive == small +ve
|
| 3028 | * number) then we wake all the non-exclusive tasks and one exclusive task.
|
| 3029 | *
|
| 3030 | * There are circumstances in which we can try to wake a task which has already
|
| 3031 | * started to run but is not in state TASK_RUNNING. try_to_wake_up() returns
|
| 3032 | * zero in this (rare) case, and we handle it by continuing to scan the queue.
|
| 3033 |
|
| 3034 | static void __wake_up_common(wait_queue_head_t *q, unsigned int mode,
|
| 3035 | int nr_exclusive, int sync, void *key)
|
| 3036 | {
|
| 3037 | struct list_head *tmp, *next;
|
| 3038 |
|
| 3039 | list_for_each_safe(tmp, next, &q->task_list) {
|
| 3040 | wait_queue_t *curr;
|
| 3041 | unsigned flags;
|
| 3042 | curr = list_entry(tmp, wait_queue_t, task_list);
|
| 3043 | flags = curr->flags;
|
| 3044 | if (curr->func(curr, mode, sync, key) &&
|
| 3045 | (flags & WQ_FLAG_EXCLUSIVE) &&
|
| 3046 | !--nr_exclusive)
|
| 3047 | break;
|
| 3048 | }
|
| 3049 | }
|
| 3050 |
|
| 3051 | |
| 3052 | * __wake_up - wake up threads blocked on a waitqueue.
|
| 3053 | * @q: the waitqueue
|
| 3054 | * @mode: which threads
|
| 3055 | * @nr_exclusive: how many wake-one or wake-many threads to wake up
|
| 3056 | * @key: is directly passed to the wakeup function
|
| 3057 |
|
| 3058 | void fastcall __wake_up(wait_queue_head_t *q, unsigned int mode,
|
| 3059 | int nr_exclusive, void *key)
|
| 3060 | {
|
| 3061 | unsigned long flags;
|
| 3062 |
|
| 3063 | spin_lock_irqsave(&q->lock, flags);
|
| 3064 | __wake_up_common(q, mode, nr_exclusive, 0, key);
|
| 3065 | spin_unlock_irqrestore(&q->lock, flags);
|
| 3066 | }
|
| 3067 |
|
| 3068 | EXPORT_SYMBOL(__wake_up);
|
| 3069 |
|
| 3070 | |
| 3071 | * Same as __wake_up but called with the spinlock in wait_queue_head_t held.
|
| 3072 |
|
| 3073 | void fastcall __wake_up_locked(wait_queue_head_t *q, unsigned int mode)
|
| 3074 | {
|
| 3075 | __wake_up_common(q, mode, 1, 0, NULL);
|
| 3076 | }
|
| 3077 |
|
| 3078 | |
| 3079 | * __wake_up_sync - wake up threads blocked on a waitqueue.
|
| 3080 | * @q: the waitqueue
|
| 3081 | * @mode: which threads
|
| 3082 | * @nr_exclusive: how many wake-one or wake-many threads to wake up
|
| 3083 | *
|
| 3084 | * The sync wakeup differs that the waker knows that it will schedule
|
| 3085 | * away soon, so while the target thread will be woken up, it will not
|
| 3086 | * be migrated to another CPU - ie. the two threads are 'synchronized'
|
| 3087 | * with each other. This can prevent needless bouncing between CPUs.
|
| 3088 | *
|
| 3089 | * On UP it can prevent extra preemption.
|
| 3090 |
|
| 3091 | void fastcall __wake_up_sync(wait_queue_head_t *q, unsigned int mode, int nr_exclusive)
|
| 3092 | {
|
| 3093 | unsigned long flags;
|
| 3094 | int sync = 1;
|
| 3095 |
|
| 3096 | if (unlikely(!q))
|
| 3097 | return;
|
| 3098 |
|
| 3099 | if (unlikely(!nr_exclusive))
|
| 3100 | sync = 0;
|
| 3101 |
|
| 3102 | spin_lock_irqsave(&q->lock, flags);
|
| 3103 | __wake_up_common(q, mode, nr_exclusive, sync, NULL);
|
| 3104 | spin_unlock_irqrestore(&q->lock, flags);
|
| 3105 | }
|
| 3106 | EXPORT_SYMBOL_GPL(__wake_up_sync);
|
| 3107 |
|
| 3108 | void fastcall complete(struct completion *x)
|
| 3109 | {
|
| 3110 | unsigned long flags;
|
| 3111 |
|
| 3112 | spin_lock_irqsave(&x->wait.lock, flags);
|
| 3113 | x->done++;
|
| 3114 | __wake_up_common(&x->wait, TASK_UNINTERRUPTIBLE | TASK_INTERRUPTIBLE,
|
| 3115 | 1, 0, NULL);
|
| 3116 | spin_unlock_irqrestore(&x->wait.lock, flags);
|
| 3117 | }
|
| 3118 | EXPORT_SYMBOL(complete);
|
| 3119 |
|
| 3120 | void fastcall complete_all(struct completion *x)
|
| 3121 | {
|
| 3122 | unsigned long flags;
|
| 3123 |
|
| 3124 | spin_lock_irqsave(&x->wait.lock, flags);
|
| 3125 | x->done += UINT_MAX/2;
|
| 3126 | __wake_up_common(&x->wait, TASK_UNINTERRUPTIBLE | TASK_INTERRUPTIBLE,
|
| 3127 | 0, 0, NULL);
|
| 3128 | spin_unlock_irqrestore(&x->wait.lock, flags);
|
| 3129 | }
|
| 3130 | EXPORT_SYMBOL(complete_all);
|
| 3131 |
|
| 3132 | void fastcall __sched wait_for_completion(struct completion *x)
|
| 3133 | {
|
| 3134 | might_sleep();
|
| 3135 | spin_lock_irq(&x->wait.lock);
|
| 3136 | if (!x->done) {
|
| 3137 | DECLARE_WAITQUEUE(wait, current);
|
| 3138 |
|
| 3139 | wait.flags |= WQ_FLAG_EXCLUSIVE;
|
| 3140 | __add_wait_queue_tail(&x->wait, &wait);
|
| 3141 | do {
|
| 3142 | __set_current_state(TASK_UNINTERRUPTIBLE);
|
| 3143 | spin_unlock_irq(&x->wait.lock);
|
| 3144 | schedule();
|
| 3145 | spin_lock_irq(&x->wait.lock);
|
| 3146 | } while (!x->done);
|
| 3147 | __remove_wait_queue(&x->wait, &wait);
|
| 3148 | }
|
| 3149 | x->done--;
|
| 3150 | spin_unlock_irq(&x->wait.lock);
|
| 3151 | }
|
| 3152 | EXPORT_SYMBOL(wait_for_completion);
|
| 3153 |
|
| 3154 | unsigned long fastcall __sched
|
| 3155 | wait_for_completion_timeout(struct completion *x, unsigned long timeout)
|
| 3156 | {
|
| 3157 | might_sleep();
|
| 3158 |
|
| 3159 | spin_lock_irq(&x->wait.lock);
|
| 3160 | if (!x->done) {
|
| 3161 | DECLARE_WAITQUEUE(wait, current);
|
| 3162 |
|
| 3163 | wait.flags |= WQ_FLAG_EXCLUSIVE;
|
| 3164 | __add_wait_queue_tail(&x->wait, &wait);
|
| 3165 | do {
|
| 3166 | __set_current_state(TASK_UNINTERRUPTIBLE);
|
| 3167 | spin_unlock_irq(&x->wait.lock);
|
| 3168 | timeout = schedule_timeout(timeout);
|
| 3169 | spin_lock_irq(&x->wait.lock);
|
| 3170 | if (!timeout) {
|
| 3171 | __remove_wait_queue(&x->wait, &wait);
|
| 3172 | goto out;
|
| 3173 | }
|
| 3174 | } while (!x->done);
|
| 3175 | __remove_wait_queue(&x->wait, &wait);
|
| 3176 | }
|
| 3177 | x->done--;
|
| 3178 | out:
|
| 3179 | spin_unlock_irq(&x->wait.lock);
|
| 3180 | return timeout;
|
| 3181 | }
|
| 3182 | EXPORT_SYMBOL(wait_for_completion_timeout);
|
| 3183 |
|
| 3184 | int fastcall __sched wait_for_completion_interruptible(struct completion *x)
|
| 3185 | {
|
| 3186 | int ret = 0;
|
| 3187 |
|
| 3188 | might_sleep();
|
| 3189 |
|
| 3190 | spin_lock_irq(&x->wait.lock);
|
| 3191 | if (!x->done) {
|
| 3192 | DECLARE_WAITQUEUE(wait, current);
|
| 3193 |
|
| 3194 | wait.flags |= WQ_FLAG_EXCLUSIVE;
|
| 3195 | __add_wait_queue_tail(&x->wait, &wait);
|
| 3196 | do {
|
| 3197 | if (signal_pending(current)) {
|
| 3198 | ret = -ERESTARTSYS;
|
| 3199 | __remove_wait_queue(&x->wait, &wait);
|
| 3200 | goto out;
|
| 3201 | }
|
| 3202 | __set_current_state(TASK_INTERRUPTIBLE);
|
| 3203 | spin_unlock_irq(&x->wait.lock);
|
| 3204 | schedule();
|
| 3205 | spin_lock_irq(&x->wait.lock);
|
| 3206 | } while (!x->done);
|
| 3207 | __remove_wait_queue(&x->wait, &wait);
|
| 3208 | }
|
| 3209 | x->done--;
|
| 3210 | out:
|
| 3211 | spin_unlock_irq(&x->wait.lock);
|
| 3212 |
|
| 3213 | return ret;
|
| 3214 | }
|
| 3215 | EXPORT_SYMBOL(wait_for_completion_interruptible);
|
| 3216 |
|
| 3217 | unsigned long fastcall __sched
|
| 3218 | wait_for_completion_interruptible_timeout(struct completion *x,
|
| 3219 | unsigned long timeout)
|
| 3220 | {
|
| 3221 | might_sleep();
|
| 3222 |
|
| 3223 | spin_lock_irq(&x->wait.lock);
|
| 3224 | if (!x->done) {
|
| 3225 | DECLARE_WAITQUEUE(wait, current);
|
| 3226 |
|
| 3227 | wait.flags |= WQ_FLAG_EXCLUSIVE;
|
| 3228 | __add_wait_queue_tail(&x->wait, &wait);
|
| 3229 | do {
|
| 3230 | if (signal_pending(current)) {
|
| 3231 | timeout = -ERESTARTSYS;
|
| 3232 | __remove_wait_queue(&x->wait, &wait);
|
| 3233 | goto out;
|
| 3234 | }
|
| 3235 | __set_current_state(TASK_INTERRUPTIBLE);
|
| 3236 | spin_unlock_irq(&x->wait.lock);
|
| 3237 | timeout = schedule_timeout(timeout);
|
| 3238 | spin_lock_irq(&x->wait.lock);
|
| 3239 | if (!timeout) {
|
| 3240 | __remove_wait_queue(&x->wait, &wait);
|
| 3241 | goto out;
|
| 3242 | }
|
| 3243 | } while (!x->done);
|
| 3244 | __remove_wait_queue(&x->wait, &wait);
|
| 3245 | }
|
| 3246 | x->done--;
|
| 3247 | out:
|
| 3248 | spin_unlock_irq(&x->wait.lock);
|
| 3249 | return timeout;
|
| 3250 | }
|
| 3251 | EXPORT_SYMBOL(wait_for_completion_interruptible_timeout);
|
| 3252 |
|
| 3253 |
|
| 3254 | #define SLEEP_ON_VAR \
|
| 3255 | unsigned long flags; \
|
| 3256 | wait_queue_t wait; \
|
| 3257 | init_waitqueue_entry(&wait, current);
|
| 3258 |
|
| 3259 | #define SLEEP_ON_HEAD \
|
| 3260 | spin_lock_irqsave(&q->lock,flags); \
|
| 3261 | __add_wait_queue(q, &wait); \
|
| 3262 | spin_unlock(&q->lock);
|
| 3263 |
|
| 3264 | #define SLEEP_ON_TAIL \
|
| 3265 | spin_lock_irq(&q->lock); \
|
| 3266 | __remove_wait_queue(q, &wait); \
|
| 3267 | spin_unlock_irqrestore(&q->lock, flags);
|
| 3268 |
|
| 3269 | void fastcall __sched interruptible_sleep_on(wait_queue_head_t *q)
|
| 3270 | {
|
| 3271 | SLEEP_ON_VAR
|
| 3272 |
|
| 3273 | current->state = TASK_INTERRUPTIBLE;
|
| 3274 |
|
| 3275 | SLEEP_ON_HEAD
|
| 3276 | schedule();
|
| 3277 | SLEEP_ON_TAIL
|
| 3278 | }
|
| 3279 |
|
| 3280 | EXPORT_SYMBOL(interruptible_sleep_on);
|
| 3281 |
|
| 3282 | long fastcall __sched interruptible_sleep_on_timeout(wait_queue_head_t *q, long timeout)
|
| 3283 | {
|
| 3284 | SLEEP_ON_VAR
|
| 3285 |
|
| 3286 | current->state = TASK_INTERRUPTIBLE;
|
| 3287 |
|
| 3288 | SLEEP_ON_HEAD
|
| 3289 | timeout = schedule_timeout(timeout);
|
| 3290 | SLEEP_ON_TAIL
|
| 3291 |
|
| 3292 | return timeout;
|
| 3293 | }
|
| 3294 |
|
| 3295 | EXPORT_SYMBOL(interruptible_sleep_on_timeout);
|
| 3296 |
|
| 3297 | void fastcall __sched sleep_on(wait_queue_head_t *q)
|
| 3298 | {
|
| 3299 | SLEEP_ON_VAR
|
| 3300 |
|
| 3301 | current->state = TASK_UNINTERRUPTIBLE;
|
| 3302 |
|
| 3303 | SLEEP_ON_HEAD
|
| 3304 | schedule();
|
| 3305 | SLEEP_ON_TAIL
|
| 3306 | }
|
| 3307 |
|
| 3308 | EXPORT_SYMBOL(sleep_on);
|
| 3309 |
|
| 3310 | long fastcall __sched sleep_on_timeout(wait_queue_head_t *q, long timeout)
|
| 3311 | {
|
| 3312 | SLEEP_ON_VAR
|
| 3313 |
|
| 3314 | current->state = TASK_UNINTERRUPTIBLE;
|
| 3315 |
|
| 3316 | SLEEP_ON_HEAD
|
| 3317 | timeout = schedule_timeout(timeout);
|
| 3318 | SLEEP_ON_TAIL
|
| 3319 |
|
| 3320 | return timeout;
|
| 3321 | }
|
| 3322 |
|
| 3323 | EXPORT_SYMBOL(sleep_on_timeout);
|
| 3324 |
|
| 3325 | void set_user_nice(task_t *p, long nice)
|
| 3326 | {
|
| 3327 | unsigned long flags;
|
| 3328 | prio_array_t *array;
|
| 3329 | runqueue_t *rq;
|
| 3330 | int old_prio, new_prio, delta;
|
| 3331 |
|
| 3332 | if (TASK_NICE(p) == nice || nice < -20 || nice > 19)
|
| 3333 | return;
|
| 3334 | |
| 3335 | * We have to be careful, if called from sys_setpriority(),
|
| 3336 | * the task might be in the middle of scheduling on another CPU.
|
| 3337 |
|
| 3338 | rq = task_rq_lock(p, &flags);
|
| 3339 | |
| 3340 | * The RT priorities are set via sched_setscheduler(), but we still
|
| 3341 | * allow the 'normal' nice value to be set - but as expected
|
| 3342 | * it wont have any effect on scheduling until the task is
|
| 3343 | * not SCHED_NORMAL:
|
| 3344 |
|
| 3345 | if (rt_task(p)) {
|
| 3346 | p->static_prio = NICE_TO_PRIO(nice);
|
| 3347 | goto out_unlock;
|
| 3348 | }
|
| 3349 | array = p->array;
|
| 3350 | if (array)
|
| 3351 | dequeue_task(p, array);
|
| 3352 |
|
| 3353 | old_prio = p->prio;
|
| 3354 | new_prio = NICE_TO_PRIO(nice);
|
| 3355 | delta = new_prio - old_prio;
|
| 3356 | p->static_prio = NICE_TO_PRIO(nice);
|
| 3357 | p->prio += delta;
|
| 3358 |
|
| 3359 | if (array) {
|
| 3360 | enqueue_task(p, array);
|
| 3361 | |
| 3362 | * If the task increased its priority or is running and
|
| 3363 | * lowered its priority, then reschedule its CPU:
|
| 3364 |
|
| 3365 | if (delta < 0 || (delta > 0 && task_running(rq, p)))
|
| 3366 | resched_task(rq->curr);
|
| 3367 | }
|
| 3368 | out_unlock:
|
| 3369 | task_rq_unlock(rq, &flags);
|
| 3370 | }
|
| 3371 |
|
| 3372 | EXPORT_SYMBOL(set_user_nice);
|
| 3373 |
|
| 3374 | |
| 3375 | * can_nice - check if a task can reduce its nice value
|
| 3376 | * @p: task
|
| 3377 | * @nice: nice value
|
| 3378 |
|
| 3379 | int can_nice(const task_t *p, const int nice)
|
| 3380 | {
|
| 3381 |
|
| 3382 | int nice_rlim = 20 - nice;
|
| 3383 | return (nice_rlim <= p->signal->rlim[RLIMIT_NICE].rlim_cur ||
|
| 3384 | capable(CAP_SYS_NICE));
|
| 3385 | }
|
| 3386 |
|
| 3387 | #ifdef __ARCH_WANT_SYS_NICE
|
| 3388 |
|
| 3389 | |
| 3390 | * sys_nice - change the priority of the current process.
|
| 3391 | * @increment: priority increment
|
| 3392 | *
|
| 3393 | * sys_setpriority is a more generic, but much slower function that
|
| 3394 | * does similar things.
|
| 3395 |
|
| 3396 | asmlinkage long sys_nice(int increment)
|
| 3397 | {
|
| 3398 | int retval;
|
| 3399 | long nice;
|
| 3400 |
|
| 3401 | |
| 3402 | * Setpriority might change our priority at the same moment.
|
| 3403 | * We don't have to worry. Conceptually one call occurs first
|
| 3404 | * and we have a single winner.
|
| 3405 |
|
| 3406 | if (increment < -40)
|
| 3407 | increment = -40;
|
| 3408 | if (increment > 40)
|
| 3409 | increment = 40;
|
| 3410 |
|
| 3411 | nice = PRIO_TO_NICE(current->static_prio) + increment;
|
| 3412 | if (nice < -20)
|
| 3413 | nice = -20;
|
| 3414 | if (nice > 19)
|
| 3415 | nice = 19;
|
| 3416 |
|
| 3417 | if (increment < 0 && !can_nice(current, nice))
|
| 3418 | return -EPERM;
|
| 3419 |
|
| 3420 | retval = security_task_setnice(current, nice);
|
| 3421 | if (retval)
|
| 3422 | return retval;
|
| 3423 |
|
| 3424 | set_user_nice(current, nice);
|
| 3425 | return 0;
|
| 3426 | }
|
| 3427 |
|
| 3428 | #endif
|
| 3429 |
|
| 3430 | |
| 3431 | * task_prio - return the priority value of a given task.
|
| 3432 | * @p: the task in question.
|
| 3433 | *
|
| 3434 | * This is the priority value as seen by users in /proc.
|
| 3435 | * RT tasks are offset by -200. Normal tasks are centered
|
| 3436 | * around 0, value goes from -16 to +15.
|
| 3437 |
|
| 3438 | int task_prio(const task_t *p)
|
| 3439 | {
|
| 3440 | return p->prio - MAX_RT_PRIO;
|
| 3441 | }
|
| 3442 |
|
| 3443 | |
| 3444 | * task_nice - return the nice value of a given task.
|
| 3445 | * @p: the task in question.
|
| 3446 |
|
| 3447 | int task_nice(const task_t *p)
|
| 3448 | {
|
| 3449 | return TASK_NICE(p);
|
| 3450 | }
|
| 3451 | EXPORT_SYMBOL_GPL(task_nice);
|
| 3452 |
|
| 3453 | |
| 3454 | * idle_cpu - is a given cpu idle currently?
|
| 3455 | * @cpu: the processor in question.
|
| 3456 |
|
| 3457 | int idle_cpu(int cpu)
|
| 3458 | {
|
| 3459 | return cpu_curr(cpu) == cpu_rq(cpu)->idle;
|
| 3460 | }
|
| 3461 |
|
| 3462 | EXPORT_SYMBOL_GPL(idle_cpu);
|
| 3463 |
|
| 3464 | |
| 3465 | * idle_task - return the idle task for a given cpu.
|
| 3466 | * @cpu: the processor in question.
|
| 3467 |
|
| 3468 | task_t *idle_task(int cpu)
|
| 3469 | {
|
| 3470 | return cpu_rq(cpu)->idle;
|
| 3471 | }
|
| 3472 |
|
| 3473 | |
| 3474 | * find_process_by_pid - find a process with a matching PID value.
|
| 3475 | * @pid: the pid in question.
|
| 3476 |
|
| 3477 | static inline task_t *find_process_by_pid(pid_t pid)
|
| 3478 | {
|
| 3479 | return pid ? find_task_by_pid(pid) : current;
|
| 3480 | }
|
| 3481 |
|
| 3482 |
|
| 3483 | static void __setscheduler(struct task_struct *p, int policy, int prio)
|
| 3484 | {
|
| 3485 | BUG_ON(p->array);
|
| 3486 | p->policy = policy;
|
| 3487 | p->rt_priority = prio;
|
| 3488 | if (policy != SCHED_NORMAL)
|
| 3489 | p->prio = MAX_RT_PRIO-1 - p->rt_priority;
|
| 3490 | else
|
| 3491 | p->prio = p->static_prio;
|
| 3492 | }
|
| 3493 |
|
| 3494 | |
| 3495 | * sched_setscheduler - change the scheduling policy and/or RT priority of
|
| 3496 | * a thread.
|
| 3497 | * @p: the task in question.
|
| 3498 | * @policy: new policy.
|
| 3499 | * @param: structure containing the new RT priority.
|
| 3500 |
|
| 3501 | int sched_setscheduler(struct task_struct *p, int policy, struct sched_param *param)
|
| 3502 | {
|
| 3503 | int retval;
|
| 3504 | int oldprio, oldpolicy = -1;
|
| 3505 | prio_array_t *array;
|
| 3506 | unsigned long flags;
|
| 3507 | runqueue_t *rq;
|
| 3508 |
|
| 3509 | recheck:
|
| 3510 |
|
| 3511 | if (policy < 0)
|
| 3512 | policy = oldpolicy = p->policy;
|
| 3513 | else if (policy != SCHED_FIFO && policy != SCHED_RR &&
|
| 3514 | policy != SCHED_NORMAL)
|
| 3515 | return -EINVAL;
|
| 3516 | |
| 3517 | * Valid priorities for SCHED_FIFO and SCHED_RR are
|
| 3518 | * 1..MAX_USER_RT_PRIO-1, valid priority for SCHED_NORMAL is 0.
|
| 3519 |
|
| 3520 | if (param->sched_priority < 0 ||
|
| 3521 | (p->mm && param->sched_priority > MAX_USER_RT_PRIO-1) ||
|
| 3522 | (!p->mm && param->sched_priority > MAX_RT_PRIO-1))
|
| 3523 | return -EINVAL;
|
| 3524 | if ((policy == SCHED_NORMAL) != (param->sched_priority == 0))
|
| 3525 | return -EINVAL;
|
| 3526 |
|
| 3527 | |
| 3528 | * Allow unprivileged RT tasks to decrease priority:
|
| 3529 |
|
| 3530 | if (!capable(CAP_SYS_NICE)) {
|
| 3531 |
|
| 3532 | if (policy != p->policy &&
|
| 3533 | !p->signal->rlim[RLIMIT_RTPRIO].rlim_cur)
|
| 3534 | return -EPERM;
|
| 3535 |
|
| 3536 | if (policy != SCHED_NORMAL &&
|
| 3537 | param->sched_priority > p->rt_priority &&
|
| 3538 | param->sched_priority >
|
| 3539 | p->signal->rlim[RLIMIT_RTPRIO].rlim_cur)
|
| 3540 | return -EPERM;
|
| 3541 |
|
| 3542 | if ((current->euid != p->euid) &&
|
| 3543 | (current->euid != p->uid))
|
| 3544 | return -EPERM;
|
| 3545 | }
|
| 3546 |
|
| 3547 | retval = security_task_setscheduler(p, policy, param);
|
| 3548 | if (retval)
|
| 3549 | return retval;
|
| 3550 | |
| 3551 | * To be able to change p->policy safely, the apropriate
|
| 3552 | * runqueue lock must be held.
|
| 3553 |
|
| 3554 | rq = task_rq_lock(p, &flags);
|
| 3555 |
|
| 3556 | if (unlikely(oldpolicy != -1 && oldpolicy != p->policy)) {
|
| 3557 | policy = oldpolicy = -1;
|
| 3558 | task_rq_unlock(rq, &flags);
|
| 3559 | goto recheck;
|
| 3560 | }
|
| 3561 | array = p->array;
|
| 3562 | if (array)
|
| 3563 | deactivate_task(p, rq);
|
| 3564 | oldprio = p->prio;
|
| 3565 | __setscheduler(p, policy, param->sched_priority);
|
| 3566 | if (array) {
|
| 3567 | __activate_task(p, rq);
|
| 3568 | |
| 3569 | * Reschedule if we are currently running on this runqueue and
|
| 3570 | * our priority decreased, or if we are not currently running on
|
| 3571 | * this runqueue and our priority is higher than the current's
|
| 3572 |
|
| 3573 | if (task_running(rq, p)) {
|
| 3574 | if (p->prio > oldprio)
|
| 3575 | resched_task(rq->curr);
|
| 3576 | } else if (TASK_PREEMPTS_CURR(p, rq))
|
| 3577 | resched_task(rq->curr);
|
| 3578 | }
|
| 3579 | task_rq_unlock(rq, &flags);
|
| 3580 | return 0;
|
| 3581 | }
|
| 3582 | EXPORT_SYMBOL_GPL(sched_setscheduler);
|
| 3583 |
|
| 3584 | static int do_sched_setscheduler(pid_t pid, int policy, struct sched_param __user *param)
|
| 3585 | {
|
| 3586 | int retval;
|
| 3587 | struct sched_param lparam;
|
| 3588 | struct task_struct *p;
|
| 3589 |
|
| 3590 | if (!param || pid < 0)
|
| 3591 | return -EINVAL;
|
| 3592 | if (copy_from_user(&lparam, param, sizeof(struct sched_param)))
|
| 3593 | return -EFAULT;
|
| 3594 | read_lock_irq(&tasklist_lock);
|
| 3595 | p = find_process_by_pid(pid);
|
| 3596 | if (!p) {
|
| 3597 | read_unlock_irq(&tasklist_lock);
|
| 3598 | return -ESRCH;
|
| 3599 | }
|
| 3600 | retval = sched_setscheduler(p, policy, &lparam);
|
| 3601 | read_unlock_irq(&tasklist_lock);
|
| 3602 | return retval;
|
| 3603 | }
|
| 3604 |
|
| 3605 | |
| 3606 | * sys_sched_setscheduler - set/change the scheduler policy and RT priority
|
| 3607 | * @pid: the pid in question.
|
| 3608 | * @policy: new policy.
|
| 3609 | * @param: structure containing the new RT priority.
|
| 3610 |
|
| 3611 | asmlinkage long sys_sched_setscheduler(pid_t pid, int policy,
|
| 3612 | struct sched_param __user *param)
|
| 3613 | {
|
| 3614 | return do_sched_setscheduler(pid, policy, param);
|
| 3615 | }
|
| 3616 |
|
| 3617 | |
| 3618 | * sys_sched_setparam - set/change the RT priority of a thread
|
| 3619 | * @pid: the pid in question.
|
| 3620 | * @param: structure containing the new RT priority.
|
| 3621 |
|
| 3622 | asmlinkage long sys_sched_setparam(pid_t pid, struct sched_param __user *param)
|
| 3623 | {
|
| 3624 | return do_sched_setscheduler(pid, -1, param);
|
| 3625 | }
|
| 3626 |
|
| 3627 | |
| 3628 | * sys_sched_getscheduler - get the policy (scheduling class) of a thread
|
| 3629 | * @pid: the pid in question.
|
| 3630 |
|
| 3631 | asmlinkage long sys_sched_getscheduler(pid_t pid)
|
| 3632 | {
|
| 3633 | int retval = -EINVAL;
|
| 3634 | task_t *p;
|
| 3635 |
|
| 3636 | if (pid < 0)
|
| 3637 | goto out_nounlock;
|
| 3638 |
|
| 3639 | retval = -ESRCH;
|
| 3640 | read_lock(&tasklist_lock);
|
| 3641 | p = find_process_by_pid(pid);
|
| 3642 | if (p) {
|
| 3643 | retval = security_task_getscheduler(p);
|
| 3644 | if (!retval)
|
| 3645 | retval = p->policy;
|
| 3646 | }
|
| 3647 | read_unlock(&tasklist_lock);
|
| 3648 |
|
| 3649 | out_nounlock:
|
| 3650 | return retval;
|
| 3651 | }
|
| 3652 |
|
| 3653 | |
| 3654 | * sys_sched_getscheduler - get the RT priority of a thread
|
| 3655 | * @pid: the pid in question.
|
| 3656 | * @param: structure containing the RT priority.
|
| 3657 |
|
| 3658 | asmlinkage long sys_sched_getparam(pid_t pid, struct sched_param __user *param)
|
| 3659 | {
|
| 3660 | struct sched_param lp;
|
| 3661 | int retval = -EINVAL;
|
| 3662 | task_t *p;
|
| 3663 |
|
| 3664 | if (!param || pid < 0)
|
| 3665 | goto out_nounlock;
|
| 3666 |
|
| 3667 | read_lock(&tasklist_lock);
|
| 3668 | p = find_process_by_pid(pid);
|
| 3669 | retval = -ESRCH;
|
| 3670 | if (!p)
|
| 3671 | goto out_unlock;
|
| 3672 |
|
| 3673 | retval = security_task_getscheduler(p);
|
| 3674 | if (retval)
|
| 3675 | goto out_unlock;
|
| 3676 |
|
| 3677 | lp.sched_priority = p->rt_priority;
|
| 3678 | read_unlock(&tasklist_lock);
|
| 3679 |
|
| 3680 | |
| 3681 | * This one might sleep, we cannot do it with a spinlock held ...
|
| 3682 |
|
| 3683 | retval = copy_to_user(param, &lp, sizeof(*param)) ? -EFAULT : 0;
|
| 3684 |
|
| 3685 | out_nounlock:
|
| 3686 | return retval;
|
| 3687 |
|
| 3688 | out_unlock:
|
| 3689 | read_unlock(&tasklist_lock);
|
| 3690 | return retval;
|
| 3691 | }
|
| 3692 |
|
| 3693 | long sched_setaffinity(pid_t pid, cpumask_t new_mask)
|
| 3694 | {
|
| 3695 | task_t *p;
|
| 3696 | int retval;
|
| 3697 | cpumask_t cpus_allowed;
|
| 3698 |
|
| 3699 | lock_cpu_hotplug();
|
| 3700 | read_lock(&tasklist_lock);
|
| 3701 |
|
| 3702 | p = find_process_by_pid(pid);
|
| 3703 | if (!p) {
|
| 3704 | read_unlock(&tasklist_lock);
|
| 3705 | unlock_cpu_hotplug();
|
| 3706 | return -ESRCH;
|
| 3707 | }
|
| 3708 |
|
| 3709 | |
| 3710 | * It is not safe to call set_cpus_allowed with the
|
| 3711 | * tasklist_lock held. We will bump the task_struct's
|
| 3712 | * usage count and then drop tasklist_lock.
|
| 3713 |
|
| 3714 | get_task_struct(p);
|
| 3715 | read_unlock(&tasklist_lock);
|
| 3716 |
|
| 3717 | retval = -EPERM;
|
| 3718 | if ((current->euid != p->euid) && (current->euid != p->uid) &&
|
| 3719 | !capable(CAP_SYS_NICE))
|
| 3720 | goto out_unlock;
|
| 3721 |
|
| 3722 | cpus_allowed = cpuset_cpus_allowed(p);
|
| 3723 | cpus_and(new_mask, new_mask, cpus_allowed);
|
| 3724 | retval = set_cpus_allowed(p, new_mask);
|
| 3725 |
|
| 3726 | out_unlock:
|
| 3727 | put_task_struct(p);
|
| 3728 | unlock_cpu_hotplug();
|
| 3729 | return retval;
|
| 3730 | }
|
| 3731 |
|
| 3732 | static int get_user_cpu_mask(unsigned long __user *user_mask_ptr, unsigned len,
|
| 3733 | cpumask_t *new_mask)
|
| 3734 | {
|
| 3735 | if (len < sizeof(cpumask_t)) {
|
| 3736 | memset(new_mask, 0, sizeof(cpumask_t));
|
| 3737 | } else if (len > sizeof(cpumask_t)) {
|
| 3738 | len = sizeof(cpumask_t);
|
| 3739 | }
|
| 3740 | return copy_from_user(new_mask, user_mask_ptr, len) ? -EFAULT : 0;
|
| 3741 | }
|
| 3742 |
|
| 3743 | |
| 3744 | * sys_sched_setaffinity - set the cpu affinity of a process
|
| 3745 | * @pid: pid of the process
|
| 3746 | * @len: length in bytes of the bitmask pointed to by user_mask_ptr
|
| 3747 | * @user_mask_ptr: user-space pointer to the new cpu mask
|
| 3748 |
|
| 3749 | asmlinkage long sys_sched_setaffinity(pid_t pid, unsigned int len,
|
| 3750 | unsigned long __user *user_mask_ptr)
|
| 3751 | {
|
| 3752 | cpumask_t new_mask;
|
| 3753 | int retval;
|
| 3754 |
|
| 3755 | retval = get_user_cpu_mask(user_mask_ptr, len, &new_mask);
|
| 3756 | if (retval)
|
| 3757 | return retval;
|
| 3758 |
|
| 3759 | return sched_setaffinity(pid, new_mask);
|
| 3760 | }
|
| 3761 |
|
| 3762 | |
| 3763 | * Represents all cpu's present in the system
|
| 3764 | * In systems capable of hotplug, this map could dynamically grow
|
| 3765 | * as new cpu's are detected in the system via any platform specific
|
| 3766 | * method, such as ACPI for e.g.
|
| 3767 |
|
| 3768 |
|
| 3769 | cpumask_t cpu_present_map;
|
| 3770 | EXPORT_SYMBOL(cpu_present_map);
|
| 3771 |
|
| 3772 | #ifndef CONFIG_SMP
|
| 3773 | cpumask_t cpu_online_map = CPU_MASK_ALL;
|
| 3774 | cpumask_t cpu_possible_map = CPU_MASK_ALL;
|
| 3775 | #endif
|
| 3776 |
|
| 3777 | long sched_getaffinity(pid_t pid, cpumask_t *mask)
|
| 3778 | {
|
| 3779 | int retval;
|
| 3780 | task_t *p;
|
| 3781 |
|
| 3782 | lock_cpu_hotplug();
|
| 3783 | read_lock(&tasklist_lock);
|
| 3784 |
|
| 3785 | retval = -ESRCH;
|
| 3786 | p = find_process_by_pid(pid);
|
| 3787 | if (!p)
|
| 3788 | goto out_unlock;
|
| 3789 |
|
| 3790 | retval = 0;
|
| 3791 | cpus_and(*mask, p->cpus_allowed, cpu_possible_map);
|
| 3792 |
|
| 3793 | out_unlock:
|
| 3794 | read_unlock(&tasklist_lock);
|
| 3795 | unlock_cpu_hotplug();
|
| 3796 | if (retval)
|
| 3797 | return retval;
|
| 3798 |
|
| 3799 | return 0;
|
| 3800 | }
|
| 3801 |
|
| 3802 | |
| 3803 | * sys_sched_getaffinity - get the cpu affinity of a process
|
| 3804 | * @pid: pid of the process
|
| 3805 | * @len: length in bytes of the bitmask pointed to by user_mask_ptr
|
| 3806 | * @user_mask_ptr: user-space pointer to hold the current cpu mask
|
| 3807 |
|
| 3808 | asmlinkage long sys_sched_getaffinity(pid_t pid, unsigned int len,
|
| 3809 | unsigned long __user *user_mask_ptr)
|
| 3810 | {
|
| 3811 | int ret;
|
| 3812 | cpumask_t mask;
|
| 3813 |
|
| 3814 | if (len < sizeof(cpumask_t))
|
| 3815 | return -EINVAL;
|
| 3816 |
|
| 3817 | ret = sched_getaffinity(pid, &mask);
|
| 3818 | if (ret < 0)
|
| 3819 | return ret;
|
| 3820 |
|
| 3821 | if (copy_to_user(user_mask_ptr, &mask, sizeof(cpumask_t)))
|
| 3822 | return -EFAULT;
|
| 3823 |
|
| 3824 | return sizeof(cpumask_t);
|
| 3825 | }
|
| 3826 |
|
| 3827 | |
| 3828 | * sys_sched_yield - yield the current processor to other threads.
|
| 3829 | *
|
| 3830 | * this function yields the current CPU by moving the calling thread
|
| 3831 | * to the expired array. If there are no other threads running on this
|
| 3832 | * CPU then this function will return.
|
| 3833 |
|
| 3834 | asmlinkage long sys_sched_yield(void)
|
| 3835 | {
|
| 3836 | runqueue_t *rq = this_rq_lock();
|
| 3837 | prio_array_t *array = current->array;
|
| 3838 | prio_array_t *target = rq->expired;
|
| 3839 |
|
| 3840 | schedstat_inc(rq, yld_cnt);
|
| 3841 | |
| 3842 | * We implement yielding by moving the task into the expired
|
| 3843 | * queue.
|
| 3844 | *
|
| 3845 | * (special rule: RT tasks will just roundrobin in the active
|
| 3846 | * array.)
|
| 3847 |
|
| 3848 | if (rt_task(current))
|
| 3849 | target = rq->active;
|
| 3850 |
|
| 3851 | if (current->array->nr_active == 1) {
|
| 3852 | schedstat_inc(rq, yld_act_empty);
|
| 3853 | if (!rq->expired->nr_active)
|
| 3854 | schedstat_inc(rq, yld_both_empty);
|
| 3855 | } else if (!rq->expired->nr_active)
|
| 3856 | schedstat_inc(rq, yld_exp_empty);
|
| 3857 |
|
| 3858 | if (array != target) {
|
| 3859 | dequeue_task(current, array);
|
| 3860 | enqueue_task(current, target);
|
| 3861 | } else
|
| 3862 | |
| 3863 | * requeue_task is cheaper so perform that if possible.
|
| 3864 |
|
| 3865 | requeue_task(current, array);
|
| 3866 |
|
| 3867 | |
| 3868 | * Since we are going to call schedule() anyway, there's
|
| 3869 | * no need to preempt or enable interrupts:
|
| 3870 |
|
| 3871 | __release(rq->lock);
|
| 3872 | _raw_spin_unlock(&rq->lock);
|
| 3873 | preempt_enable_no_resched();
|
| 3874 |
|
| 3875 | schedule();
|
| 3876 |
|
| 3877 | return 0;
|
| 3878 | }
|
| 3879 |
|
| 3880 | static inline void __cond_resched(void)
|
| 3881 | {
|
| 3882 | |
| 3883 | * The BKS might be reacquired before we have dropped
|
| 3884 | * PREEMPT_ACTIVE, which could trigger a second
|
| 3885 | * cond_resched() call.
|
| 3886 |
|
| 3887 | if (unlikely(preempt_count()))
|
| 3888 | return;
|
| 3889 | do {
|
| 3890 | add_preempt_count(PREEMPT_ACTIVE);
|
| 3891 | schedule();
|
| 3892 | sub_preempt_count(PREEMPT_ACTIVE);
|
| 3893 | } while (need_resched());
|
| 3894 | }
|
| 3895 |
|
| 3896 | int __sched cond_resched(void)
|
| 3897 | {
|
| 3898 | if (need_resched()) {
|
| 3899 | __cond_resched();
|
| 3900 | return 1;
|
| 3901 | }
|
| 3902 | return 0;
|
| 3903 | }
|
| 3904 |
|
| 3905 | EXPORT_SYMBOL(cond_resched);
|
| 3906 |
|
| 3907 | |
| 3908 | * cond_resched_lock() - if a reschedule is pending, drop the given lock,
|
| 3909 | * call schedule, and on return reacquire the lock.
|
| 3910 | *
|
| 3911 | * This works OK both with and without CONFIG_PREEMPT. We do strange low-level
|
| 3912 | * operations here to prevent schedule() from being called twice (once via
|
| 3913 | * spin_unlock(), once by hand).
|
| 3914 |
|
| 3915 | int cond_resched_lock(spinlock_t * lock)
|
| 3916 | {
|
| 3917 | int ret = 0;
|
| 3918 |
|
| 3919 | if (need_lockbreak(lock)) {
|
| 3920 | spin_unlock(lock);
|
| 3921 | cpu_relax();
|
| 3922 | ret = 1;
|
| 3923 | spin_lock(lock);
|
| 3924 | }
|
| 3925 | if (need_resched()) {
|
| 3926 | _raw_spin_unlock(lock);
|
| 3927 | preempt_enable_no_resched();
|
| 3928 | __cond_resched();
|
| 3929 | ret = 1;
|
| 3930 | spin_lock(lock);
|
| 3931 | }
|
| 3932 | return ret;
|
| 3933 | }
|
| 3934 |
|
| 3935 | EXPORT_SYMBOL(cond_resched_lock);
|
| 3936 |
|
| 3937 | int __sched cond_resched_softirq(void)
|
| 3938 | {
|
| 3939 | BUG_ON(!in_softirq());
|
| 3940 |
|
| 3941 | if (need_resched()) {
|
| 3942 | __local_bh_enable();
|
| 3943 | __cond_resched();
|
| 3944 | local_bh_disable();
|
| 3945 | return 1;
|
| 3946 | }
|
| 3947 | return 0;
|
| 3948 | }
|
| 3949 |
|
| 3950 | EXPORT_SYMBOL(cond_resched_softirq);
|
| 3951 |
|
| 3952 |
|
| 3953 | |
| 3954 | * yield - yield the current processor to other threads.
|
| 3955 | *
|
| 3956 | * this is a shortcut for kernel-space yielding - it marks the
|
| 3957 | * thread runnable and calls sys_sched_yield().
|
| 3958 |
|
| 3959 | void __sched yield(void)
|
| 3960 | {
|
| 3961 | set_current_state(TASK_RUNNING);
|
| 3962 | sys_sched_yield();
|
| 3963 | }
|
| 3964 |
|
| 3965 | EXPORT_SYMBOL(yield);
|
| 3966 |
|
| 3967 | |
| 3968 | * This task is about to go to sleep on IO. Increment rq->nr_iowait so
|
| 3969 | * that process accounting knows that this is a task in IO wait state.
|
| 3970 | *
|
| 3971 | * But don't do that if it is a deliberate, throttling IO wait (this task
|
| 3972 | * has set its backing_dev_info: the queue against which it should throttle)
|
| 3973 |
|
| 3974 | void __sched io_schedule(void)
|
| 3975 | {
|
| 3976 | struct runqueue *rq = &per_cpu(runqueues, raw_smp_processor_id());
|
| 3977 |
|
| 3978 | atomic_inc(&rq->nr_iowait);
|
| 3979 | schedule();
|
| 3980 | atomic_dec(&rq->nr_iowait);
|
| 3981 | }
|
| 3982 |
|
| 3983 | EXPORT_SYMBOL(io_schedule);
|
| 3984 |
|
| 3985 | long __sched io_schedule_timeout(long timeout)
|
| 3986 | {
|
| 3987 | struct runqueue *rq = &per_cpu(runqueues, raw_smp_processor_id());
|
| 3988 | long ret;
|
| 3989 |
|
| 3990 | atomic_inc(&rq->nr_iowait);
|
| 3991 | ret = schedule_timeout(timeout);
|
| 3992 | atomic_dec(&rq->nr_iowait);
|
| 3993 | return ret;
|
| 3994 | }
|
| 3995 |
|
| 3996 | |
| 3997 | * sys_sched_get_priority_max - return maximum RT priority.
|
| 3998 | * @policy: scheduling class.
|
| 3999 | *
|
| 4000 | * this syscall returns the maximum rt_priority that can be used
|
| 4001 | * by a given scheduling class.
|
| 4002 |
|
| 4003 | asmlinkage long sys_sched_get_priority_max(int policy)
|
| 4004 | {
|
| 4005 | int ret = -EINVAL;
|
| 4006 |
|
| 4007 | switch (policy) {
|
| 4008 | case SCHED_FIFO:
|
| 4009 | case SCHED_RR:
|
| 4010 | ret = MAX_USER_RT_PRIO-1;
|
| 4011 | break;
|
| 4012 | case SCHED_NORMAL:
|
| 4013 | ret = 0;
|
| 4014 | break;
|
| 4015 | }
|
| 4016 | return ret;
|
| 4017 | }
|
| 4018 |
|
| 4019 | |
| 4020 | * sys_sched_get_priority_min - return minimum RT priority.
|
| 4021 | * @policy: scheduling class.
|
| 4022 | *
|
| 4023 | * this syscall returns the minimum rt_priority that can be used
|
| 4024 | * by a given scheduling class.
|
| 4025 |
|
| 4026 | asmlinkage long sys_sched_get_priority_min(int policy)
|
| 4027 | {
|
| 4028 | int ret = -EINVAL;
|
| 4029 |
|
| 4030 | switch (policy) {
|
| 4031 | case SCHED_FIFO:
|
| 4032 | case SCHED_RR:
|
| 4033 | ret = 1;
|
| 4034 | break;
|
| 4035 | case SCHED_NORMAL:
|
| 4036 | ret = 0;
|
| 4037 | }
|
| 4038 | return ret;
|
| 4039 | }
|
| 4040 |
|
| 4041 | |
| 4042 | * sys_sched_rr_get_interval - return the default timeslice of a process.
|
| 4043 | * @pid: pid of the process.
|
| 4044 | * @interval: userspace pointer to the timeslice value.
|
| 4045 | *
|
| 4046 | * this syscall writes the default timeslice value of a given process
|
| 4047 | * into the user-space timespec buffer. A value of '0' means infinity.
|
| 4048 |
|
| 4049 | asmlinkage
|
| 4050 | long sys_sched_rr_get_interval(pid_t pid, struct timespec __user *interval)
|
| 4051 | {
|
| 4052 | int retval = -EINVAL;
|
| 4053 | struct timespec t;
|
| 4054 | task_t *p;
|
| 4055 |
|
| 4056 | if (pid < 0)
|
| 4057 | goto out_nounlock;
|
| 4058 |
|
| 4059 | retval = -ESRCH;
|
| 4060 | read_lock(&tasklist_lock);
|
| 4061 | p = find_process_by_pid(pid);
|
| 4062 | if (!p)
|
| 4063 | goto out_unlock;
|
| 4064 |
|
| 4065 | retval = security_task_getscheduler(p);
|
| 4066 | if (retval)
|
| 4067 | goto out_unlock;
|
| 4068 |
|
| 4069 | jiffies_to_timespec(p->policy & SCHED_FIFO ?
|
| 4070 | 0 : task_timeslice(p), &t);
|
| 4071 | read_unlock(&tasklist_lock);
|
| 4072 | retval = copy_to_user(interval, &t, sizeof(t)) ? -EFAULT : 0;
|
| 4073 | out_nounlock:
|
| 4074 | return retval;
|
| 4075 | out_unlock:
|
| 4076 | read_unlock(&tasklist_lock);
|
| 4077 | return retval;
|
| 4078 | }
|
| 4079 |
|
| 4080 | static inline struct task_struct *eldest_child(struct task_struct *p)
|
| 4081 | {
|
| 4082 | if (list_empty(&p->children)) return NULL;
|
| 4083 | return list_entry(p->children.next,struct task_struct,sibling);
|
| 4084 | }
|
| 4085 |
|
| 4086 | static inline struct task_struct *older_sibling(struct task_struct *p)
|
| 4087 | {
|
| 4088 | if (p->sibling.prev==&p->parent->children) return NULL;
|
| 4089 | return list_entry(p->sibling.prev,struct task_struct,sibling);
|
| 4090 | }
|
| 4091 |
|
| 4092 | static inline struct task_struct *younger_sibling(struct task_struct *p)
|
| 4093 | {
|
| 4094 | if (p->sibling.next==&p->parent->children) return NULL;
|
| 4095 | return list_entry(p->sibling.next,struct task_struct,sibling);
|
| 4096 | }
|
| 4097 |
|
| 4098 | static void show_task(task_t * p)
|
| 4099 | {
|
| 4100 | task_t *relative;
|
| 4101 | unsigned state;
|
| 4102 | unsigned long free = 0;
|
| 4103 | static const char *stat_nam[] = { "R", "S", "D", "T", "t", "Z", "X" };
|
| 4104 |
|
| 4105 | printk("%-13.13s ", p->comm);
|
| 4106 | state = p->state ? __ffs(p->state) + 1 : 0;
|
| 4107 | if (state < ARRAY_SIZE(stat_nam))
|
| 4108 | printk(stat_nam[state]);
|
| 4109 | else
|
| 4110 | printk("?");
|
| 4111 | #if (BITS_PER_LONG == 32)
|
| 4112 | if (state == TASK_RUNNING)
|
| 4113 | printk(" running ");
|
| 4114 | else
|
| 4115 | printk(" %08lX ", thread_saved_pc(p));
|
| 4116 | #else
|
| 4117 | if (state == TASK_RUNNING)
|
| 4118 | printk(" running task ");
|
| 4119 | else
|
| 4120 | printk(" %016lx ", thread_saved_pc(p));
|
| 4121 | #endif
|
| 4122 | #ifdef CONFIG_DEBUG_STACK_USAGE
|
| 4123 | {
|
| 4124 | unsigned long * n = (unsigned long *) (p->thread_info+1);
|
| 4125 | while (!*n)
|
| 4126 | n++;
|
| 4127 | free = (unsigned long) n - (unsigned long)(p->thread_info+1);
|
| 4128 | }
|
| 4129 | #endif
|
| 4130 | printk("%5lu %5d %6d ", free, p->pid, p->parent->pid);
|
| 4131 | if ((relative = eldest_child(p)))
|
| 4132 | printk("%5d ", relative->pid);
|
| 4133 | else
|
| 4134 | printk(" ");
|
| 4135 | if ((relative = younger_sibling(p)))
|
| 4136 | printk("%7d", relative->pid);
|
| 4137 | else
|
| 4138 | printk(" ");
|
| 4139 | if ((relative = older_sibling(p)))
|
| 4140 | printk(" %5d", relative->pid);
|
| 4141 | else
|
| 4142 | printk(" ");
|
| 4143 | if (!p->mm)
|
| 4144 | printk(" (L-TLB)\n");
|
| 4145 | else
|
| 4146 | printk(" (NOTLB)\n");
|
| 4147 |
|
| 4148 | if (state != TASK_RUNNING)
|
| 4149 | show_stack(p, NULL);
|
| 4150 | }
|
| 4151 |
|
| 4152 | void show_state(void)
|
| 4153 | {
|
| 4154 | task_t *g, *p;
|
| 4155 |
|
| 4156 | #if (BITS_PER_LONG == 32)
|
| 4157 | printk("\n"
|
| 4158 | " sibling\n");
|
| 4159 | printk(" task PC pid father child younger older\n");
|
| 4160 | #else
|
| 4161 | printk("\n"
|
| 4162 | " sibling\n");
|
| 4163 | printk(" task PC pid father child younger older\n");
|
| 4164 | #endif
|
| 4165 | read_lock(&tasklist_lock);
|
| 4166 | do_each_thread(g, p) {
|
| 4167 | |
| 4168 | * reset the NMI-timeout, listing all files on a slow
|
| 4169 | * console might take alot of time:
|
| 4170 |
|
| 4171 | touch_nmi_watchdog();
|
| 4172 | show_task(p);
|
| 4173 | } while_each_thread(g, p);
|
| 4174 |
|
| 4175 | read_unlock(&tasklist_lock);
|
| 4176 | }
|
| 4177 |
|
| 4178 | |
| 4179 | * init_idle - set up an idle thread for a given CPU
|
| 4180 | * @idle: task in question
|
| 4181 | * @cpu: cpu the idle task belongs to
|
| 4182 | *
|
| 4183 | * NOTE: this function does not set the idle thread's NEED_RESCHED
|
| 4184 | * flag, to make booting more robust.
|
| 4185 |
|
| 4186 | void __devinit init_idle(task_t *idle, int cpu)
|
| 4187 | {
|
| 4188 | runqueue_t *rq = cpu_rq(cpu);
|
| 4189 | unsigned long flags;
|
| 4190 |
|
| 4191 | idle->sleep_avg = 0;
|
| 4192 | idle->array = NULL;
|
| 4193 | idle->prio = MAX_PRIO;
|
| 4194 | idle->state = TASK_RUNNING;
|
| 4195 | idle->cpus_allowed = cpumask_of_cpu(cpu);
|
| 4196 | set_task_cpu(idle, cpu);
|
| 4197 |
|
| 4198 | spin_lock_irqsave(&rq->lock, flags);
|
| 4199 | rq->curr = rq->idle = idle;
|
| 4200 | #if defined(CONFIG_SMP) && defined(__ARCH_WANT_UNLOCKED_CTXSW)
|
| 4201 | idle->oncpu = 1;
|
| 4202 | #endif
|
| 4203 | spin_unlock_irqrestore(&rq->lock, flags);
|
| 4204 |
|
| 4205 |
|
| 4206 | #if defined(CONFIG_PREEMPT) && !defined(CONFIG_PREEMPT_BKL)
|
| 4207 | idle->thread_info->preempt_count = (idle->lock_depth >= 0);
|
| 4208 | #else
|
| 4209 | idle->thread_info->preempt_count = 0;
|
| 4210 | #endif
|
| 4211 | }
|
| 4212 |
|
| 4213 | |
| 4214 | * In a system that switches off the HZ timer nohz_cpu_mask
|
| 4215 | * indicates which cpus entered this state. This is used
|
| 4216 | * in the rcu update to wait only for active cpus. For system
|
| 4217 | * which do not switch off the HZ timer nohz_cpu_mask should
|
| 4218 | * always be CPU_MASK_NONE.
|
| 4219 |
|
| 4220 | cpumask_t nohz_cpu_mask = CPU_MASK_NONE;
|
| 4221 |
|
| 4222 | #ifdef CONFIG_SMP
|
| 4223 | |
| 4224 | * This is how migration works:
|
| 4225 | *
|
| 4226 | * 1) we queue a migration_req_t structure in the source CPU's
|
| 4227 | * runqueue and wake up that CPU's migration thread.
|
| 4228 | * 2) we down() the locked semaphore => thread blocks.
|
| 4229 | * 3) migration thread wakes up (implicitly it forces the migrated
|
| 4230 | * thread off the CPU)
|
| 4231 | * 4) it gets the migration request and checks whether the migrated
|
| 4232 | * task is still in the wrong runqueue.
|
| 4233 | * 5) if it's in the wrong runqueue then the migration thread removes
|
| 4234 | * it and puts it into the right queue.
|
| 4235 | * 6) migration thread up()s the semaphore.
|
| 4236 | * 7) we wake up and the migration is done.
|
| 4237 |
|
| 4238 |
|
| 4239 | |
| 4240 | * Change a given task's CPU affinity. Migrate the thread to a
|
| 4241 | * proper CPU and schedule it away if the CPU it's executing on
|
| 4242 | * is removed from the allowed bitmask.
|
| 4243 | *
|
| 4244 | * NOTE: the caller must have a valid reference to the task, the
|
| 4245 | * task must not exit() & deallocate itself prematurely. The
|
| 4246 | * call is not atomic; no spinlocks may be held.
|
| 4247 |
|
| 4248 | int set_cpus_allowed(task_t *p, cpumask_t new_mask)
|
| 4249 | {
|
| 4250 | unsigned long flags;
|
| 4251 | int ret = 0;
|
| 4252 | migration_req_t req;
|
| 4253 | runqueue_t *rq;
|
| 4254 |
|
| 4255 | rq = task_rq_lock(p, &flags);
|
| 4256 | if (!cpus_intersects(new_mask, cpu_online_map)) {
|
| 4257 | ret = -EINVAL;
|
| 4258 | goto out;
|
| 4259 | }
|
| 4260 |
|
| 4261 | p->cpus_allowed = new_mask;
|
| 4262 |
|
| 4263 | if (cpu_isset(task_cpu(p), new_mask))
|
| 4264 | goto out;
|
| 4265 |
|
| 4266 | if (migrate_task(p, any_online_cpu(new_mask), &req)) {
|
| 4267 |
|
| 4268 | task_rq_unlock(rq, &flags);
|
| 4269 | wake_up_process(rq->migration_thread);
|
| 4270 | wait_for_completion(&req.done);
|
| 4271 | tlb_migrate_finish(p->mm);
|
| 4272 | return 0;
|
| 4273 | }
|
| 4274 | out:
|
| 4275 | task_rq_unlock(rq, &flags);
|
| 4276 | return ret;
|
| 4277 | }
|
| 4278 |
|
| 4279 | EXPORT_SYMBOL_GPL(set_cpus_allowed);
|
| 4280 |
|
| 4281 | |
| 4282 | * Move (not current) task off this cpu, onto dest cpu. We're doing
|
| 4283 | * this because either it can't run here any more (set_cpus_allowed()
|
| 4284 | * away from this CPU, or CPU going down), or because we're
|
| 4285 | * attempting to rebalance this task on exec (sched_exec).
|
| 4286 | *
|
| 4287 | * So we race with normal scheduler movements, but that's OK, as long
|
| 4288 | * as the task is no longer on this CPU.
|
| 4289 |
|
| 4290 | static void __migrate_task(struct task_struct *p, int src_cpu, int dest_cpu)
|
| 4291 | {
|
| 4292 | runqueue_t *rq_dest, *rq_src;
|
| 4293 |
|
| 4294 | if (unlikely(cpu_is_offline(dest_cpu)))
|
| 4295 | return;
|
| 4296 |
|
| 4297 | rq_src = cpu_rq(src_cpu);
|
| 4298 | rq_dest = cpu_rq(dest_cpu);
|
| 4299 |
|
| 4300 | double_rq_lock(rq_src, rq_dest);
|
| 4301 |
|
| 4302 | if (task_cpu(p) != src_cpu)
|
| 4303 | goto out;
|
| 4304 |
|
| 4305 | if (!cpu_isset(dest_cpu, p->cpus_allowed))
|
| 4306 | goto out;
|
| 4307 |
|
| 4308 | set_task_cpu(p, dest_cpu);
|
| 4309 | if (p->array) {
|
| 4310 | |
| 4311 | * Sync timestamp with rq_dest's before activating.
|
| 4312 | * The same thing could be achieved by doing this step
|
| 4313 | * afterwards, and pretending it was a local activate.
|
| 4314 | * This way is cleaner and logically correct.
|
| 4315 |
|
| 4316 | p->timestamp = p->timestamp - rq_src->timestamp_last_tick
|
| 4317 | + rq_dest->timestamp_last_tick;
|
| 4318 | deactivate_task(p, rq_src);
|
| 4319 | activate_task(p, rq_dest, 0);
|
| 4320 | if (TASK_PREEMPTS_CURR(p, rq_dest))
|
| 4321 | resched_task(rq_dest->curr);
|
| 4322 | }
|
| 4323 |
|
| 4324 | out:
|
| 4325 | double_rq_unlock(rq_src, rq_dest);
|
| 4326 | }
|
| 4327 |
|
| 4328 | |
| 4329 | * migration_thread - this is a highprio system thread that performs
|
| 4330 | * thread migration by bumping thread off CPU then 'pushing' onto
|
| 4331 | * another runqueue.
|
| 4332 |
|
| 4333 | static int migration_thread(void * data)
|
| 4334 | {
|
| 4335 | runqueue_t *rq;
|
| 4336 | int cpu = (long)data;
|
| 4337 |
|
| 4338 | rq = cpu_rq(cpu);
|
| 4339 | BUG_ON(rq->migration_thread != current);
|
| 4340 |
|
| 4341 | set_current_state(TASK_INTERRUPTIBLE);
|
| 4342 | while (!kthread_should_stop()) {
|
| 4343 | struct list_head *head;
|
| 4344 | migration_req_t *req;
|
| 4345 |
|
| 4346 | try_to_freeze();
|
| 4347 |
|
| 4348 | spin_lock_irq(&rq->lock);
|
| 4349 |
|
| 4350 | if (cpu_is_offline(cpu)) {
|
| 4351 | spin_unlock_irq(&rq->lock);
|
| 4352 | goto wait_to_die;
|
| 4353 | }
|
| 4354 |
|
| 4355 | if (rq->active_balance) {
|
| 4356 | active_load_balance(rq, cpu);
|
| 4357 | rq->active_balance = 0;
|
| 4358 | }
|
| 4359 |
|
| 4360 | head = &rq->migration_queue;
|
| 4361 |
|
| 4362 | if (list_empty(head)) {
|
| 4363 | spin_unlock_irq(&rq->lock);
|
| 4364 | schedule();
|
| 4365 | set_current_state(TASK_INTERRUPTIBLE);
|
| 4366 | continue;
|
| 4367 | }
|
| 4368 | req = list_entry(head->next, migration_req_t, list);
|
| 4369 | list_del_init(head->next);
|
| 4370 |
|
| 4371 | spin_unlock(&rq->lock);
|
| 4372 | __migrate_task(req->task, cpu, req->dest_cpu);
|
| 4373 | local_irq_enable();
|
| 4374 |
|
| 4375 | complete(&req->done);
|
| 4376 | }
|
| 4377 | __set_current_state(TASK_RUNNING);
|
| 4378 | return 0;
|
| 4379 |
|
| 4380 | wait_to_die:
|
| 4381 |
|
| 4382 | set_current_state(TASK_INTERRUPTIBLE);
|
| 4383 | while (!kthread_should_stop()) {
|
| 4384 | schedule();
|
| 4385 | set_current_state(TASK_INTERRUPTIBLE);
|
| 4386 | }
|
| 4387 | __set_current_state(TASK_RUNNING);
|
| 4388 | return 0;
|
| 4389 | }
|
| 4390 |
|
| 4391 | #ifdef CONFIG_HOTPLUG_CPU
|
| 4392 |
|
| 4393 | static void move_task_off_dead_cpu(int dead_cpu, struct task_struct *tsk)
|
| 4394 | {
|
| 4395 | int dest_cpu;
|
| 4396 | cpumask_t mask;
|
| 4397 |
|
| 4398 |
|
| 4399 | mask = node_to_cpumask(cpu_to_node(dead_cpu));
|
| 4400 | cpus_and(mask, mask, tsk->cpus_allowed);
|
| 4401 | dest_cpu = any_online_cpu(mask);
|
| 4402 |
|
| 4403 |
|
| 4404 | if (dest_cpu == NR_CPUS)
|
| 4405 | dest_cpu = any_online_cpu(tsk->cpus_allowed);
|
| 4406 |
|
| 4407 |
|
| 4408 | if (dest_cpu == NR_CPUS) {
|
| 4409 | cpus_setall(tsk->cpus_allowed);
|
| 4410 | dest_cpu = any_online_cpu(tsk->cpus_allowed);
|
| 4411 |
|
| 4412 | |
| 4413 | * Don't tell them about moving exiting tasks or
|
| 4414 | * kernel threads (both mm NULL), since they never
|
| 4415 | * leave kernel.
|
| 4416 |
|
| 4417 | if (tsk->mm && printk_ratelimit())
|
| 4418 | printk(KERN_INFO "process %d (%s) no "
|
| 4419 | "longer affine to cpu%d\n",
|
| 4420 | tsk->pid, tsk->comm, dead_cpu);
|
| 4421 | }
|
| 4422 | __migrate_task(tsk, dead_cpu, dest_cpu);
|
| 4423 | }
|
| 4424 |
|
| 4425 | |
| 4426 | * While a dead CPU has no uninterruptible tasks queued at this point,
|
| 4427 | * it might still have a nonzero ->nr_uninterruptible counter, because
|
| 4428 | * for performance reasons the counter is not stricly tracking tasks to
|
| 4429 | * their home CPUs. So we just add the counter to another CPU's counter,
|
| 4430 | * to keep the global sum constant after CPU-down:
|
| 4431 |
|
| 4432 | static void migrate_nr_uninterruptible(runqueue_t *rq_src)
|
| 4433 | {
|
| 4434 | runqueue_t *rq_dest = cpu_rq(any_online_cpu(CPU_MASK_ALL));
|
| 4435 | unsigned long flags;
|
| 4436 |
|
| 4437 | local_irq_save(flags);
|
| 4438 | double_rq_lock(rq_src, rq_dest);
|
| 4439 | rq_dest->nr_uninterruptible += rq_src->nr_uninterruptible;
|
| 4440 | rq_src->nr_uninterruptible = 0;
|
| 4441 | double_rq_unlock(rq_src, rq_dest);
|
| 4442 | local_irq_restore(flags);
|
| 4443 | }
|
| 4444 |
|
| 4445 |
|
| 4446 | static void migrate_live_tasks(int src_cpu)
|
| 4447 | {
|
| 4448 | struct task_struct *tsk, *t;
|
| 4449 |
|
| 4450 | write_lock_irq(&tasklist_lock);
|
| 4451 |
|
| 4452 | do_each_thread(t, tsk) {
|
| 4453 | if (tsk == current)
|
| 4454 | continue;
|
| 4455 |
|
| 4456 | if (task_cpu(tsk) == src_cpu)
|
| 4457 | move_task_off_dead_cpu(src_cpu, tsk);
|
| 4458 | } while_each_thread(t, tsk);
|
| 4459 |
|
| 4460 | write_unlock_irq(&tasklist_lock);
|
| 4461 | }
|
| 4462 |
|
| 4463 | |
| 4464 | * It does so by boosting its priority to highest possible and adding it to
|
| 4465 | * the _front_ of runqueue. Used by CPU offline code.
|
| 4466 |
|
| 4467 | void sched_idle_next(void)
|
| 4468 | {
|
| 4469 | int cpu = smp_processor_id();
|
| 4470 | runqueue_t *rq = this_rq();
|
| 4471 | struct task_struct *p = rq->idle;
|
| 4472 | unsigned long flags;
|
| 4473 |
|
| 4474 |
|
| 4475 | BUG_ON(cpu_online(cpu));
|
| 4476 |
|
| 4477 | |
| 4478 | * and interrupts disabled on current cpu.
|
| 4479 |
|
| 4480 | spin_lock_irqsave(&rq->lock, flags);
|
| 4481 |
|
| 4482 | __setscheduler(p, SCHED_FIFO, MAX_RT_PRIO-1);
|
| 4483 |
|
| 4484 | __activate_idle_task(p, rq);
|
| 4485 |
|
| 4486 | spin_unlock_irqrestore(&rq->lock, flags);
|
| 4487 | }
|
| 4488 |
|
| 4489 | |
| 4490 | * offline.
|
| 4491 |
|
| 4492 | void idle_task_exit(void)
|
| 4493 | {
|
| 4494 | struct mm_struct *mm = current->active_mm;
|
| 4495 |
|
| 4496 | BUG_ON(cpu_online(smp_processor_id()));
|
| 4497 |
|
| 4498 | if (mm != &init_mm)
|
| 4499 | switch_mm(mm, &init_mm, current);
|
| 4500 | mmdrop(mm);
|
| 4501 | }
|
| 4502 |
|
| 4503 | static void migrate_dead(unsigned int dead_cpu, task_t *tsk)
|
| 4504 | {
|
| 4505 | struct runqueue *rq = cpu_rq(dead_cpu);
|
| 4506 |
|
| 4507 |
|
| 4508 | BUG_ON(tsk->exit_state != EXIT_ZOMBIE && tsk->exit_state != EXIT_DEAD);
|
| 4509 |
|
| 4510 |
|
| 4511 | BUG_ON(tsk->flags & PF_DEAD);
|
| 4512 |
|
| 4513 | get_task_struct(tsk);
|
| 4514 |
|
| 4515 | |
| 4516 | * Drop lock around migration; if someone else moves it,
|
| 4517 | * that's OK. No task can be added to this CPU, so iteration is
|
| 4518 | * fine.
|
| 4519 |
|
| 4520 | spin_unlock_irq(&rq->lock);
|
| 4521 | move_task_off_dead_cpu(dead_cpu, tsk);
|
| 4522 | spin_lock_irq(&rq->lock);
|
| 4523 |
|
| 4524 | put_task_struct(tsk);
|
| 4525 | }
|
| 4526 |
|
| 4527 |
|
| 4528 | static void migrate_dead_tasks(unsigned int dead_cpu)
|
| 4529 | {
|
| 4530 | unsigned arr, i;
|
| 4531 | struct runqueue *rq = cpu_rq(dead_cpu);
|
| 4532 |
|
| 4533 | for (arr = 0; arr < 2; arr++) {
|
| 4534 | for (i = 0; i < MAX_PRIO; i++) {
|
| 4535 | struct list_head *list = &rq->arrays[arr].queue[i];
|
| 4536 | while (!list_empty(list))
|
| 4537 | migrate_dead(dead_cpu,
|
| 4538 | list_entry(list->next, task_t,
|
| 4539 | run_list));
|
| 4540 | }
|
| 4541 | }
|
| 4542 | }
|
| 4543 | #endif
|
| 4544 |
|
| 4545 | |
| 4546 | * migration_call - callback that gets triggered when a CPU is added.
|
| 4547 | * Here we can start up the necessary migration thread for the new CPU.
|
| 4548 |
|
| 4549 | static int migration_call(struct notifier_block *nfb, unsigned long action,
|
| 4550 | void *hcpu)
|
| 4551 | {
|
| 4552 | int cpu = (long)hcpu;
|
| 4553 | struct task_struct *p;
|
| 4554 | struct runqueue *rq;
|
| 4555 | unsigned long flags;
|
| 4556 |
|
| 4557 | switch (action) {
|
| 4558 | case CPU_UP_PREPARE:
|
| 4559 | p = kthread_create(migration_thread, hcpu, "migration/%d",cpu);
|
| 4560 | if (IS_ERR(p))
|
| 4561 | return NOTIFY_BAD;
|
| 4562 | p->flags |= PF_NOFREEZE;
|
| 4563 | kthread_bind(p, cpu);
|
| 4564 |
|
| 4565 | rq = task_rq_lock(p, &flags);
|
| 4566 | __setscheduler(p, SCHED_FIFO, MAX_RT_PRIO-1);
|
| 4567 | task_rq_unlock(rq, &flags);
|
| 4568 | cpu_rq(cpu)->migration_thread = p;
|
| 4569 | break;
|
| 4570 | case CPU_ONLINE:
|
| 4571 |
|
| 4572 | wake_up_process(cpu_rq(cpu)->migration_thread);
|
| 4573 | break;
|
| 4574 | #ifdef CONFIG_HOTPLUG_CPU
|
| 4575 | case CPU_UP_CANCELED:
|
| 4576 |
|
| 4577 | kthread_bind(cpu_rq(cpu)->migration_thread,smp_processor_id());
|
| 4578 | kthread_stop(cpu_rq(cpu)->migration_thread);
|
| 4579 | cpu_rq(cpu)->migration_thread = NULL;
|
| 4580 | break;
|
| 4581 | case CPU_DEAD:
|
| 4582 | migrate_live_tasks(cpu);
|
| 4583 | rq = cpu_rq(cpu);
|
| 4584 | kthread_stop(rq->migration_thread);
|
| 4585 | rq->migration_thread = NULL;
|
| 4586 |
|
| 4587 | rq = task_rq_lock(rq->idle, &flags);
|
| 4588 | deactivate_task(rq->idle, rq);
|
| 4589 | rq->idle->static_prio = MAX_PRIO;
|
| 4590 | __setscheduler(rq->idle, SCHED_NORMAL, 0);
|
| 4591 | migrate_dead_tasks(cpu);
|
| 4592 | task_rq_unlock(rq, &flags);
|
| 4593 | migrate_nr_uninterruptible(rq);
|
| 4594 | BUG_ON(rq->nr_running != 0);
|
| 4595 |
|
| 4596 | |
| 4597 | * they didn't do lock_cpu_hotplug(). Just wake up
|
| 4598 | * the requestors.
|
| 4599 | spin_lock_irq(&rq->lock);
|
| 4600 | while (!list_empty(&rq->migration_queue)) {
|
| 4601 | migration_req_t *req;
|
| 4602 | req = list_entry(rq->migration_queue.next,
|
| 4603 | migration_req_t, list);
|
| 4604 | list_del_init(&req->list);
|
| 4605 | complete(&req->done);
|
| 4606 | }
|
| 4607 | spin_unlock_irq(&rq->lock);
|
| 4608 | break;
|
| 4609 | #endif
|
| 4610 | }
|
| 4611 | return NOTIFY_OK;
|
| 4612 | }
|
| 4613 |
|
| 4614 | |
| 4615 | * happens before everything else.
|
| 4616 |
|
| 4617 | static struct notifier_block __devinitdata migration_notifier = {
|
| 4618 | .notifier_call = migration_call,
|
| 4619 | .priority = 10
|
| 4620 | };
|
| 4621 |
|
| 4622 | int __init migration_init(void)
|
| 4623 | {
|
| 4624 | void *cpu = (void *)(long)smp_processor_id();
|
| 4625 |
|
| 4626 | migration_call(&migration_notifier, CPU_UP_PREPARE, cpu);
|
| 4627 | migration_call(&migration_notifier, CPU_ONLINE, cpu);
|
| 4628 | register_cpu_notifier(&migration_notifier);
|
| 4629 | return 0;
|
| 4630 | }
|
| 4631 | #endif
|
| 4632 |
|
| 4633 | #ifdef CONFIG_SMP
|
| 4634 | #undef SCHED_DOMAIN_DEBUG
|
| 4635 | #ifdef SCHED_DOMAIN_DEBUG
|
| 4636 | static void sched_domain_debug(struct sched_domain *sd, int cpu)
|
| 4637 | {
|
| 4638 | int level = 0;
|
| 4639 |
|
| 4640 | if (!sd) {
|
| 4641 | printk(KERN_DEBUG "CPU%d attaching NULL sched-domain.\n", cpu);
|
| 4642 | return;
|
| 4643 | }
|
| 4644 |
|
| 4645 | printk(KERN_DEBUG "CPU%d attaching sched-domain:\n", cpu);
|
| 4646 |
|
| 4647 | do {
|
| 4648 | int i;
|
| 4649 | char str[NR_CPUS];
|
| 4650 | struct sched_group *group = sd->groups;
|
| 4651 | cpumask_t groupmask;
|
| 4652 |
|
| 4653 | cpumask_scnprintf(str, NR_CPUS, sd->span);
|
| 4654 | cpus_clear(groupmask);
|
| 4655 |
|
| 4656 | printk(KERN_DEBUG);
|
| 4657 | for (i = 0; i < level + 1; i++)
|
| 4658 | printk(" ");
|
| 4659 | printk("domain %d: ", level);
|
| 4660 |
|
| 4661 | if (!(sd->flags & SD_LOAD_BALANCE)) {
|
| 4662 | printk("does not load-balance\n");
|
| 4663 | if (sd->parent)
|
| 4664 | printk(KERN_ERR "ERROR: !SD_LOAD_BALANCE domain has parent");
|
| 4665 | break;
|
| 4666 | }
|
| 4667 |
|
| 4668 | printk("span %s\n", str);
|
| 4669 |
|
| 4670 | if (!cpu_isset(cpu, sd->span))
|
| 4671 | printk(KERN_ERR "ERROR: domain->span does not contain CPU%d\n", cpu);
|
| 4672 | if (!cpu_isset(cpu, group->cpumask))
|
| 4673 | printk(KERN_ERR "ERROR: domain->groups does not contain CPU%d\n", cpu);
|
| 4674 |
|
| 4675 | printk(KERN_DEBUG);
|
| 4676 | for (i = 0; i < level + 2; i++)
|
| 4677 | printk(" ");
|
| 4678 | printk("groups:");
|
| 4679 | do {
|
| 4680 | if (!group) {
|
| 4681 | printk("\n");
|
| 4682 | printk(KERN_ERR "ERROR: group is NULL\n");
|
| 4683 | break;
|
| 4684 | }
|
| 4685 |
|
| 4686 | if (!group->cpu_power) {
|
| 4687 | printk("\n");
|
| 4688 | printk(KERN_ERR "ERROR: domain->cpu_power not set\n");
|
| 4689 | }
|
| 4690 |
|
| 4691 | if (!cpus_weight(group->cpumask)) {
|
| 4692 | printk("\n");
|
| 4693 | printk(KERN_ERR "ERROR: empty group\n");
|
| 4694 | }
|
| 4695 |
|
| 4696 | if (cpus_intersects(groupmask, group->cpumask)) {
|
| 4697 | printk("\n");
|
| 4698 | printk(KERN_ERR "ERROR: repeated CPUs\n");
|
| 4699 | }
|
| 4700 |
|
| 4701 | cpus_or(groupmask, groupmask, group->cpumask);
|
| 4702 |
|
| 4703 | cpumask_scnprintf(str, NR_CPUS, group->cpumask);
|
| 4704 | printk(" %s", str);
|
| 4705 |
|
| 4706 | group = group->next;
|
| 4707 | } while (group != sd->groups);
|
| 4708 | printk("\n");
|
| 4709 |
|
| 4710 | if (!cpus_equal(sd->span, groupmask))
|
| 4711 | printk(KERN_ERR "ERROR: groups don't span domain->span\n");
|
| 4712 |
|
| 4713 | level++;
|
| 4714 | sd = sd->parent;
|
| 4715 |
|
| 4716 | if (sd) {
|
| 4717 | if (!cpus_subset(groupmask, sd->span))
|
| 4718 | printk(KERN_ERR "ERROR: parent span is not a superset of domain->span\n");
|
| 4719 | }
|
| 4720 |
|
| 4721 | } while (sd);
|
| 4722 | }
|
| 4723 | #else
|
| 4724 | #define sched_domain_debug(sd, cpu) {}
|
| 4725 | #endif
|
| 4726 |
|
| 4727 | static int sd_degenerate(struct sched_domain *sd)
|
| 4728 | {
|
| 4729 | if (cpus_weight(sd->span) == 1)
|
| 4730 | return 1;
|
| 4731 |
|
| 4732 |
|
| 4733 | if (sd->flags & (SD_LOAD_BALANCE |
|
| 4734 | SD_BALANCE_NEWIDLE |
|
| 4735 | SD_BALANCE_FORK |
|
| 4736 | SD_BALANCE_EXEC)) {
|
| 4737 | if (sd->groups != sd->groups->next)
|
| 4738 | return 0;
|
| 4739 | }
|
| 4740 |
|
| 4741 |
|
| 4742 | if (sd->flags & (SD_WAKE_IDLE |
|
| 4743 | SD_WAKE_AFFINE |
|
| 4744 | SD_WAKE_BALANCE))
|
| 4745 | return 0;
|
| 4746 |
|
| 4747 | return 1;
|
| 4748 | }
|
| 4749 |
|
| 4750 | static int sd_parent_degenerate(struct sched_domain *sd,
|
| 4751 | struct sched_domain *parent)
|
| 4752 | {
|
| 4753 | unsigned long cflags = sd->flags, pflags = parent->flags;
|
| 4754 |
|
| 4755 | if (sd_degenerate(parent))
|
| 4756 | return 1;
|
| 4757 |
|
| 4758 | if (!cpus_equal(sd->span, parent->span))
|
| 4759 | return 0;
|
| 4760 |
|
| 4761 |
|
| 4762 |
|
| 4763 | if (cflags & SD_WAKE_AFFINE)
|
| 4764 | pflags &= ~SD_WAKE_BALANCE;
|
| 4765 |
|
| 4766 | if (parent->groups == parent->groups->next) {
|
| 4767 | pflags &= ~(SD_LOAD_BALANCE |
|
| 4768 | SD_BALANCE_NEWIDLE |
|
| 4769 | SD_BALANCE_FORK |
|
| 4770 | SD_BALANCE_EXEC);
|
| 4771 | }
|
| 4772 | if (~cflags & pflags)
|
| 4773 | return 0;
|
| 4774 |
|
| 4775 | return 1;
|
| 4776 | }
|
| 4777 |
|
| 4778 | |
| 4779 | * Attach the domain 'sd' to 'cpu' as its base domain. Callers must
|
| 4780 | * hold the hotplug lock.
|
| 4781 |
|
| 4782 | void cpu_attach_domain(struct sched_domain *sd, int cpu)
|
| 4783 | {
|
| 4784 | runqueue_t *rq = cpu_rq(cpu);
|
| 4785 | struct sched_domain *tmp;
|
| 4786 |
|
| 4787 |
|
| 4788 | for (tmp = sd; tmp; tmp = tmp->parent) {
|
| 4789 | struct sched_domain *parent = tmp->parent;
|
| 4790 | if (!parent)
|
| 4791 | break;
|
| 4792 | if (sd_parent_degenerate(tmp, parent))
|
| 4793 | tmp->parent = parent->parent;
|
| 4794 | }
|
| 4795 |
|
| 4796 | if (sd && sd_degenerate(sd))
|
| 4797 | sd = sd->parent;
|
| 4798 |
|
| 4799 | sched_domain_debug(sd, cpu);
|
| 4800 |
|
| 4801 | rcu_assign_pointer(rq->sd, sd);
|
| 4802 | }
|
| 4803 |
|
| 4804 |
|
| 4805 | cpumask_t __devinitdata cpu_isolated_map = CPU_MASK_NONE;
|
| 4806 |
|
| 4807 |
|
| 4808 | static int __init isolated_cpu_setup(char *str)
|
| 4809 | {
|
| 4810 | int ints[NR_CPUS], i;
|
| 4811 |
|
| 4812 | str = get_options(str, ARRAY_SIZE(ints), ints);
|
| 4813 | cpus_clear(cpu_isolated_map);
|
| 4814 | for (i = 1; i <= ints[0]; i++)
|
| 4815 | if (ints[i] < NR_CPUS)
|
| 4816 | cpu_set(ints[i], cpu_isolated_map);
|
| 4817 | return 1;
|
| 4818 | }
|
| 4819 |
|
| 4820 | __setup ("isolcpus=", isolated_cpu_setup);
|
| 4821 |
|
| 4822 | |
| 4823 | * init_sched_build_groups takes an array of groups, the cpumask we wish
|
| 4824 | * to span, and a pointer to a function which identifies what group a CPU
|
| 4825 | * belongs to. The return value of group_fn must be a valid index into the
|
| 4826 | * groups[] array, and must be >= 0 and < NR_CPUS (due to the fact that we
|
| 4827 | * keep track of groups covered with a cpumask_t).
|
| 4828 | *
|
| 4829 | * init_sched_build_groups will build a circular linked list of the groups
|
| 4830 | * covered by the given span, and will set each group's ->cpumask correctly,
|
| 4831 | * and ->cpu_power to 0.
|
| 4832 |
|
| 4833 | void init_sched_build_groups(struct sched_group groups[],
|
| 4834 | cpumask_t span, int (*group_fn)(int cpu))
|
| 4835 | {
|
| 4836 | struct sched_group *first = NULL, *last = NULL;
|
| 4837 | cpumask_t covered = CPU_MASK_NONE;
|
| 4838 | int i;
|
| 4839 |
|
| 4840 | for_each_cpu_mask(i, span) {
|
| 4841 | int group = group_fn(i);
|
| 4842 | struct sched_group *sg = &groups[group];
|
| 4843 | int j;
|
| 4844 |
|
| 4845 | if (cpu_isset(i, covered))
|
| 4846 | continue;
|
| 4847 |
|
| 4848 | sg->cpumask = CPU_MASK_NONE;
|
| 4849 | sg->cpu_power = 0;
|
| 4850 |
|
| 4851 | for_each_cpu_mask(j, span) {
|
| 4852 | if (group_fn(j) != group)
|
| 4853 | continue;
|
| 4854 |
|
| 4855 | cpu_set(j, covered);
|
| 4856 | cpu_set(j, sg->cpumask);
|
| 4857 | }
|
| 4858 | if (!first)
|
| 4859 | first = sg;
|
| 4860 | if (last)
|
| 4861 | last->next = sg;
|
| 4862 | last = sg;
|
| 4863 | }
|
| 4864 | last->next = first;
|
| 4865 | }
|
| 4866 |
|
| 4867 |
|
| 4868 | #ifdef ARCH_HAS_SCHED_DOMAIN
|
| 4869 | extern void build_sched_domains(const cpumask_t *cpu_map);
|
| 4870 | extern void arch_init_sched_domains(const cpumask_t *cpu_map);
|
| 4871 | extern void arch_destroy_sched_domains(const cpumask_t *cpu_map);
|
| 4872 | #else
|
| 4873 | #ifdef CONFIG_SCHED_SMT
|
| 4874 | static DEFINE_PER_CPU(struct sched_domain, cpu_domains);
|
| 4875 | static struct sched_group sched_group_cpus[NR_CPUS];
|
| 4876 | static int cpu_to_cpu_group(int cpu)
|
| 4877 | {
|
| 4878 | return cpu;
|
| 4879 | }
|
| 4880 | #endif
|
| 4881 |
|
| 4882 | static DEFINE_PER_CPU(struct sched_domain, phys_domains);
|
| 4883 | static struct sched_group sched_group_phys[NR_CPUS];
|
| 4884 | static int cpu_to_phys_group(int cpu)
|
| 4885 | {
|
| 4886 | #ifdef CONFIG_SCHED_SMT
|
| 4887 | return first_cpu(cpu_sibling_map[cpu]);
|
| 4888 | #else
|
| 4889 | return cpu;
|
| 4890 | #endif
|
| 4891 | }
|
| 4892 |
|
| 4893 | #ifdef CONFIG_NUMA
|
| 4894 |
|
| 4895 | static DEFINE_PER_CPU(struct sched_domain, node_domains);
|
| 4896 | static struct sched_group sched_group_nodes[MAX_NUMNODES];
|
| 4897 | static int cpu_to_node_group(int cpu)
|
| 4898 | {
|
| 4899 | return cpu_to_node(cpu);
|
| 4900 | }
|
| 4901 | #endif
|
| 4902 |
|
| 4903 | #if defined(CONFIG_SCHED_SMT) && defined(CONFIG_NUMA)
|
| 4904 | |
| 4905 | * The domains setup code relies on siblings not spanning
|
| 4906 | * multiple nodes. Make sure the architecture has a proper
|
| 4907 | * siblings map:
|
| 4908 |
|
| 4909 | static void check_sibling_maps(void)
|
| 4910 | {
|
| 4911 | int i, j;
|
| 4912 |
|
| 4913 | for_each_online_cpu(i) {
|
| 4914 | for_each_cpu_mask(j, cpu_sibling_map[i]) {
|
| 4915 | if (cpu_to_node(i) != cpu_to_node(j)) {
|
| 4916 | printk(KERN_INFO "warning: CPU %d siblings map "
|
| 4917 | "to different node - isolating "
|
| 4918 | "them.\n", i);
|
| 4919 | cpu_sibling_map[i] = cpumask_of_cpu(i);
|
| 4920 | break;
|
| 4921 | }
|
| 4922 | }
|
| 4923 | }
|
| 4924 | }
|
| 4925 | #endif
|
| 4926 |
|
| 4927 | |
| 4928 | * Build sched domains for a given set of cpus and attach the sched domains
|
| 4929 | * to the individual cpus
|
| 4930 |
|
| 4931 | static void build_sched_domains(const cpumask_t *cpu_map)
|
| 4932 | {
|
| 4933 | int i;
|
| 4934 |
|
| 4935 | |
| 4936 | * Set up domains for cpus specified by the cpu_map.
|
| 4937 |
|
| 4938 | for_each_cpu_mask(i, *cpu_map) {
|
| 4939 | int group;
|
| 4940 | struct sched_domain *sd = NULL, *p;
|
| 4941 | cpumask_t nodemask = node_to_cpumask(cpu_to_node(i));
|
| 4942 |
|
| 4943 | cpus_and(nodemask, nodemask, *cpu_map);
|
| 4944 |
|
| 4945 | #ifdef CONFIG_NUMA
|
| 4946 | sd = &per_cpu(node_domains, i);
|
| 4947 | group = cpu_to_node_group(i);
|
| 4948 | *sd = SD_NODE_INIT;
|
| 4949 | sd->span = *cpu_map;
|
| 4950 | sd->groups = &sched_group_nodes[group];
|
| 4951 | #endif
|
| 4952 |
|
| 4953 | p = sd;
|
| 4954 | sd = &per_cpu(phys_domains, i);
|
| 4955 | group = cpu_to_phys_group(i);
|
| 4956 | *sd = SD_CPU_INIT;
|
| 4957 | sd->span = nodemask;
|
| 4958 | sd->parent = p;
|
| 4959 | sd->groups = &sched_group_phys[group];
|
| 4960 |
|
| 4961 | #ifdef CONFIG_SCHED_SMT
|
| 4962 | p = sd;
|
| 4963 | sd = &per_cpu(cpu_domains, i);
|
| 4964 | group = cpu_to_cpu_group(i);
|
| 4965 | *sd = SD_SIBLING_INIT;
|
| 4966 | sd->span = cpu_sibling_map[i];
|
| 4967 | cpus_and(sd->span, sd->span, *cpu_map);
|
| 4968 | sd->parent = p;
|
| 4969 | sd->groups = &sched_group_cpus[group];
|
| 4970 | #endif
|
| 4971 | }
|
| 4972 |
|
| 4973 | #ifdef CONFIG_SCHED_SMT
|
| 4974 |
|
| 4975 | for_each_online_cpu(i) {
|
| 4976 | cpumask_t this_sibling_map = cpu_sibling_map[i];
|
| 4977 | cpus_and(this_sibling_map, this_sibling_map, *cpu_map);
|
| 4978 | if (i != first_cpu(this_sibling_map))
|
| 4979 | continue;
|
| 4980 |
|
| 4981 | init_sched_build_groups(sched_group_cpus, this_sibling_map,
|
| 4982 | &cpu_to_cpu_group);
|
| 4983 | }
|
| 4984 | #endif
|
| 4985 |
|
| 4986 |
|
| 4987 | for (i = 0; i < MAX_NUMNODES; i++) {
|
| 4988 | cpumask_t nodemask = node_to_cpumask(i);
|
| 4989 |
|
| 4990 | cpus_and(nodemask, nodemask, *cpu_map);
|
| 4991 | if (cpus_empty(nodemask))
|
| 4992 | continue;
|
| 4993 |
|
| 4994 | init_sched_build_groups(sched_group_phys, nodemask,
|
| 4995 | &cpu_to_phys_group);
|
| 4996 | }
|
| 4997 |
|
| 4998 | #ifdef CONFIG_NUMA
|
| 4999 |
|
| 5000 | init_sched_build_groups(sched_group_nodes, *cpu_map,
|
| 5001 | &cpu_to_node_group);
|
| 5002 | #endif
|
| 5003 |
|
| 5004 |
|
| 5005 | for_each_cpu_mask(i, *cpu_map) {
|
| 5006 | int power;
|
| 5007 | struct sched_domain *sd;
|
| 5008 | #ifdef CONFIG_SCHED_SMT
|
| 5009 | sd = &per_cpu(cpu_domains, i);
|
| 5010 | power = SCHED_LOAD_SCALE;
|
| 5011 | sd->groups->cpu_power = power;
|
| 5012 | #endif
|
| 5013 |
|
| 5014 | sd = &per_cpu(phys_domains, i);
|
| 5015 | power = SCHED_LOAD_SCALE + SCHED_LOAD_SCALE *
|
| 5016 | (cpus_weight(sd->groups->cpumask)-1) / 10;
|
| 5017 | sd->groups->cpu_power = power;
|
| 5018 |
|
| 5019 | #ifdef CONFIG_NUMA
|
| 5020 | if (i == first_cpu(sd->groups->cpumask)) {
|
| 5021 |
|
| 5022 | sd = &per_cpu(node_domains, i);
|
| 5023 | sd->groups->cpu_power += power;
|
| 5024 | }
|
| 5025 | #endif
|
| 5026 | }
|
| 5027 |
|
| 5028 |
|
| 5029 | for_each_cpu_mask(i, *cpu_map) {
|
| 5030 | struct sched_domain *sd;
|
| 5031 | #ifdef CONFIG_SCHED_SMT
|
| 5032 | sd = &per_cpu(cpu_domains, i);
|
| 5033 | #else
|
| 5034 | sd = &per_cpu(phys_domains, i);
|
| 5035 | #endif
|
| 5036 | cpu_attach_domain(sd, i);
|
| 5037 | }
|
| 5038 | }
|
| 5039 | |
| 5040 | * Set up scheduler domains and groups. Callers must hold the hotplug lock.
|
| 5041 |
|
| 5042 | static void arch_init_sched_domains(cpumask_t *cpu_map)
|
| 5043 | {
|
| 5044 | cpumask_t cpu_default_map;
|
| 5045 |
|
| 5046 | #if defined(CONFIG_SCHED_SMT) && defined(CONFIG_NUMA)
|
| 5047 | check_sibling_maps();
|
| 5048 | #endif
|
| 5049 | |
| 5050 | * Setup mask for cpus without special case scheduling requirements.
|
| 5051 | * For now this just excludes isolated cpus, but could be used to
|
| 5052 | * exclude other special cases in the future.
|
| 5053 |
|
| 5054 | cpus_andnot(cpu_default_map, *cpu_map, cpu_isolated_map);
|
| 5055 |
|
| 5056 | build_sched_domains(&cpu_default_map);
|
| 5057 | }
|
| 5058 |
|
| 5059 | static void arch_destroy_sched_domains(const cpumask_t *cpu_map)
|
| 5060 | {
|
| 5061 |
|
| 5062 | }
|
| 5063 |
|
| 5064 | #endif
|
| 5065 |
|
| 5066 | |
| 5067 | * Detach sched domains from a group of cpus specified in cpu_map
|
| 5068 | * These cpus will now be attached to the NULL domain
|
| 5069 |
|
| 5070 | static inline void detach_destroy_domains(const cpumask_t *cpu_map)
|
| 5071 | {
|
| 5072 | int i;
|
| 5073 |
|
| 5074 | for_each_cpu_mask(i, *cpu_map)
|
| 5075 | cpu_attach_domain(NULL, i);
|
| 5076 | synchronize_sched();
|
| 5077 | arch_destroy_sched_domains(cpu_map);
|
| 5078 | }
|
| 5079 |
|
| 5080 | |
| 5081 | * Partition sched domains as specified by the cpumasks below.
|
| 5082 | * This attaches all cpus from the cpumasks to the NULL domain,
|
| 5083 | * waits for a RCU quiescent period, recalculates sched
|
| 5084 | * domain information and then attaches them back to the
|
| 5085 | * correct sched domains
|
| 5086 | * Call with hotplug lock held
|
| 5087 |
|
| 5088 | void partition_sched_domains(cpumask_t *partition1, cpumask_t *partition2)
|
| 5089 | {
|
| 5090 | cpumask_t change_map;
|
| 5091 |
|
| 5092 | cpus_and(*partition1, *partition1, cpu_online_map);
|
| 5093 | cpus_and(*partition2, *partition2, cpu_online_map);
|
| 5094 | cpus_or(change_map, *partition1, *partition2);
|
| 5095 |
|
| 5096 |
|
| 5097 | detach_destroy_domains(&change_map);
|
| 5098 | if (!cpus_empty(*partition1))
|
| 5099 | build_sched_domains(partition1);
|
| 5100 | if (!cpus_empty(*partition2))
|
| 5101 | build_sched_domains(partition2);
|
| 5102 | }
|
| 5103 |
|
| 5104 | #ifdef CONFIG_HOTPLUG_CPU
|
| 5105 | |
| 5106 | * Force a reinitialization of the sched domains hierarchy. The domains
|
| 5107 | * and groups cannot be updated in place without racing with the balancing
|
| 5108 | * code, so we temporarily attach all running cpus to the NULL domain
|
| 5109 | * which will prevent rebalancing while the sched domains are recalculated.
|
| 5110 |
|
| 5111 | static int update_sched_domains(struct notifier_block *nfb,
|
| 5112 | unsigned long action, void *hcpu)
|
| 5113 | {
|
| 5114 | switch (action) {
|
| 5115 | case CPU_UP_PREPARE:
|
| 5116 | case CPU_DOWN_PREPARE:
|
| 5117 | detach_destroy_domains(&cpu_online_map);
|
| 5118 | return NOTIFY_OK;
|
| 5119 |
|
| 5120 | case CPU_UP_CANCELED:
|
| 5121 | case CPU_DOWN_FAILED:
|
| 5122 | case CPU_ONLINE:
|
| 5123 | case CPU_DEAD:
|
| 5124 | |
| 5125 | * Fall through and re-initialise the domains.
|
| 5126 |
|
| 5127 | break;
|
| 5128 | default:
|
| 5129 | return NOTIFY_DONE;
|
| 5130 | }
|
| 5131 |
|
| 5132 |
|
| 5133 | arch_init_sched_domains(&cpu_online_map);
|
| 5134 |
|
| 5135 | return NOTIFY_OK;
|
| 5136 | }
|
| 5137 | #endif
|
| 5138 |
|
| 5139 | void __init sched_init_smp(void)
|
| 5140 | {
|
| 5141 | lock_cpu_hotplug();
|
| 5142 | arch_init_sched_domains(&cpu_online_map);
|
| 5143 | unlock_cpu_hotplug();
|
| 5144 |
|
| 5145 | hotcpu_notifier(update_sched_domains, 0);
|
| 5146 | }
|
| 5147 | #else
|
| 5148 | void __init sched_init_smp(void)
|
| 5149 | {
|
| 5150 | }
|
| 5151 | #endif
|
| 5152 |
|
| 5153 | int in_sched_functions(unsigned long addr)
|
| 5154 | {
|
| 5155 |
|
| 5156 | extern char __sched_text_start[], __sched_text_end[];
|
| 5157 | return in_lock_functions(addr) ||
|
| 5158 | (addr >= (unsigned long)__sched_text_start
|
| 5159 | && addr < (unsigned long)__sched_text_end);
|
| 5160 | }
|
| 5161 |
|
| 5162 | void __init sched_init(void)
|
| 5163 | {
|
| 5164 | runqueue_t *rq;
|
| 5165 | int i, j, k;
|
| 5166 |
|
| 5167 | for (i = 0; i < NR_CPUS; i++) {
|
| 5168 | prio_array_t *array;
|
| 5169 |
|
| 5170 | rq = cpu_rq(i);
|
| 5171 | spin_lock_init(&rq->lock);
|
| 5172 | rq->nr_running = 0;
|
| 5173 | rq->active = rq->arrays;
|
| 5174 | rq->expired = rq->arrays + 1;
|
| 5175 | rq->best_expired_prio = MAX_PRIO;
|
| 5176 |
|
| 5177 | #ifdef CONFIG_SMP
|
| 5178 | rq->sd = NULL;
|
| 5179 | for (j = 1; j < 3; j++)
|
| 5180 | rq->cpu_load[j] = 0;
|
| 5181 | rq->active_balance = 0;
|
| 5182 | rq->push_cpu = 0;
|
| 5183 | rq->migration_thread = NULL;
|
| 5184 | INIT_LIST_HEAD(&rq->migration_queue);
|
| 5185 | #endif
|
| 5186 | atomic_set(&rq->nr_iowait, 0);
|
| 5187 |
|
| 5188 | for (j = 0; j < 2; j++) {
|
| 5189 | array = rq->arrays + j;
|
| 5190 | for (k = 0; k < MAX_PRIO; k++) {
|
| 5191 | INIT_LIST_HEAD(array->queue + k);
|
| 5192 | __clear_bit(k, array->bitmap);
|
| 5193 | }
|
| 5194 | |
| 5195 | __set_bit(MAX_PRIO, array->bitmap);
|
| 5196 | }
|
| 5197 | }
|
| 5198 |
|
| 5199 | |
| 5200 | * The boot idle thread does lazy MMU switching as well:
|
| 5201 |
|
| 5202 | atomic_inc(&init_mm.mm_count);
|
| 5203 | enter_lazy_tlb(&init_mm, current);
|
| 5204 |
|
| 5205 | |
| 5206 | * Make us the idle thread. Technically, schedule() should not be
|
| 5207 | * called from this thread, however somewhere below it might be,
|
| 5208 | * but because we are the idle thread, we just pick up running again
|
| 5209 | * when this runqueue becomes "idle".
|
| 5210 |
|
| 5211 | init_idle(current, smp_processor_id());
|
| 5212 | }
|
| 5213 |
|
| 5214 | #ifdef CONFIG_DEBUG_SPINLOCK_SLEEP
|
| 5215 | void __might_sleep(char *file, int line)
|
| 5216 | {
|
| 5217 | #if defined(in_atomic)
|
| 5218 | static unsigned long prev_jiffy;
|
| 5219 |
|
| 5220 | if ((in_atomic() || irqs_disabled()) &&
|
| 5221 | system_state == SYSTEM_RUNNING && !oops_in_progress) {
|
| 5222 | if (time_before(jiffies, prev_jiffy + HZ) && prev_jiffy)
|
| 5223 | return;
|
| 5224 | prev_jiffy = jiffies;
|
| 5225 | printk(KERN_ERR "Debug: sleeping function called from invalid"
|
| 5226 | " context at %s:%d\n", file, line);
|
| 5227 | printk("in_atomic():%d, irqs_disabled():%d\n",
|
| 5228 | in_atomic(), irqs_disabled());
|
| 5229 | dump_stack();
|
| 5230 | }
|
| 5231 | #endif
|
| 5232 | }
|
| 5233 | EXPORT_SYMBOL(__might_sleep);
|
| 5234 | #endif
|
| 5235 |
|
| 5236 | #ifdef CONFIG_MAGIC_SYSRQ
|
| 5237 | void normalize_rt_tasks(void)
|
| 5238 | {
|
| 5239 | struct task_struct *p;
|
| 5240 | prio_array_t *array;
|
| 5241 | unsigned long flags;
|
| 5242 | runqueue_t *rq;
|
| 5243 |
|
| 5244 | read_lock_irq(&tasklist_lock);
|
| 5245 | for_each_process (p) {
|
| 5246 | if (!rt_task(p))
|
| 5247 | continue;
|
| 5248 |
|
| 5249 | rq = task_rq_lock(p, &flags);
|
| 5250 |
|
| 5251 | array = p->array;
|
| 5252 | if (array)
|
| 5253 | deactivate_task(p, task_rq(p));
|
| 5254 | __setscheduler(p, SCHED_NORMAL, 0);
|
| 5255 | if (array) {
|
| 5256 | __activate_task(p, task_rq(p));
|
| 5257 | resched_task(rq->curr);
|
| 5258 | }
|
| 5259 |
|
| 5260 | task_rq_unlock(rq, &flags);
|
| 5261 | }
|
| 5262 | read_unlock_irq(&tasklist_lock);
|
| 5263 | }
|
| 5264 |
|
| 5265 | #endif
|